Add Cosmos3-Edge LIBERO checkpoints

.gitattributes

@@ -49,3 +49,7 @@ gr00t_qwen3vl_0.6b_libero_spatial_full_finetune_bs64/tokenizer/tokenizer.json fi
 pi05_paligemma_robocasa_full_data_full_finetune_bs256/pi05_paligemma_robocasa_full_data_full_finetune_2026_07_21_01_22_59.jsonl filter=lfs diff=lfs merge=lfs -text
 pi05_paligemma_robocasa_full_data_full_finetune_bs256/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 fastwam_base/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+dit4dit_libero_all_full_finetune_bc071270ff2_bs256/dit4dit_libero_all_full_finetune_2026_08_12_20_51_03.jsonl filter=lfs diff=lfs merge=lfs -text
+dit4dit_libero_all_full_finetune_bc071270ff2_bs256/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+cosmos3_edge_libero_full_finetune_bs2048/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+cosmos3_nano_libero_full_finetune_bs2048/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -84,6 +84,7 @@ FluxVLA Engine supports multiple VLA model families, each optimized for differen
 | Model | Training Data | Performance | Download |
 |:---:|:---:|:---:|:---|
 | PI0.5 PaliGemma | LIBERO-10 | 96.0% avg | [Hugging Face](https://huggingface.co/limxdynamics/FluxVLAEngine/tree/main/pi05_paligemma_libero_10_full_finetune_bs64) |
+| Cosmos3-Edge | LIBERO benchmark suites (mixed training) | 94.4% avg | [Hugging Face](https://huggingface.co/limxdynamics/FluxVLAEngine/tree/main/cosmos3_edge_libero_full_finetune_bs2048) |
 | GR00T Eagle 3B | LIBERO-10 | 89.4% avg | [Hugging Face](https://huggingface.co/limxdynamics/FluxVLAEngine/tree/main/gr00t_eagle_3b_libero_10_full_finetune_bs64) |
 
 ## 3. Evaluation Results
@@ -98,8 +99,9 @@ FluxVLA demonstrates state-of-the-art performance across multiple LIBERO benchma
 |:---:|:---:|:---:|:---:|:---:|:---:|
 | **FluxVLA (Pi0.5)** | 98.6 | 99.0 | 97.8 | 96.0±1.0 | 97.85 |
 | **FluxVLA (Qwen3VL+GR00T)** | 98.6 | 99.6 | 95.6 | 92.2±1.8 | 96.50 |
-| **FluxVLA (GR00T)** | 96.2 | 96.8 | 93.4 | 89.4±1.5 | 93.95 |
 | **FluxVLA (DreamZero)** | 96.8 | 97.4 | 90.8±1.5 | 93.6 | 94.65 |
+| **FluxVLA (Cosmos3-Edge)** | 95.6 | 95.6 | 91.6 | 94.8 | 94.4 |
+| **FluxVLA (GR00T)** | 96.2 | 96.8 | 93.4 | 89.4±1.5 | 93.95 |
 
 </div>
 
@@ -392,9 +394,9 @@ If you use FluxVLA in your research, please cite:
 
 ## 10. License
 
-This project is released under the **Apache-2.0 License**. See [LICENSE](LICENSE) for details.
+The FluxVLA code is released under the **Apache-2.0 License**. See [LICENSE](LICENSE) for details.
 
-Both the code repository and the model weights are freely available for research and commercial use under the Apache-2.0 License.
+Model checkpoints derived from third-party base models remain subject to their respective licenses. The Cosmos3-Edge checkpoints are derived from NVIDIA Cosmos3-Edge and are subject to the [Open Model Development and Weight License 1.1 (OpenMDW 1.1)](https://openmdw.ai/license/1-1/).
 
 ## 11. Acknowledgements
 
@@ -430,4 +432,4 @@ You can also:
 - 🌐 GitHub: <https://github.com/limxdynamics/FluxVLA>
 - 🤗 Hugging Face: <https://huggingface.co/limxdynamics/FluxVLAEngine>
 - 📖 English Documentation: <https://fluxvla.limxdynamics.com>
-- 📖 中文文档: <https://fluxvla.limxdynamics.com/zh/>
+- 📖 中文文档: <https://fluxvla.limxdynamics.com/zh/>

cosmos3_edge_libero_full_finetune_bs2048/checkpoints/step-005000-epoch-22-loss=0.5737.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5b028fd50784f5070a390de63f9df06187c0fc8a4ab8e415113a68a124e8a0e
+size 16297490208

cosmos3_edge_libero_full_finetune_bs2048/config.json

@@ -0,0 +1,695 @@
+{
+  "eval": {
+    "dataset": {
+      "extra_tensor_keys": [
+        "conditioning_fps",
+        "prepend_state_to_action"
+      ],
+      "img_buffer_len": 1,
+      "transforms": [
+        {
+          "embodiment_id": 5,
+          "img_keys": [
+            "agentview_image",
+            "robot0_eye_in_hand_image"
+          ],
+          "type": "ProcessLiberoEvalInputs"
+        },
+        {
+          "conditioning_fps": 20.0,
+          "prepend_state_to_action": false,
+          "type": "SetCosmos3ActionMetadata"
+        },
+        {
+          "image_resize_strategy": "resize-crop",
+          "input_sizes": [
+            [
+              3,
+              256,
+              256
+            ],
+            [
+              3,
+              256,
+              256
+            ]
+          ],
+          "means": [
+            [
+              127.5,
+              127.5,
+              127.5
+            ],
+            [
+              127.5,
+              127.5,
+              127.5
+            ]
+          ],
+          "stds": [
+            [
+              127.5,
+              127.5,
+              127.5
+            ],
+            [
+              127.5,
+              127.5,
+              127.5
+            ]
+          ],
+          "type": "TransformImage"
+        },
+        {
+          "action_metadata": {
+            "append_viewpoint": false,
+            "conditioning_fps": 20.0,
+            "frame_window_size": 17,
+            "video_height": 192,
+            "video_width": 320
+          },
+          "cfg_dropout_rate": 0.0,
+          "format_prompt_as_json": true,
+          "max_len": 512,
+          "output_attention_mask_key": "lang_masks",
+          "output_key": "lang_tokens",
+          "tokenizer": {
+            "model_max_length": 4096,
+            "model_path": "./checkpoints/Cosmos3-Edge",
+            "padding_side": "right",
+            "trust_remote_code": true,
+            "type": "PretrainedTokenizer"
+          },
+          "type": "ProcessCosmos3Prompt"
+        },
+        {
+          "gripper_key": "robot0_gripper_qpos",
+          "norm_type": "mean_std",
+          "out_key": "states",
+          "pos_key": "robot0_eef_pos",
+          "quat_key": "robot0_eef_quat",
+          "state_dim": 64,
+          "type": "LiberoProprioFromInputs"
+        },
+        {
+          "frame_window_size": 1,
+          "num_views": 2,
+          "tile_direction": "horizontal",
+          "type": "PrepareVideo"
+        },
+        {
+          "height": 192,
+          "type": "ResizeAndReflectPad",
+          "width": 320
+        }
+      ],
+      "type": "LiberoParquetEvalDataset"
+    },
+    "denormalize_action": {
+      "action_dim": 10,
+      "norm_type": "quantile_rot",
+      "type": "DenormalizeLiberoFramewiseRot6DAction"
+    },
+    "enable_mixed_precision_training": true,
+    "eval_chunk_size": 16,
+    "inference_seed": 7,
+    "mixed_precision_dtype": "bf16",
+    "model_family": "cosmos3",
+    "norm_stats_key": "all_libero_no_noops",
+    "num_inference_steps": 30,
+    "num_steps_wait": 10,
+    "num_trials_per_task": 50,
+    "resize_size": 256,
+    "run_id_suffix": "mixed-v21-shift10-after-train",
+    "seed": 7,
+    "task_ids": null,
+    "task_suite_name": [
+      "libero_spatial",
+      "libero_object",
+      "libero_goal",
+      "libero_10"
+    ],
+    "type": "LiberoEvalRunner"
+  },
+  "inference_model": {
+    "action_horizon": 16,
+    "action_in_proj": {
+      "input_size": 64,
+      "num_domains": 32,
+      "output_size": 2048,
+      "type": "DomainAwareLinear"
+    },
+    "action_out_proj": {
+      "input_size": 2048,
+      "num_domains": 32,
+      "output_size": 64,
+      "type": "DomainAwareLinear"
+    },
+    "base_fps": 24.0,
+    "enable_fps_modulation": true,
+    "enable_vision_loss": true,
+    "freeze_non_moe_vlm_backbone": true,
+    "freeze_vlm_backbone": false,
+    "latent_patch_size": 2,
+    "max_action_dim": 64,
+    "name_mapping": {
+      ".self_attn.k_norm_moe_gen.": ".self_attn.norm_added_k.",
+      ".self_attn.k_proj.": ".self_attn.to_k.",
+      ".self_attn.k_proj_moe_gen.": ".self_attn.add_k_proj.",
+      ".self_attn.o_proj.": ".self_attn.to_out.",
+      ".self_attn.o_proj_moe_gen.": ".self_attn.to_add_out.",
+      ".self_attn.q_norm_moe_gen.": ".self_attn.norm_added_q.",
+      ".self_attn.q_proj.": ".self_attn.to_q.",
+      ".self_attn.q_proj_moe_gen.": ".self_attn.add_q_proj.",
+      ".self_attn.v_proj.": ".self_attn.to_v.",
+      ".self_attn.v_proj_moe_gen.": ".self_attn.add_v_proj.",
+      "action_in_proj.": "action_proj_in.",
+      "action_modality_embed": "action_modality_embed",
+      "action_out_proj.": "action_proj_out.",
+      "time_embedder.mlp.0.": "time_embedder.linear_1.",
+      "time_embedder.mlp.2.": "time_embedder.linear_2.",
+      "vision_in_proj.projector.": "proj_in.",
+      "vision_out_proj.projector.": "proj_out.",
+      "vlm_backbone.lm_head.weight": "lm_head.weight",
+      "vlm_backbone.model.language_model.embed_tokens.weight": "embed_tokens.weight",
+      "vlm_backbone.model.language_model.layers.": "layers.",
+      "vlm_backbone.model.language_model.norm.weight": "norm.weight",
+      "vlm_backbone.model.language_model.norm_moe_gen.weight": "norm_moe_gen.weight"
+    },
+    "num_embodiment_domains": 32,
+    "ori_action_dim": 10,
+    "packed_attention_backend": "flash2",
+    "position_embedding_type": "unified_3d_mrope",
+    "pretrained_name_or_path": "./checkpoints/Cosmos3-Edge/transformer",
+    "rectified_flow_inference_config": {
+      "num_steps": 30,
+      "num_train_timesteps": 1000,
+      "scheduler_type": "unipc",
+      "shift": 10.0,
+      "use_dynamic_shifting": false,
+      "use_karras_sigmas": false
+    },
+    "rectified_flow_training_config": {
+      "action_loss_weight": 10.0,
+      "independent_action_schedule": false,
+      "normalize_loss_by_active": false,
+      "shift": {
+        "256": 3,
+        "480": 5,
+        "720": 10
+      },
+      "shift_action": null,
+      "train_time_action_distribution": "logitnormal",
+      "train_time_image_distribution": "logitnormal",
+      "train_time_video_distribution": "waver",
+      "train_time_weight": "uniform",
+      "use_discrete_rf": false,
+      "use_dynamic_shift": false,
+      "use_high_sigma_strategy": false,
+      "use_high_sigma_strategy_action": false,
+      "vision_loss_weight": 10.0
+    },
+    "reinitialize_action_policy": true,
+    "special_tokens": {
+      "end_of_generation": 21,
+      "eos_token_id": 11,
+      "start_of_generation": 20
+    },
+    "strict_mapping": true,
+    "timestep_scale": 0.001,
+    "type": "Cosmos3FlowMatching",
+    "unified_3d_mrope_reset_spatial_ids": true,
+    "unified_3d_mrope_temporal_modality_margin": 15000,
+    "vision_in_proj": {
+      "in_dim": 192,
+      "out_dim": 2048,
+      "type": "LinearProjector"
+    },
+    "vision_latent_dim": 48,
+    "vision_out_proj": {
+      "in_dim": 2048,
+      "out_dim": 192,
+      "type": "LinearProjector"
+    },
+    "vision_vae": {
+      "encode_exact_durations": [
+        17
+      ],
+      "pretrained_name_or_path": "./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth",
+      "type": "Cosmos3Wan22VAE"
+    },
+    "vlm_backbone": {
+      "include_visual": false,
+      "skip_init_weights": true,
+      "type": "Cosmos3MoTBackbone",
+      "vlm_config": {
+        "attention_bias": false,
+        "bos_token_id": 1,
+        "enable_mrope": true,
+        "eos_token_id": 11,
+        "head_dim": 128,
+        "hidden_size": 2048,
+        "intermediate_size": 9216,
+        "layer_norm_epsilon": 1e-05,
+        "max_position_embeddings": 131072,
+        "mlp_bias": false,
+        "mlp_hidden_act": "relu2",
+        "model_type": "nemotron_3_dense_vl_text",
+        "mrope_section": [
+          24,
+          20,
+          20
+        ],
+        "num_attention_heads": 16,
+        "num_hidden_layers": 28,
+        "num_key_value_heads": 8,
+        "pad_token_id": 11,
+        "rope_theta": 100000000.0,
+        "tie_word_embeddings": false,
+        "use_und_k_norm_for_gen": true,
+        "vocab_size": 131072
+      }
+    }
+  },
+  "model": {
+    "action_horizon": 16,
+    "action_in_proj": {
+      "input_size": 64,
+      "num_domains": 32,
+      "output_size": 2048,
+      "type": "DomainAwareLinear"
+    },
+    "action_out_proj": {
+      "input_size": 2048,
+      "num_domains": 32,
+      "output_size": 64,
+      "type": "DomainAwareLinear"
+    },
+    "base_fps": 24.0,
+    "enable_fps_modulation": true,
+    "enable_vision_loss": true,
+    "freeze_non_moe_vlm_backbone": true,
+    "freeze_vlm_backbone": false,
+    "latent_patch_size": 2,
+    "max_action_dim": 64,
+    "name_mapping": {
+      ".self_attn.k_norm_moe_gen.": ".self_attn.norm_added_k.",
+      ".self_attn.k_proj.": ".self_attn.to_k.",
+      ".self_attn.k_proj_moe_gen.": ".self_attn.add_k_proj.",
+      ".self_attn.o_proj.": ".self_attn.to_out.",
+      ".self_attn.o_proj_moe_gen.": ".self_attn.to_add_out.",
+      ".self_attn.q_norm_moe_gen.": ".self_attn.norm_added_q.",
+      ".self_attn.q_proj.": ".self_attn.to_q.",
+      ".self_attn.q_proj_moe_gen.": ".self_attn.add_q_proj.",
+      ".self_attn.v_proj.": ".self_attn.to_v.",
+      ".self_attn.v_proj_moe_gen.": ".self_attn.add_v_proj.",
+      "action_in_proj.": "action_proj_in.",
+      "action_modality_embed": "action_modality_embed",
+      "action_out_proj.": "action_proj_out.",
+      "time_embedder.mlp.0.": "time_embedder.linear_1.",
+      "time_embedder.mlp.2.": "time_embedder.linear_2.",
+      "vision_in_proj.projector.": "proj_in.",
+      "vision_out_proj.projector.": "proj_out.",
+      "vlm_backbone.lm_head.weight": "lm_head.weight",
+      "vlm_backbone.model.language_model.embed_tokens.weight": "embed_tokens.weight",
+      "vlm_backbone.model.language_model.layers.": "layers.",
+      "vlm_backbone.model.language_model.norm.weight": "norm.weight",
+      "vlm_backbone.model.language_model.norm_moe_gen.weight": "norm_moe_gen.weight"
+    },
+    "num_embodiment_domains": 32,
+    "ori_action_dim": 10,
+    "packed_attention_backend": "flash2",
+    "position_embedding_type": "unified_3d_mrope",
+    "pretrained_name_or_path": "./checkpoints/Cosmos3-Edge/transformer",
+    "rectified_flow_inference_config": {
+      "num_steps": 30,
+      "num_train_timesteps": 1000,
+      "scheduler_type": "unipc",
+      "shift": 10.0,
+      "use_dynamic_shifting": false,
+      "use_karras_sigmas": false
+    },
+    "rectified_flow_training_config": {
+      "action_loss_weight": 10.0,
+      "independent_action_schedule": false,
+      "normalize_loss_by_active": false,
+      "shift": {
+        "256": 3,
+        "480": 5,
+        "720": 10
+      },
+      "shift_action": null,
+      "train_time_action_distribution": "logitnormal",
+      "train_time_image_distribution": "logitnormal",
+      "train_time_video_distribution": "waver",
+      "train_time_weight": "uniform",
+      "use_discrete_rf": false,
+      "use_dynamic_shift": false,
+      "use_high_sigma_strategy": false,
+      "use_high_sigma_strategy_action": false,
+      "vision_loss_weight": 10.0
+    },
+    "reinitialize_action_policy": true,
+    "special_tokens": {
+      "end_of_generation": 21,
+      "eos_token_id": 11,
+      "start_of_generation": 20
+    },
+    "strict_mapping": true,
+    "timestep_scale": 0.001,
+    "type": "Cosmos3FlowMatching",
+    "unified_3d_mrope_reset_spatial_ids": true,
+    "unified_3d_mrope_temporal_modality_margin": 15000,
+    "vision_in_proj": {
+      "in_dim": 192,
+      "out_dim": 2048,
+      "type": "LinearProjector"
+    },
+    "vision_latent_dim": 48,
+    "vision_out_proj": {
+      "in_dim": 2048,
+      "out_dim": 192,
+      "type": "LinearProjector"
+    },
+    "vision_vae": {
+      "encode_exact_durations": [
+        17
+      ],
+      "pretrained_name_or_path": "./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth",
+      "type": "Cosmos3Wan22VAE"
+    },
+    "vlm_backbone": {
+      "include_visual": false,
+      "skip_init_weights": true,
+      "type": "Cosmos3MoTBackbone",
+      "vlm_config": {
+        "attention_bias": false,
+        "bos_token_id": 1,
+        "enable_mrope": true,
+        "eos_token_id": 11,
+        "head_dim": 128,
+        "hidden_size": 2048,
+        "intermediate_size": 9216,
+        "layer_norm_epsilon": 1e-05,
+        "max_position_embeddings": 131072,
+        "mlp_bias": false,
+        "mlp_hidden_act": "relu2",
+        "model_type": "nemotron_3_dense_vl_text",
+        "mrope_section": [
+          24,
+          20,
+          20
+        ],
+        "num_attention_heads": 16,
+        "num_hidden_layers": 28,
+        "num_key_value_heads": 8,
+        "pad_token_id": 11,
+        "rope_theta": 100000000.0,
+        "tie_word_embeddings": false,
+        "use_und_k_norm_for_gen": true,
+        "vocab_size": 131072
+      }
+    }
+  },
+  "runner": {
+    "change_key_name": false,
+    "collator": {
+      "list_keys": [
+        "sequence_plan"
+      ],
+      "meta_keys": [
+        "task_description",
+        "stats",
+        "info",
+        "timestamp"
+      ],
+      "pad_id": 11,
+      "sequence_keys": [
+        "text_token_ids"
+      ],
+      "tensor_keys": [
+        "images",
+        "actions",
+        "embodiment_ids",
+        "raw_action_dim",
+        "conditioning_fps",
+        "action_fps"
+      ],
+      "type": "Cosmos3Collator"
+    },
+    "enable_gradient_checkpointing": true,
+    "enable_mixed_precision_training": true,
+    "grad_accumulation_steps": 4,
+    "lr_scheduler": {
+      "cycle_length": 16000,
+      "type": "linear-warmup+linear-decay",
+      "warmup_steps": 500
+    },
+    "max_epochs": null,
+    "max_grad_norm": 1.0,
+    "max_keep_ckpts": 2,
+    "max_steps": 5000,
+    "metric": {
+      "active_trackers": [
+        "jsonl",
+        "wandb"
+      ],
+      "grad_accumulation_steps": 4,
+      "run_dir": "work_dirs",
+      "type": "VLAMetric",
+      "window_size": 1
+    },
+    "mixed_precision_dtype": "bf16",
+    "optimizer": {
+      "betas": [
+        0.9,
+        0.99
+      ],
+      "eps": 1e-08,
+      "exclude_1d_from_weight_decay": false,
+      "fused": true,
+      "lr": 5e-05,
+      "paramwise_learning_rate": {
+        "action_in_proj.": 0.00025,
+        "action_modality_embed": 0.00025,
+        "action_out_proj.": 0.00025
+      },
+      "type": "AdamW",
+      "weight_decay": 0.05
+    },
+    "sampler": null,
+    "save_iter_interval": 500,
+    "sharding_strategy": "full-shard",
+    "tokenizer": {
+      "model_max_length": 4096,
+      "model_path": "./checkpoints/Cosmos3-Edge",
+      "padding_side": "right",
+      "trust_remote_code": true,
+      "type": "PretrainedTokenizer"
+    },
+    "type": "FSDPTrainRunner"
+  },
+  "seed": 7,
+  "train_dataloader": {
+    "dataset": {
+      "datasets": {
+        "action_key": "action",
+        "action_window_size": 16,
+        "data_root_path": [
+          "./datasets/libero_spatial_no_noops_lerobotv2.1",
+          "./datasets/libero_spatial_no_noops_lerobotv2.1",
+          "./datasets/libero_object_no_noops_lerobotv2.1",
+          "./datasets/libero_object_no_noops_lerobotv2.1",
+          "./datasets/libero_goal_no_noops_lerobotv2.1",
+          "./datasets/libero_goal_no_noops_lerobotv2.1",
+          "./datasets/libero_10_no_noops_lerobotv2.1"
+        ],
+        "frame_window_size": 17,
+        "require_full_window": true,
+        "statistic_name": "all_libero_no_noops",
+        "transforms": [
+          {
+            "embodiment_id": 5,
+            "name_mappings": {
+              "actions": [
+                "actions"
+              ],
+              "observation.state": [
+                "states"
+              ]
+            },
+            "parquet_keys": [
+              "observation.state",
+              "timestamp",
+              "actions",
+              "info",
+              "stats"
+            ],
+            "type": "ProcessParquetInputs",
+            "video_keys": [
+              "observation.images.image",
+              "observation.images.wrist_image"
+            ]
+          },
+          {
+            "type": "LiberoFramewiseActionToRot6D"
+          },
+          {
+            "action_metadata": {
+              "append_viewpoint": false,
+              "conditioning_fps": 20.0,
+              "frame_window_size": 17,
+              "video_height": 192,
+              "video_width": 320
+            },
+            "cfg_dropout_rate": 0.1,
+            "format_prompt_as_json": true,
+            "max_len": 512,
+            "tokenizer": {
+              "model_max_length": 4096,
+              "model_path": "./checkpoints/Cosmos3-Edge",
+              "padding_side": "right",
+              "trust_remote_code": true,
+              "type": "PretrainedTokenizer"
+            },
+            "type": "ProcessCosmos3Prompt"
+          },
+          {
+            "height": 256,
+            "type": "ResizeImages",
+            "width": 256
+          },
+          {
+            "type": "SimpleNormalizeImages"
+          },
+          {
+            "action_dim": 64,
+            "action_key": "action",
+            "action_norm_type": "quantile",
+            "state_dim": 64,
+            "state_key": "proprio",
+            "state_norm_type": "none",
+            "type": "NormalizeStatesAndActions"
+          },
+          {
+            "conditioning_fps": 20.0,
+            "frame_window_size": 17,
+            "mode": "wam",
+            "prepend_state_to_action": false,
+            "raw_action_dim": 10,
+            "type": "BuildCosmos3Sequence"
+          },
+          {
+            "frame_window_size": 17,
+            "num_views": 2,
+            "tile_direction": "horizontal",
+            "type": "PrepareVideo"
+          },
+          {
+            "height": 192,
+            "type": "ResizeAndReflectPad",
+            "width": 320
+          }
+        ],
+        "type": "ParquetDataset",
+        "use_delta": false,
+        "window_start_idx": 0
+      },
+      "name_mappings": {
+        "action": [
+          "action"
+        ],
+        "observation.state": [
+          "proprio"
+        ]
+      },
+      "statistic_keys": [
+        "observation.state",
+        "timestamp",
+        "action"
+      ],
+      "statistic_name": "all_libero_no_noops",
+      "statistics_overrides": {
+        "all_libero_no_noops": {
+          "action": {
+            "max": [
+              0.9375,
+              0.9375,
+              0.9375,
+              1.0,
+              0.368853,
+              0.341214,
+              0.356395,
+              1.0,
+              0.348251,
+              1.0
+            ],
+            "mean": [
+              0.050704,
+              0.097407,
+              -0.094833,
+              0.994873,
+              -0.004579,
+              -0.004288,
+              0.004389,
+              0.996104,
+              0.001109,
+              0.476725
+            ],
+            "min": [
+              -0.9375,
+              -0.9375,
+              -0.9375,
+              0.902028,
+              -0.356085,
+              -0.367416,
+              -0.370434,
+              0.921907,
+              -0.255,
+              0.0
+            ],
+            "q01": [
+              -0.723214,
+              -0.808929,
+              -0.9375,
+              0.934955,
+              -0.223431,
+              -0.189878,
+              -0.334735,
+              0.938516,
+              -0.107736,
+              0.0
+            ],
+            "q99": [
+              0.9375,
+              0.870536,
+              0.9375,
+              1.0,
+              0.331,
+              0.163153,
+              0.226216,
+              1.0,
+              0.127158,
+              1.0
+            ],
+            "std": [
+              0.333621,
+              0.387175,
+              0.45714,
+              0.010807,
+              0.077802,
+              0.063386,
+              0.078571,
+              0.009994,
+              0.038504,
+              0.49946
+            ]
+          }
+        }
+      },
+      "type": "DistributedRepeatingDataset"
+    },
+    "per_device_batch_size": 32,
+    "per_device_num_workers": 4
+  }
+}

cosmos3_edge_libero_full_finetune_bs2048/config.yaml

@@ -0,0 +1,569 @@
+eval:
+  dataset:
+    extra_tensor_keys:
+    - conditioning_fps
+    - prepend_state_to_action
+    img_buffer_len: 1
+    transforms:
+    - embodiment_id: 5
+      img_keys:
+      - agentview_image
+      - robot0_eye_in_hand_image
+      type: ProcessLiberoEvalInputs
+    - conditioning_fps: 20.0
+      prepend_state_to_action: false
+      type: SetCosmos3ActionMetadata
+    - image_resize_strategy: resize-crop
+      input_sizes:
+      - - 3
+        - 256
+        - 256
+      - - 3
+        - 256
+        - 256
+      means:
+      - - 127.5
+        - 127.5
+        - 127.5
+      - - 127.5
+        - 127.5
+        - 127.5
+      stds:
+      - - 127.5
+        - 127.5
+        - 127.5
+      - - 127.5
+        - 127.5
+        - 127.5
+      type: TransformImage
+    - action_metadata:
+        append_viewpoint: false
+        conditioning_fps: 20.0
+        frame_window_size: 17
+        video_height: 192
+        video_width: 320
+      cfg_dropout_rate: 0.0
+      format_prompt_as_json: true
+      max_len: 512
+      output_attention_mask_key: lang_masks
+      output_key: lang_tokens
+      tokenizer:
+        model_max_length: 4096
+        model_path: ./checkpoints/Cosmos3-Edge
+        padding_side: right
+        trust_remote_code: true
+        type: PretrainedTokenizer
+      type: ProcessCosmos3Prompt
+    - gripper_key: robot0_gripper_qpos
+      norm_type: mean_std
+      out_key: states
+      pos_key: robot0_eef_pos
+      quat_key: robot0_eef_quat
+      state_dim: 64
+      type: LiberoProprioFromInputs
+    - frame_window_size: 1
+      num_views: 2
+      tile_direction: horizontal
+      type: PrepareVideo
+    - height: 192
+      type: ResizeAndReflectPad
+      width: 320
+    type: LiberoParquetEvalDataset
+  denormalize_action:
+    action_dim: 10
+    norm_type: quantile_rot
+    type: DenormalizeLiberoFramewiseRot6DAction
+  enable_mixed_precision_training: true
+  eval_chunk_size: 16
+  inference_seed: 7
+  mixed_precision_dtype: bf16
+  model_family: cosmos3
+  norm_stats_key: all_libero_no_noops
+  num_inference_steps: 30
+  num_steps_wait: 10
+  num_trials_per_task: 50
+  resize_size: 256
+  run_id_suffix: mixed-v21-shift10-after-train
+  seed: 7
+  task_ids: null
+  task_suite_name:
+  - libero_spatial
+  - libero_object
+  - libero_goal
+  - libero_10
+  type: LiberoEvalRunner
+inference_model:
+  action_horizon: 16
+  action_in_proj:
+    input_size: 64
+    num_domains: 32
+    output_size: 2048
+    type: DomainAwareLinear
+  action_out_proj:
+    input_size: 2048
+    num_domains: 32
+    output_size: 64
+    type: DomainAwareLinear
+  base_fps: 24.0
+  enable_fps_modulation: true
+  enable_vision_loss: true
+  freeze_non_moe_vlm_backbone: true
+  freeze_vlm_backbone: false
+  latent_patch_size: 2
+  max_action_dim: 64
+  name_mapping:
+    .self_attn.k_norm_moe_gen.: .self_attn.norm_added_k.
+    .self_attn.k_proj.: .self_attn.to_k.
+    .self_attn.k_proj_moe_gen.: .self_attn.add_k_proj.
+    .self_attn.o_proj.: .self_attn.to_out.
+    .self_attn.o_proj_moe_gen.: .self_attn.to_add_out.
+    .self_attn.q_norm_moe_gen.: .self_attn.norm_added_q.
+    .self_attn.q_proj.: .self_attn.to_q.
+    .self_attn.q_proj_moe_gen.: .self_attn.add_q_proj.
+    .self_attn.v_proj.: .self_attn.to_v.
+    .self_attn.v_proj_moe_gen.: .self_attn.add_v_proj.
+    action_in_proj.: action_proj_in.
+    action_modality_embed: action_modality_embed
+    action_out_proj.: action_proj_out.
+    time_embedder.mlp.0.: time_embedder.linear_1.
+    time_embedder.mlp.2.: time_embedder.linear_2.
+    vision_in_proj.projector.: proj_in.
+    vision_out_proj.projector.: proj_out.
+    vlm_backbone.lm_head.weight: lm_head.weight
+    vlm_backbone.model.language_model.embed_tokens.weight: embed_tokens.weight
+    vlm_backbone.model.language_model.layers.: layers.
+    vlm_backbone.model.language_model.norm.weight: norm.weight
+    vlm_backbone.model.language_model.norm_moe_gen.weight: norm_moe_gen.weight
+  num_embodiment_domains: 32
+  ori_action_dim: 10
+  packed_attention_backend: flash2
+  position_embedding_type: unified_3d_mrope
+  pretrained_name_or_path: ./checkpoints/Cosmos3-Edge/transformer
+  rectified_flow_inference_config:
+    num_steps: 30
+    num_train_timesteps: 1000
+    scheduler_type: unipc
+    shift: 10.0
+    use_dynamic_shifting: false
+    use_karras_sigmas: false
+  rectified_flow_training_config:
+    action_loss_weight: 10.0
+    independent_action_schedule: false
+    normalize_loss_by_active: false
+    shift:
+      '256': 3
+      '480': 5
+      '720': 10
+    shift_action: null
+    train_time_action_distribution: logitnormal
+    train_time_image_distribution: logitnormal
+    train_time_video_distribution: waver
+    train_time_weight: uniform
+    use_discrete_rf: false
+    use_dynamic_shift: false
+    use_high_sigma_strategy: false
+    use_high_sigma_strategy_action: false
+    vision_loss_weight: 10.0
+  reinitialize_action_policy: true
+  special_tokens:
+    end_of_generation: 21
+    eos_token_id: 11
+    start_of_generation: 20
+  strict_mapping: true
+  timestep_scale: 0.001
+  type: Cosmos3FlowMatching
+  unified_3d_mrope_reset_spatial_ids: true
+  unified_3d_mrope_temporal_modality_margin: 15000
+  vision_in_proj:
+    in_dim: 192
+    out_dim: 2048
+    type: LinearProjector
+  vision_latent_dim: 48
+  vision_out_proj:
+    in_dim: 2048
+    out_dim: 192
+    type: LinearProjector
+  vision_vae:
+    encode_exact_durations:
+    - 17
+    pretrained_name_or_path: ./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
+    type: Cosmos3Wan22VAE
+  vlm_backbone:
+    include_visual: false
+    skip_init_weights: true
+    type: Cosmos3MoTBackbone
+    vlm_config:
+      attention_bias: false
+      bos_token_id: 1
+      enable_mrope: true
+      eos_token_id: 11
+      head_dim: 128
+      hidden_size: 2048
+      intermediate_size: 9216
+      layer_norm_epsilon: 1.0e-05
+      max_position_embeddings: 131072
+      mlp_bias: false
+      mlp_hidden_act: relu2
+      model_type: nemotron_3_dense_vl_text
+      mrope_section:
+      - 24
+      - 20
+      - 20
+      num_attention_heads: 16
+      num_hidden_layers: 28
+      num_key_value_heads: 8
+      pad_token_id: 11
+      rope_theta: 100000000.0
+      tie_word_embeddings: false
+      use_und_k_norm_for_gen: true
+      vocab_size: 131072
+model:
+  action_horizon: 16
+  action_in_proj:
+    input_size: 64
+    num_domains: 32
+    output_size: 2048
+    type: DomainAwareLinear
+  action_out_proj:
+    input_size: 2048
+    num_domains: 32
+    output_size: 64
+    type: DomainAwareLinear
+  base_fps: 24.0
+  enable_fps_modulation: true
+  enable_vision_loss: true
+  freeze_non_moe_vlm_backbone: true
+  freeze_vlm_backbone: false
+  latent_patch_size: 2
+  max_action_dim: 64
+  name_mapping:
+    .self_attn.k_norm_moe_gen.: .self_attn.norm_added_k.
+    .self_attn.k_proj.: .self_attn.to_k.
+    .self_attn.k_proj_moe_gen.: .self_attn.add_k_proj.
+    .self_attn.o_proj.: .self_attn.to_out.
+    .self_attn.o_proj_moe_gen.: .self_attn.to_add_out.
+    .self_attn.q_norm_moe_gen.: .self_attn.norm_added_q.
+    .self_attn.q_proj.: .self_attn.to_q.
+    .self_attn.q_proj_moe_gen.: .self_attn.add_q_proj.
+    .self_attn.v_proj.: .self_attn.to_v.
+    .self_attn.v_proj_moe_gen.: .self_attn.add_v_proj.
+    action_in_proj.: action_proj_in.
+    action_modality_embed: action_modality_embed
+    action_out_proj.: action_proj_out.
+    time_embedder.mlp.0.: time_embedder.linear_1.
+    time_embedder.mlp.2.: time_embedder.linear_2.
+    vision_in_proj.projector.: proj_in.
+    vision_out_proj.projector.: proj_out.
+    vlm_backbone.lm_head.weight: lm_head.weight
+    vlm_backbone.model.language_model.embed_tokens.weight: embed_tokens.weight
+    vlm_backbone.model.language_model.layers.: layers.
+    vlm_backbone.model.language_model.norm.weight: norm.weight
+    vlm_backbone.model.language_model.norm_moe_gen.weight: norm_moe_gen.weight
+  num_embodiment_domains: 32
+  ori_action_dim: 10
+  packed_attention_backend: flash2
+  position_embedding_type: unified_3d_mrope
+  pretrained_name_or_path: ./checkpoints/Cosmos3-Edge/transformer
+  rectified_flow_inference_config:
+    num_steps: 30
+    num_train_timesteps: 1000
+    scheduler_type: unipc
+    shift: 10.0
+    use_dynamic_shifting: false
+    use_karras_sigmas: false
+  rectified_flow_training_config:
+    action_loss_weight: 10.0
+    independent_action_schedule: false
+    normalize_loss_by_active: false
+    shift:
+      '256': 3
+      '480': 5
+      '720': 10
+    shift_action: null
+    train_time_action_distribution: logitnormal
+    train_time_image_distribution: logitnormal
+    train_time_video_distribution: waver
+    train_time_weight: uniform
+    use_discrete_rf: false
+    use_dynamic_shift: false
+    use_high_sigma_strategy: false
+    use_high_sigma_strategy_action: false
+    vision_loss_weight: 10.0
+  reinitialize_action_policy: true
+  special_tokens:
+    end_of_generation: 21
+    eos_token_id: 11
+    start_of_generation: 20
+  strict_mapping: true
+  timestep_scale: 0.001
+  type: Cosmos3FlowMatching
+  unified_3d_mrope_reset_spatial_ids: true
+  unified_3d_mrope_temporal_modality_margin: 15000
+  vision_in_proj:
+    in_dim: 192
+    out_dim: 2048
+    type: LinearProjector
+  vision_latent_dim: 48
+  vision_out_proj:
+    in_dim: 2048
+    out_dim: 192
+    type: LinearProjector
+  vision_vae:
+    encode_exact_durations:
+    - 17
+    pretrained_name_or_path: ./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
+    type: Cosmos3Wan22VAE
+  vlm_backbone:
+    include_visual: false
+    skip_init_weights: true
+    type: Cosmos3MoTBackbone
+    vlm_config:
+      attention_bias: false
+      bos_token_id: 1
+      enable_mrope: true
+      eos_token_id: 11
+      head_dim: 128
+      hidden_size: 2048
+      intermediate_size: 9216
+      layer_norm_epsilon: 1.0e-05
+      max_position_embeddings: 131072
+      mlp_bias: false
+      mlp_hidden_act: relu2
+      model_type: nemotron_3_dense_vl_text
+      mrope_section:
+      - 24
+      - 20
+      - 20
+      num_attention_heads: 16
+      num_hidden_layers: 28
+      num_key_value_heads: 8
+      pad_token_id: 11
+      rope_theta: 100000000.0
+      tie_word_embeddings: false
+      use_und_k_norm_for_gen: true
+      vocab_size: 131072
+runner:
+  change_key_name: false
+  collator:
+    list_keys:
+    - sequence_plan
+    meta_keys:
+    - task_description
+    - stats
+    - info
+    - timestamp
+    pad_id: 11
+    sequence_keys:
+    - text_token_ids
+    tensor_keys:
+    - images
+    - actions
+    - embodiment_ids
+    - raw_action_dim
+    - conditioning_fps
+    - action_fps
+    type: Cosmos3Collator
+  enable_gradient_checkpointing: true
+  enable_mixed_precision_training: true
+  grad_accumulation_steps: 4
+  lr_scheduler:
+    cycle_length: 16000
+    type: linear-warmup+linear-decay
+    warmup_steps: 500
+  max_epochs: null
+  max_grad_norm: 1.0
+  max_keep_ckpts: 2
+  max_steps: 5000
+  metric:
+    active_trackers:
+    - jsonl
+    - wandb
+    grad_accumulation_steps: 4
+    run_dir: work_dirs
+    type: VLAMetric
+    window_size: 1
+  mixed_precision_dtype: bf16
+  optimizer:
+    betas:
+    - 0.9
+    - 0.99
+    eps: 1.0e-08
+    exclude_1d_from_weight_decay: false
+    fused: true
+    lr: 5.0e-05
+    paramwise_learning_rate:
+      action_in_proj.: 0.00025
+      action_modality_embed: 0.00025
+      action_out_proj.: 0.00025
+    type: AdamW
+    weight_decay: 0.05
+  sampler: null
+  save_iter_interval: 500
+  sharding_strategy: full-shard
+  tokenizer:
+    model_max_length: 4096
+    model_path: ./checkpoints/Cosmos3-Edge
+    padding_side: right
+    trust_remote_code: true
+    type: PretrainedTokenizer
+  type: FSDPTrainRunner
+seed: 7
+train_dataloader:
+  dataset:
+    datasets:
+      action_key: action
+      action_window_size: 16
+      data_root_path:
+      - ./datasets/libero_spatial_no_noops_lerobotv2.1
+      - ./datasets/libero_spatial_no_noops_lerobotv2.1
+      - ./datasets/libero_object_no_noops_lerobotv2.1
+      - ./datasets/libero_object_no_noops_lerobotv2.1
+      - ./datasets/libero_goal_no_noops_lerobotv2.1
+      - ./datasets/libero_goal_no_noops_lerobotv2.1
+      - ./datasets/libero_10_no_noops_lerobotv2.1
+      frame_window_size: 17
+      require_full_window: true
+      statistic_name: all_libero_no_noops
+      transforms:
+      - embodiment_id: 5
+        name_mappings:
+          actions:
+          - actions
+          observation.state:
+          - states
+        parquet_keys:
+        - observation.state
+        - timestamp
+        - actions
+        - info
+        - stats
+        type: ProcessParquetInputs
+        video_keys:
+        - observation.images.image
+        - observation.images.wrist_image
+      - type: LiberoFramewiseActionToRot6D
+      - action_metadata:
+          append_viewpoint: false
+          conditioning_fps: 20.0
+          frame_window_size: 17
+          video_height: 192
+          video_width: 320
+        cfg_dropout_rate: 0.1
+        format_prompt_as_json: true
+        max_len: 512
+        tokenizer:
+          model_max_length: 4096
+          model_path: ./checkpoints/Cosmos3-Edge
+          padding_side: right
+          trust_remote_code: true
+          type: PretrainedTokenizer
+        type: ProcessCosmos3Prompt
+      - height: 256
+        type: ResizeImages
+        width: 256
+      - type: SimpleNormalizeImages
+      - action_dim: 64
+        action_key: action
+        action_norm_type: quantile
+        state_dim: 64
+        state_key: proprio
+        state_norm_type: none
+        type: NormalizeStatesAndActions
+      - conditioning_fps: 20.0
+        frame_window_size: 17
+        mode: wam
+        prepend_state_to_action: false
+        raw_action_dim: 10
+        type: BuildCosmos3Sequence
+      - frame_window_size: 17
+        num_views: 2
+        tile_direction: horizontal
+        type: PrepareVideo
+      - height: 192
+        type: ResizeAndReflectPad
+        width: 320
+      type: ParquetDataset
+      use_delta: false
+      window_start_idx: 0
+    name_mappings:
+      action:
+      - action
+      observation.state:
+      - proprio
+    statistic_keys:
+    - observation.state
+    - timestamp
+    - action
+    statistic_name: all_libero_no_noops
+    statistics_overrides:
+      all_libero_no_noops:
+        action:
+          max:
+          - 0.9375
+          - 0.9375
+          - 0.9375
+          - 1.0
+          - 0.368853
+          - 0.341214
+          - 0.356395
+          - 1.0
+          - 0.348251
+          - 1.0
+          mean:
+          - 0.050704
+          - 0.097407
+          - -0.094833
+          - 0.994873
+          - -0.004579
+          - -0.004288
+          - 0.004389
+          - 0.996104
+          - 0.001109
+          - 0.476725
+          min:
+          - -0.9375
+          - -0.9375
+          - -0.9375
+          - 0.902028
+          - -0.356085
+          - -0.367416
+          - -0.370434
+          - 0.921907
+          - -0.255
+          - 0.0
+          q01:
+          - -0.723214
+          - -0.808929
+          - -0.9375
+          - 0.934955
+          - -0.223431
+          - -0.189878
+          - -0.334735
+          - 0.938516
+          - -0.107736
+          - 0.0
+          q99:
+          - 0.9375
+          - 0.870536
+          - 0.9375
+          - 1.0
+          - 0.331
+          - 0.163153
+          - 0.226216
+          - 1.0
+          - 0.127158
+          - 1.0
+          std:
+          - 0.333621
+          - 0.387175
+          - 0.45714
+          - 0.010807
+          - 0.077802
+          - 0.063386
+          - 0.078571
+          - 0.009994
+          - 0.038504
+          - 0.49946
+    type: DistributedRepeatingDataset
+  per_device_batch_size: 32
+  per_device_num_workers: 4

cosmos3_edge_libero_full_finetune_bs2048/dataset_statistics.json

@@ -0,0 +1,138 @@
+{
+  "all_libero_no_noops": {
+    "proprio": {
+      "mean": [
+        -0.047496414237850684,
+        0.03435983471608021,
+        0.7514258672095461,
+        2.983973905120257,
+        -0.1419395165888572,
+        -0.118218935315545,
+        0.026695325676034158,
+        -0.026933706126891162
+      ],
+      "std": [
+        0.10441071071075952,
+        0.15324089039592545,
+        0.39886824990682457,
+        0.34338260700499407,
+        0.8068781715283699,
+        0.31379284611306424,
+        0.014319374319652562,
+        0.014208265894281714
+      ],
+      "min": [
+        -0.48278069496154785,
+        -0.3309336006641388,
+        0.008128181099891663,
+        1.002794623374939,
+        -3.6312508583068848,
+        -1.842738389968872,
+        -0.005453015677630901,
+        -0.042015016078948975
+      ],
+      "max": [
+        0.2103137969970703,
+        0.3904264271259308,
+        1.472778081893921,
+        3.7248642444610596,
+        3.5618896484375,
+        1.3863215446472168,
+        0.04232141748070717,
+        0.0013126095291227102
+      ],
+      "q01": null,
+      "q99": null
+    },
+    "timestamp": {
+      "mean": [
+        4.252653686389771
+      ],
+      "std": [
+        3.2577109230601815
+      ],
+      "min": [
+        0.0
+      ],
+      "max": [
+        25.2
+      ],
+      "q01": null,
+      "q99": null
+    },
+    "action": {
+      "mean": [
+        0.050704,
+        0.097407,
+        -0.094833,
+        0.994873,
+        -0.004579,
+        -0.004288,
+        0.004389,
+        0.996104,
+        0.001109,
+        0.476725
+      ],
+      "std": [
+        0.333621,
+        0.387175,
+        0.45714,
+        0.010807,
+        0.077802,
+        0.063386,
+        0.078571,
+        0.009994,
+        0.038504,
+        0.49946
+      ],
+      "min": [
+        -0.9375,
+        -0.9375,
+        -0.9375,
+        0.902028,
+        -0.356085,
+        -0.367416,
+        -0.370434,
+        0.921907,
+        -0.255,
+        0.0
+      ],
+      "max": [
+        0.9375,
+        0.9375,
+        0.9375,
+        1.0,
+        0.368853,
+        0.341214,
+        0.356395,
+        1.0,
+        0.348251,
+        1.0
+      ],
+      "q01": [
+        -0.723214,
+        -0.808929,
+        -0.9375,
+        0.934955,
+        -0.223431,
+        -0.189878,
+        -0.334735,
+        0.938516,
+        -0.107736,
+        0.0
+      ],
+      "q99": [
+        0.9375,
+        0.870536,
+        0.9375,
+        1.0,
+        0.331,
+        0.163153,
+        0.226216,
+        1.0,
+        0.127158,
+        1.0
+      ]
+    }
+  }
+}

cosmos3_edge_libero_full_finetune_bs2048/run-metrics.jsonl

@@ -0,0 +1 @@
+{"hparams": "{'model': {'action_horizon': 16, 'action_in_proj': {'input_size': 64, 'num_domains': 32, 'output_size': 2048, 'type': 'DomainAwareLinear'}, 'action_out_proj': {'input_size': 2048, 'num_domains': 32, 'output_size': 64, 'type': 'DomainAwareLinear'}, 'base_fps': 24.0, 'enable_fps_modulation': True, 'enable_vision_loss': True, 'freeze_non_moe_vlm_backbone': True, 'freeze_vlm_backbone': False, 'latent_patch_size': 2, 'max_action_dim': 64, 'name_mapping': {'.self_attn.k_norm_moe_gen.': '.self_attn.norm_added_k.', '.self_attn.k_proj.': '.self_attn.to_k.', '.self_attn.k_proj_moe_gen.': '.self_attn.add_k_proj.', '.self_attn.o_proj.': '.self_attn.to_out.', '.self_attn.o_proj_moe_gen.': '.self_attn.to_add_out.', '.self_attn.q_norm_moe_gen.': '.self_attn.norm_added_q.', '.self_attn.q_proj.': '.self_attn.to_q.', '.self_attn.q_proj_moe_gen.': '.self_attn.add_q_proj.', '.self_attn.v_proj.': '.self_attn.to_v.', '.self_attn.v_proj_moe_gen.': '.self_attn.add_v_proj.', 'action_in_proj.': 'action_proj_in.', 'action_modality_embed': 'action_modality_embed', 'action_out_proj.': 'action_proj_out.', 'time_embedder.mlp.0.': 'time_embedder.linear_1.', 'time_embedder.mlp.2.': 'time_embedder.linear_2.', 'vision_in_proj.projector.': 'proj_in.', 'vision_out_proj.projector.': 'proj_out.', 'vlm_backbone.lm_head.weight': 'lm_head.weight', 'vlm_backbone.model.language_model.embed_tokens.weight': 'embed_tokens.weight', 'vlm_backbone.model.language_model.layers.': 'layers.', 'vlm_backbone.model.language_model.norm.weight': 'norm.weight', 'vlm_backbone.model.language_model.norm_moe_gen.weight': 'norm_moe_gen.weight'}, 'num_embodiment_domains': 32, 'ori_action_dim': 10, 'packed_attention_backend': 'flash2', 'position_embedding_type': 'unified_3d_mrope', 'pretrained_name_or_path': './checkpoints/Cosmos3-Edge/transformer', 'reinitialize_action_policy': True, 'rectified_flow_inference_config': {'num_steps': 30, 'num_train_timesteps': 1000, 'scheduler_type': 'unipc', 'shift': 10.0, 'use_dynamic_shifting': False, 'use_karras_sigmas': False}, 'rectified_flow_training_config': {'action_loss_weight': 10.0, 'independent_action_schedule': False, 'normalize_loss_by_active': False, 'shift': {'256': 3, '480': 5, '720': 10}, 'shift_action': None, 'train_time_action_distribution': 'logitnormal', 'train_time_image_distribution': 'logitnormal', 'train_time_video_distribution': 'waver', 'train_time_weight': 'uniform', 'use_discrete_rf': False, 'use_dynamic_shift': False, 'use_high_sigma_strategy': False, 'use_high_sigma_strategy_action': False, 'vision_loss_weight': 10.0}, 'special_tokens': {'end_of_generation': 21, 'eos_token_id': 11, 'start_of_generation': 20}, 'strict_mapping': True, 'timestep_scale': 0.001, 'type': 'Cosmos3FlowMatching', 'unified_3d_mrope_reset_spatial_ids': True, 'unified_3d_mrope_temporal_modality_margin': 15000, 'vision_in_proj': {'in_dim': 192, 'out_dim': 2048, 'type': 'LinearProjector'}, 'vision_latent_dim': 48, 'vision_out_proj': {'in_dim': 2048, 'out_dim': 192, 'type': 'LinearProjector'}, 'vision_vae': {'encode_exact_durations': [17], 'pretrained_name_or_path': './checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth', 'type': 'Cosmos3Wan22VAE'}, 'vlm_backbone': {'include_visual': False, 'skip_init_weights': True, 'type': 'Cosmos3MoTBackbone', 'vlm_config': {'attention_bias': False, 'bos_token_id': 1, 'enable_mrope': True, 'eos_token_id': 11, 'head_dim': 128, 'hidden_size': 2048, 'intermediate_size': 9216, 'layer_norm_epsilon': 1e-05, 'max_position_embeddings': 131072, 'mlp_bias': False, 'mlp_hidden_act': 'relu2', 'model_type': 'nemotron_3_dense_vl_text', 'mrope_section': [24, 20, 20], 'num_attention_heads': 16, 'num_hidden_layers': 28, 'num_key_value_heads': 8, 'pad_token_id': 11, 'rope_theta': 100000000.0, 'tie_word_embeddings': False, 'use_und_k_norm_for_gen': True, 'vocab_size': 131072}}}, 'inference_model': {'action_horizon': 16, 'action_in_proj': {'input_size': 64, 'num_domains': 32, 'output_size': 2048, 'type': 'DomainAwareLinear'}, 'action_out_proj': {'input_size': 2048, 'num_domains': 32, 'output_size': 64, 'type': 'DomainAwareLinear'}, 'base_fps': 24.0, 'enable_fps_modulation': True, 'enable_vision_loss': True, 'freeze_non_moe_vlm_backbone': True, 'freeze_vlm_backbone': False, 'latent_patch_size': 2, 'max_action_dim': 64, 'name_mapping': {'.self_attn.k_norm_moe_gen.': '.self_attn.norm_added_k.', '.self_attn.k_proj.': '.self_attn.to_k.', '.self_attn.k_proj_moe_gen.': '.self_attn.add_k_proj.', '.self_attn.o_proj.': '.self_attn.to_out.', '.self_attn.o_proj_moe_gen.': '.self_attn.to_add_out.', '.self_attn.q_norm_moe_gen.': '.self_attn.norm_added_q.', '.self_attn.q_proj.': '.self_attn.to_q.', '.self_attn.q_proj_moe_gen.': '.self_attn.add_q_proj.', '.self_attn.v_proj.': '.self_attn.to_v.', '.self_attn.v_proj_moe_gen.': '.self_attn.add_v_proj.', 'action_in_proj.': 'action_proj_in.', 'action_modality_embed': 'action_modality_embed', 'action_out_proj.': 'action_proj_out.', 'time_embedder.mlp.0.': 'time_embedder.linear_1.', 'time_embedder.mlp.2.': 'time_embedder.linear_2.', 'vision_in_proj.projector.': 'proj_in.', 'vision_out_proj.projector.': 'proj_out.', 'vlm_backbone.lm_head.weight': 'lm_head.weight', 'vlm_backbone.model.language_model.embed_tokens.weight': 'embed_tokens.weight', 'vlm_backbone.model.language_model.layers.': 'layers.', 'vlm_backbone.model.language_model.norm.weight': 'norm.weight', 'vlm_backbone.model.language_model.norm_moe_gen.weight': 'norm_moe_gen.weight'}, 'num_embodiment_domains': 32, 'ori_action_dim': 10, 'packed_attention_backend': 'flash2', 'position_embedding_type': 'unified_3d_mrope', 'pretrained_name_or_path': './checkpoints/Cosmos3-Edge/transformer', 'reinitialize_action_policy': True, 'rectified_flow_inference_config': {'num_steps': 30, 'num_train_timesteps': 1000, 'scheduler_type': 'unipc', 'shift': 10.0, 'use_dynamic_shifting': False, 'use_karras_sigmas': False}, 'rectified_flow_training_config': {'action_loss_weight': 10.0, 'independent_action_schedule': False, 'normalize_loss_by_active': False, 'shift': {'256': 3, '480': 5, '720': 10}, 'shift_action': None, 'train_time_action_distribution': 'logitnormal', 'train_time_image_distribution': 'logitnormal', 'train_time_video_distribution': 'waver', 'train_time_weight': 'uniform', 'use_discrete_rf': False, 'use_dynamic_shift': False, 'use_high_sigma_strategy': False, 'use_high_sigma_strategy_action': False, 'vision_loss_weight': 10.0}, 'special_tokens': {'end_of_generation': 21, 'eos_token_id': 11, 'start_of_generation': 20}, 'strict_mapping': True, 'timestep_scale': 0.001, 'type': 'Cosmos3FlowMatching', 'unified_3d_mrope_reset_spatial_ids': True, 'unified_3d_mrope_temporal_modality_margin': 15000, 'vision_in_proj': {'in_dim': 192, 'out_dim': 2048, 'type': 'LinearProjector'}, 'vision_latent_dim': 48, 'vision_out_proj': {'in_dim': 2048, 'out_dim': 192, 'type': 'LinearProjector'}, 'vision_vae': {'encode_exact_durations': [17], 'pretrained_name_or_path': './checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth', 'type': 'Cosmos3Wan22VAE'}, 'vlm_backbone': {'include_visual': False, 'skip_init_weights': True, 'type': 'Cosmos3MoTBackbone', 'vlm_config': {'attention_bias': False, 'bos_token_id': 1, 'enable_mrope': True, 'eos_token_id': 11, 'head_dim': 128, 'hidden_size': 2048, 'intermediate_size': 9216, 'layer_norm_epsilon': 1e-05, 'max_position_embeddings': 131072, 'mlp_bias': False, 'mlp_hidden_act': 'relu2', 'model_type': 'nemotron_3_dense_vl_text', 'mrope_section': [24, 20, 20], 'num_attention_heads': 16, 'num_hidden_layers': 28, 'num_key_value_heads': 8, 'pad_token_id': 11, 'rope_theta': 100000000.0, 'tie_word_embeddings': False, 'use_und_k_norm_for_gen': True, 'vocab_size': 131072}}}, 'runner': {'change_key_name': False, 'collator': {'list_keys': ['sequence_plan'], 'meta_keys': ['task_description', 'stats', 'info', 'timestamp'], 'pad_id': 11, 'sequence_keys': ['text_token_ids'], 'tensor_keys': ['images', 'actions', 'embodiment_ids', 'raw_action_dim', 'conditioning_fps', 'action_fps'], 'type': 'Cosmos3Collator'}, 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'grad_accumulation_steps': 4, 'lr_scheduler': {'cycle_length': 16000, 'type': 'linear-warmup+linear-decay', 'warmup_steps': 500}, 'max_epochs': None, 'max_grad_norm': 1.0, 'max_keep_ckpts': 2, 'max_steps': 5000, 'metric': {'active_trackers': ('jsonl', 'wandb'), 'grad_accumulation_steps': 4, 'run_dir': '/mnt/data/cpfs/users/leo/wk_dir/cosmos3/libero_mixed_edge_v21_192x320_gb2048_2x8_h100_5k', 'type': 'VLAMetric', 'window_size': 1}, 'mixed_precision_dtype': 'bf16', 'optimizer': {'betas': (0.9, 0.99), 'eps': 1e-08, 'exclude_1d_from_weight_decay': False, 'fused': True, 'lr': 5e-05, 'paramwise_learning_rate': {'action_in_proj.': 0.00025, 'action_modality_embed': 0.00025, 'action_out_proj.': 0.00025}, 'type': 'AdamW', 'weight_decay': 0.05}, 'sampler': None, 'save_iter_interval': 500, 'sharding_strategy': 'full-shard', 'tokenizer': {'model_max_length': 4096, 'model_path': './checkpoints/Cosmos3-Edge', 'padding_side': 'right', 'trust_remote_code': True, 'type': 'PretrainedTokenizer'}, 'type': 'FSDPTrainRunner', 'cfg': Config (path: configs/cosmos3/cosmos3edge_libero_full_finetune.py): {'model': {'action_horizon': 16, 'action_in_proj': {'input_size': 64, 'num_domains': 32, 'output_size': 2048, 'type': 'DomainAwareLinear'}, 'action_out_proj': {'input_size': 2048, 'num_domains': 32, 'output_size': 64, 'type': 'DomainAwareLinear'}, 'base_fps': 24.0, 'enable_fps_modulation': True, 'enable_vision_loss': True, 'freeze_non_moe_vlm_backbone': True, 'freeze_vlm_backbone': False, 'latent_patch_size': 2, 'max_action_dim': 64, 'name_mapping': {'.self_attn.k_norm_moe_gen.': '.self_attn.norm_added_k.', '.self_attn.k_proj.': '.self_attn.to_k.', '.self_attn.k_proj_moe_gen.': '.self_attn.add_k_proj.', '.self_attn.o_proj.': '.self_attn.to_out.', '.self_attn.o_proj_moe_gen.': '.self_attn.to_add_out.', '.self_attn.q_norm_moe_gen.': '.self_attn.norm_added_q.', '.self_attn.q_proj.': '.self_attn.to_q.', '.self_attn.q_proj_moe_gen.': '.self_attn.add_q_proj.', '.self_attn.v_proj.': '.self_attn.to_v.', '.self_attn.v_proj_moe_gen.': '.self_attn.add_v_proj.', 'action_in_proj.': 'action_proj_in.', 'action_modality_embed': 'action_modality_embed', 'action_out_proj.': 'action_proj_out.', 'time_embedder.mlp.0.': 'time_embedder.linear_1.', 'time_embedder.mlp.2.': 'time_embedder.linear_2.', 'vision_in_proj.projector.': 'proj_in.', 'vision_out_proj.projector.': 'proj_out.', 'vlm_backbone.lm_head.weight': 'lm_head.weight', 'vlm_backbone.model.language_model.embed_tokens.weight': 'embed_tokens.weight', 'vlm_backbone.model.language_model.layers.': 'layers.', 'vlm_backbone.model.language_model.norm.weight': 'norm.weight', 'vlm_backbone.model.language_model.norm_moe_gen.weight': 'norm_moe_gen.weight'}, 'num_embodiment_domains': 32, 'ori_action_dim': 10, 'packed_attention_backend': 'flash2', 'position_embedding_type': 'unified_3d_mrope', 'pretrained_name_or_path': './checkpoints/Cosmos3-Edge/transformer', 'reinitialize_action_policy': True, 'rectified_flow_inference_config': {'num_steps': 30, 'num_train_timesteps': 1000, 'scheduler_type': 'unipc', 'shift': 10.0, 'use_dynamic_shifting': False, 'use_karras_sigmas': False}, 'rectified_flow_training_config': {'action_loss_weight': 10.0, 'independent_action_schedule': False, 'normalize_loss_by_active': False, 'shift': {'256': 3, '480': 5, '720': 10}, 'shift_action': None, 'train_time_action_distribution': 'logitnormal', 'train_time_image_distribution': 'logitnormal', 'train_time_video_distribution': 'waver', 'train_time_weight': 'uniform', 'use_discrete_rf': False, 'use_dynamic_shift': False, 'use_high_sigma_strategy': False, 'use_high_sigma_strategy_action': False, 'vision_loss_weight': 10.0}, 'special_tokens': {'end_of_generation': 21, 'eos_token_id': 11, 'start_of_generation': 20}, 'strict_mapping': True, 'timestep_scale': 0.001, 'type': 'Cosmos3FlowMatching', 'unified_3d_mrope_reset_spatial_ids': True, 'unified_3d_mrope_temporal_modality_margin': 15000, 'vision_in_proj': {'in_dim': 192, 'out_dim': 2048, 'type': 'LinearProjector'}, 'vision_latent_dim': 48, 'vision_out_proj': {'in_dim': 2048, 'out_dim': 192, 'type': 'LinearProjector'}, 'vision_vae': {'encode_exact_durations': [17], 'pretrained_name_or_path': './checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth', 'type': 'Cosmos3Wan22VAE'}, 'vlm_backbone': {'include_visual': False, 'skip_init_weights': True, 'type': 'Cosmos3MoTBackbone', 'vlm_config': {'attention_bias': False, 'bos_token_id': 1, 'enable_mrope': True, 'eos_token_id': 11, 'head_dim': 128, 'hidden_size': 2048, 'intermediate_size': 9216, 'layer_norm_epsilon': 1e-05, 'max_position_embeddings': 131072, 'mlp_bias': False, 'mlp_hidden_act': 'relu2', 'model_type': 'nemotron_3_dense_vl_text', 'mrope_section': [24, 20, 20], 'num_attention_heads': 16, 'num_hidden_layers': 28, 'num_key_value_heads': 8, 'pad_token_id': 11, 'rope_theta': 100000000.0, 'tie_word_embeddings': False, 'use_und_k_norm_for_gen': True, 'vocab_size': 131072}}}, 'inference_model': {'action_horizon': 16, 'action_in_proj': {'input_size': 64, 'num_domains': 32, 'output_size': 2048, 'type': 'DomainAwareLinear'}, 'action_out_proj': {'input_size': 2048, 'num_domains': 32, 'output_size': 64, 'type': 'DomainAwareLinear'}, 'base_fps': 24.0, 'enable_fps_modulation': True, 'enable_vision_loss': True, 'freeze_non_moe_vlm_backbone': True, 'freeze_vlm_backbone': False, 'latent_patch_size': 2, 'max_action_dim': 64, 'name_mapping': {'.self_attn.k_norm_moe_gen.': '.self_attn.norm_added_k.', '.self_attn.k_proj.': '.self_attn.to_k.', '.self_attn.k_proj_moe_gen.': '.self_attn.add_k_proj.', '.self_attn.o_proj.': '.self_attn.to_out.', '.self_attn.o_proj_moe_gen.': '.self_attn.to_add_out.', '.self_attn.q_norm_moe_gen.': '.self_attn.norm_added_q.', '.self_attn.q_proj.': '.self_attn.to_q.', '.self_attn.q_proj_moe_gen.': '.self_attn.add_q_proj.', '.self_attn.v_proj.': '.self_attn.to_v.', '.self_attn.v_proj_moe_gen.': '.self_attn.add_v_proj.', 'action_in_proj.': 'action_proj_in.', 'action_modality_embed': 'action_modality_embed', 'action_out_proj.': 'action_proj_out.', 'time_embedder.mlp.0.': 'time_embedder.linear_1.', 'time_embedder.mlp.2.': 'time_embedder.linear_2.', 'vision_in_proj.projector.': 'proj_in.', 'vision_out_proj.projector.': 'proj_out.', 'vlm_backbone.lm_head.weight': 'lm_head.weight', 'vlm_backbone.model.language_model.embed_tokens.weight': 'embed_tokens.weight', 'vlm_backbone.model.language_model.layers.': 'layers.', 'vlm_backbone.model.language_model.norm.weight': 'norm.weight', 'vlm_backbone.model.language_model.norm_moe_gen.weight': 'norm_moe_gen.weight'}, 'num_embodiment_domains': 32, 'ori_action_dim': 10, 'packed_attention_backend': 'flash2', 'position_embedding_type': 'unified_3d_mrope', 'pretrained_name_or_path': './checkpoints/Cosmos3-Edge/transformer', 'reinitialize_action_policy': True, 'rectified_flow_inference_config': {'num_steps': 30, 'num_train_timesteps': 1000, 'scheduler_type': 'unipc', 'shift': 10.0, 'use_dynamic_shifting': False, 'use_karras_sigmas': False}, 'rectified_flow_training_config': {'action_loss_weight': 10.0, 'independent_action_schedule': False, 'normalize_loss_by_active': False, 'shift': {'256': 3, '480': 5, '720': 10}, 'shift_action': None, 'train_time_action_distribution': 'logitnormal', 'train_time_image_distribution': 'logitnormal', 'train_time_video_distribution': 'waver', 'train_time_weight': 'uniform', 'use_discrete_rf': False, 'use_dynamic_shift': False, 'use_high_sigma_strategy': False, 'use_high_sigma_strategy_action': False, 'vision_loss_weight': 10.0}, 'special_tokens': {'end_of_generation': 21, 'eos_token_id': 11, 'start_of_generation': 20}, 'strict_mapping': True, 'timestep_scale': 0.001, 'type': 'Cosmos3FlowMatching', 'unified_3d_mrope_reset_spatial_ids': True, 'unified_3d_mrope_temporal_modality_margin': 15000, 'vision_in_proj': {'in_dim': 192, 'out_dim': 2048, 'type': 'LinearProjector'}, 'vision_latent_dim': 48, 'vision_out_proj': {'in_dim': 2048, 'out_dim': 192, 'type': 'LinearProjector'}, 'vision_vae': {'encode_exact_durations': [17], 'pretrained_name_or_path': './checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth', 'type': 'Cosmos3Wan22VAE'}, 'vlm_backbone': {'include_visual': False, 'skip_init_weights': True, 'type': 'Cosmos3MoTBackbone', 'vlm_config': {'attention_bias': False, 'bos_token_id': 1, 'enable_mrope': True, 'eos_token_id': 11, 'head_dim': 128, 'hidden_size': 2048, 'intermediate_size': 9216, 'layer_norm_epsilon': 1e-05, 'max_position_embeddings': 131072, 'mlp_bias': False, 'mlp_hidden_act': 'relu2', 'model_type': 'nemotron_3_dense_vl_text', 'mrope_section': [24, 20, 20], 'num_attention_heads': 16, 'num_hidden_layers': 28, 'num_key_value_heads': 8, 'pad_token_id': 11, 'rope_theta': 100000000.0, 'tie_word_embeddings': False, 'use_und_k_norm_for_gen': True, 'vocab_size': 131072}}}, 'runner': {'change_key_name': False, 'collator': {'list_keys': ['sequence_plan'], 'meta_keys': ['task_description', 'stats', 'info', 'timestamp'], 'pad_id': 11, 'sequence_keys': ['text_token_ids'], 'tensor_keys': ['images', 'actions', 'embodiment_ids', 'raw_action_dim', 'conditioning_fps', 'action_fps'], 'type': 'Cosmos3Collator'}, 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'grad_accumulation_steps': 4, 'lr_scheduler': {'cycle_length': 16000, 'type': 'linear-warmup+linear-decay', 'warmup_steps': 500}, 'max_epochs': None, 'max_grad_norm': 1.0, 'max_keep_ckpts': 2, 'max_steps': 5000, 'metric': {'active_trackers': ('jsonl', 'wandb'), 'grad_accumulation_steps': 4, 'run_dir': '/mnt/data/cpfs/users/leo/wk_dir/cosmos3/libero_mixed_edge_v21_192x320_gb2048_2x8_h100_5k', 'type': 'VLAMetric', 'window_size': 1}, 'mixed_precision_dtype': 'bf16', 'optimizer': {'betas': (0.9, 0.99), 'eps': 1e-08, 'exclude_1d_from_weight_decay': False, 'fused': True, 'lr': 5e-05, 'paramwise_learning_rate': {'action_in_proj.': 0.00025, 'action_modality_embed': 0.00025, 'action_out_proj.': 0.00025}, 'type': 'AdamW', 'weight_decay': 0.05}, 'sampler': None, 'save_iter_interval': 500, 'sharding_strategy': 'full-shard', 'tokenizer': {'model_max_length': 4096, 'model_path': './checkpoints/Cosmos3-Edge', 'padding_side': 'right', 'trust_remote_code': True, 'type': 'PretrainedTokenizer'}, 'type': 'FSDPTrainRunner', 'cfg': Config (path: configs/cosmos3/cosmos3edge_libero_full_finetune.py): {...}, 'args': Namespace(config='configs/cosmos3/cosmos3edge_libero_full_finetune.py', work_dir='/mnt/data/cpfs/users/leo/wk_dir/cosmos3/libero_mixed_edge_v21_192x320_gb2048_2x8_h100_5k', cfg_options={'train_dataloader.per_device_batch_size': 32, 'runner.grad_accumulation_steps': 4, 'runner.metric.grad_accumulation_steps': 4, 'inference_model.rectified_flow_inference_config.shift': 10.0, 'eval.task_suite_name': ['libero_spatial', 'libero_object', 'libero_goal', 'libero_10'], 'eval.num_trials_per_task': 50, 'eval.task_ids': None, 'eval.run_id_suffix': 'mixed-v21-shift10-after-train'}, eval_after_train=True, resume_from=None)}, 'seed': 7, 'train_dataloader': {'dataset': {'datasets': {'action_key': 'action', 'action_window_size': 16, 'data_root_path': ['./datasets/libero_spatial_no_noops_lerobotv2.1', './datasets/libero_spatial_no_noops_lerobotv2.1', './datasets/libero_object_no_noops_lerobotv2.1', './datasets/libero_object_no_noops_lerobotv2.1', './datasets/libero_goal_no_noops_lerobotv2.1', './datasets/libero_goal_no_noops_lerobotv2.1', './datasets/libero_10_no_noops_lerobotv2.1'], 'frame_window_size': 17, 'require_full_window': True, 'statistic_name': 'all_libero_no_noops', 'transforms': [{'embodiment_id': 5, 'name_mappings': {'actions': ['actions'], 'observation.state': ['states']}, 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats'], 'type': 'ProcessParquetInputs', 'video_keys': ['observation.images.image', 'observation.images.wrist_image']}, {'type': 'LiberoFramewiseActionToRot6D'}, {'action_metadata': {'append_viewpoint': False, 'conditioning_fps': 20.0, 'frame_window_size': 17, 'video_height': 192, 'video_width': 320}, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'max_len': 512, 'tokenizer': {'model_max_length': 4096, 'model_path': './checkpoints/Cosmos3-Edge', 'padding_side': 'right', 'trust_remote_code': True, 'type': 'PretrainedTokenizer'}, 'type': 'ProcessCosmos3Prompt'}, {'height': 256, 'type': 'ResizeImages', 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'action_dim': 64, 'action_key': 'action', 'action_norm_type': 'quantile', 'state_norm_type': 'none', 'state_dim': 64, 'state_key': 'proprio', 'type': 'NormalizeStatesAndActions'}, {'conditioning_fps': 20.0, 'frame_window_size': 17, 'mode': 'wam', 'prepend_state_to_action': False, 'raw_action_dim': 10, 'type': 'BuildCosmos3Sequence'}, {'frame_window_size': 17, 'num_views': 2, 'tile_direction': 'horizontal', 'type': 'PrepareVideo'}, {'height': 192, 'type': 'ResizeAndReflectPad', 'width': 320}], 'type': 'ParquetDataset', 'use_delta': False, 'window_start_idx': 0}, 'name_mappings': {'action': ['action'], 'observation.state': ['proprio']}, 'statistic_keys': ['observation.state', 'timestamp', 'action'], 'statistic_name': 'all_libero_no_noops', 'statistics_overrides': {'all_libero_no_noops': {'action': {'mean': [0.050704, 0.097407, -0.094833, 0.994873, -0.004579, -0.004288, 0.004389, 0.996104, 0.001109, 0.476725], 'std': [0.333621, 0.387175, 0.45714, 0.010807, 0.077802, 0.063386, 0.078571, 0.009994, 0.038504, 0.49946], 'min': [-0.9375, -0.9375, -0.9375, 0.902028, -0.356085, -0.367416, -0.370434, 0.921907, -0.255, 0.0], 'max': [0.9375, 0.9375, 0.9375, 1.0, 0.368853, 0.341214, 0.356395, 1.0, 0.348251, 1.0], 'q01': [-0.723214, -0.808929, -0.9375, 0.934955, -0.223431, -0.189878, -0.334735, 0.938516, -0.107736, 0.0], 'q99': [0.9375, 0.870536, 0.9375, 1.0, 0.331, 0.163153, 0.226216, 1.0, 0.127158, 1.0]}}}, 'type': 'DistributedRepeatingDataset'}, 'per_device_batch_size': 32, 'per_device_num_workers': 4}, 'eval': {'dataset': {'extra_tensor_keys': ['conditioning_fps', 'prepend_state_to_action'], 'img_buffer_len': 1, 'transforms': [{'embodiment_id': 5, 'img_keys': ['agentview_image', 'robot0_eye_in_hand_image'], 'type': 'ProcessLiberoEvalInputs'}, {'conditioning_fps': 20.0, 'prepend_state_to_action': False, 'type': 'SetCosmos3ActionMetadata'}, {'image_resize_strategy': 'resize-crop', 'input_sizes': [[3, 256, 256], [3, 256, 256]], 'means': [[127.5, 127.5, 127.5], [127.5, 127.5, 127.5]], 'stds': [[127.5, 127.5, 127.5], [127.5, 127.5, 127.5]], 'type': 'TransformImage'}, {'action_metadata': {'append_viewpoint': False, 'conditioning_fps': 20.0, 'frame_window_size': 17, 'video_height': 192, 'video_width': 320}, 'cfg_dropout_rate': 0.0, 'format_prompt_as_json': True, 'max_len': 512, 'output_attention_mask_key': 'lang_masks', 'output_key': 'lang_tokens', 'tokenizer': {'model_max_length': 4096, 'model_path': './checkpoints/Cosmos3-Edge', 'padding_side': 'right', 'trust_remote_code': True, 'type': 'PretrainedTokenizer'}, 'type': 'ProcessCosmos3Prompt'}, {'gripper_key': 'robot0_gripper_qpos', 'norm_type': 'mean_std', 'out_key': 'states', 'pos_key': 'robot0_eef_pos', 'quat_key': 'robot0_eef_quat', 'state_dim': 64, 'type': 'LiberoProprioFromInputs'}, {'frame_window_size': 1, 'num_views': 2, 'tile_direction': 'horizontal', 'type': 'PrepareVideo'}, {'height': 192, 'type': 'ResizeAndReflectPad', 'width': 320}], 'type': 'LiberoParquetEvalDataset'}, 'denormalize_action': {'action_dim': 10, 'norm_type': 'quantile_rot', 'type': 'DenormalizeLiberoFramewiseRot6DAction'}, 'enable_mixed_precision_training': True, 'eval_chunk_size': 16, 'inference_seed': 7, 'mixed_precision_dtype': 'bf16', 'model_family': 'cosmos3', 'norm_stats_key': 'all_libero_no_noops', 'num_inference_steps': 30, 'num_steps_wait': 10, 'num_trials_per_task': 50, 'resize_size': 256, 'seed': 7, 'task_ids': None, 'task_suite_name': ['libero_spatial', 'libero_object', 'libero_goal', 'libero_10'], 'type': 'LiberoEvalRunner', 'run_id_suffix': 'mixed-v21-shift10-after-train'}}, 'args': Namespace(config='configs/cosmos3/cosmos3edge_libero_full_finetune.py', work_dir='/mnt/data/cpfs/users/leo/wk_dir/cosmos3/libero_mixed_edge_v21_192x320_gb2048_2x8_h100_5k', cfg_options={'train_dataloader.per_device_batch_size': 32, 'runner.grad_accumulation_steps': 4, 'runner.metric.grad_accumulation_steps': 4, 'inference_model.rectified_flow_inference_config.shift': 10.0, 'eval.task_suite_name': ['libero_spatial', 'libero_object', 'libero_goal', 'libero_10'], 'eval.num_trials_per_task': 50, 'eval.task_ids': None, 'eval.run_id_suffix': 'mixed-v21-shift10-after-train'}, eval_after_train=True, resume_from=None)}, 'seed': 7, 'train_dataloader': {'dataset': {'datasets': {'action_key': 'action', 'action_window_size': 16, 'data_root_path': ['./datasets/libero_spatial_no_noops_lerobotv2.1', './datasets/libero_spatial_no_noops_lerobotv2.1', './datasets/libero_object_no_noops_lerobotv2.1', './datasets/libero_object_no_noops_lerobotv2.1', './datasets/libero_goal_no_noops_lerobotv2.1', './datasets/libero_goal_no_noops_lerobotv2.1', './datasets/libero_10_no_noops_lerobotv2.1'], 'frame_window_size': 17, 'require_full_window': True, 'statistic_name': 'all_libero_no_noops', 'transforms': [{'embodiment_id': 5, 'name_mappings': {'actions': ['actions'], 'observation.state': ['states']}, 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats'], 'type': 'ProcessParquetInputs', 'video_keys': ['observation.images.image', 'observation.images.wrist_image']}, {'type': 'LiberoFramewiseActionToRot6D'}, {'action_metadata': {'append_viewpoint': False, 'conditioning_fps': 20.0, 'frame_window_size': 17, 'video_height': 192, 'video_width': 320}, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'max_len': 512, 'tokenizer': {'model_max_length': 4096, 'model_path': './checkpoints/Cosmos3-Edge', 'padding_side': 'right', 'trust_remote_code': True, 'type': 'PretrainedTokenizer'}, 'type': 'ProcessCosmos3Prompt'}, {'height': 256, 'type': 'ResizeImages', 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'action_dim': 64, 'action_key': 'action', 'action_norm_type': 'quantile', 'state_norm_type': 'none', 'state_dim': 64, 'state_key': 'proprio', 'type': 'NormalizeStatesAndActions'}, {'conditioning_fps': 20.0, 'frame_window_size': 17, 'mode': 'wam', 'prepend_state_to_action': False, 'raw_action_dim': 10, 'type': 'BuildCosmos3Sequence'}, {'frame_window_size': 17, 'num_views': 2, 'tile_direction': 'horizontal', 'type': 'PrepareVideo'}, {'height': 192, 'type': 'ResizeAndReflectPad', 'width': 320}], 'type': 'ParquetDataset', 'use_delta': False, 'window_start_idx': 0}, 'name_mappings': {'action': ['action'], 'observation.state': ['proprio']}, 'statistic_keys': ['observation.state', 'timestamp', 'action'], 'statistic_name': 'all_libero_no_noops', 'statistics_overrides': {'all_libero_no_noops': {'action': {'mean': [0.050704, 0.097407, -0.094833, 0.994873, -0.004579, -0.004288, 0.004389, 0.996104, 0.001109, 0.476725], 'std': [0.333621, 0.387175, 0.45714, 0.010807, 0.077802, 0.063386, 0.078571, 0.009994, 0.038504, 0.49946], 'min': [-0.9375, -0.9375, -0.9375, 0.902028, -0.356085, -0.367416, -0.370434, 0.921907, -0.255, 0.0], 'max': [0.9375, 0.9375, 0.9375, 1.0, 0.368853, 0.341214, 0.356395, 1.0, 0.348251, 1.0], 'q01': [-0.723214, -0.808929, -0.9375, 0.934955, -0.223431, -0.189878, -0.334735, 0.938516, -0.107736, 0.0], 'q99': [0.9375, 0.870536, 0.9375, 1.0, 0.331, 0.163153, 0.226216, 1.0, 0.127158, 1.0]}}}, 'type': 'DistributedRepeatingDataset'}, 'per_device_batch_size': 32, 'per_device_num_workers': 4}, 'eval': {'dataset': {'extra_tensor_keys': ['conditioning_fps', 'prepend_state_to_action'], 'img_buffer_len': 1, 'transforms': [{'embodiment_id': 5, 'img_keys': ['agentview_image', 'robot0_eye_in_hand_image'], 'type': 'ProcessLiberoEvalInputs'}, {'conditioning_fps': 20.0, 'prepend_state_to_action': False, 'type': 'SetCosmos3ActionMetadata'}, {'image_resize_strategy': 'resize-crop', 'input_sizes': [[3, 256, 256], [3, 256, 256]], 'means': [[127.5, 127.5, 127.5], [127.5, 127.5, 127.5]], 'stds': [[127.5, 127.5, 127.5], [127.5, 127.5, 127.5]], 'type': 'TransformImage'}, {'action_metadata': {'append_viewpoint': False, 'conditioning_fps': 20.0, 'frame_window_size': 17, 'video_height': 192, 'video_width': 320}, 'cfg_dropout_rate': 0.0, 'format_prompt_as_json': True, 'max_len': 512, 'output_attention_mask_key': 'lang_masks', 'output_key': 'lang_tokens', 'tokenizer': {'model_max_length': 4096, 'model_path': './checkpoints/Cosmos3-Edge', 'padding_side': 'right', 'trust_remote_code': True, 'type': 'PretrainedTokenizer'}, 'type': 'ProcessCosmos3Prompt'}, {'gripper_key': 'robot0_gripper_qpos', 'norm_type': 'mean_std', 'out_key': 'states', 'pos_key': 'robot0_eef_pos', 'quat_key': 'robot0_eef_quat', 'state_dim': 64, 'type': 'LiberoProprioFromInputs'}, {'frame_window_size': 1, 'num_views': 2, 'tile_direction': 'horizontal', 'type': 'PrepareVideo'}, {'height': 192, 'type': 'ResizeAndReflectPad', 'width': 320}], 'type': 'LiberoParquetEvalDataset'}, 'denormalize_action': {'action_dim': 10, 'norm_type': 'quantile_rot', 'type': 'DenormalizeLiberoFramewiseRot6DAction'}, 'enable_mixed_precision_training': True, 'eval_chunk_size': 16, 'inference_seed': 7, 'mixed_precision_dtype': 'bf16', 'model_family': 'cosmos3', 'norm_stats_key': 'all_libero_no_noops', 'num_inference_steps': 30, 'num_steps_wait': 10, 'num_trials_per_task': 50, 'resize_size': 256, 'seed': 7, 'task_ids': None, 'task_suite_name': ['libero_spatial', 'libero_object', 'libero_goal', 'libero_10'], 'type': 'LiberoEvalRunner', 'run_id_suffix': 'mixed-v21-shift10-after-train'}}", "run_id": "cosmos3edge_libero_full_finetune_2026_08_17_03_08_18"}

cosmos3_edge_libero_full_finetune_bs2048/tokenizer/chat_template.jinja

@@ -0,0 +1,324 @@
+
+{% macro render_extra_keys(json_dict, handled_keys) %}
+    {%- if json_dict is mapping %}
+        {%- for json_key in json_dict if json_key not in handled_keys %}
+            {%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
+                {{- '
+<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
+            {%- else %}
+                {{-'
+<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
+            {%- endif %}
+        {%- endfor %}
+    {%- endif %}
+{% endmacro %}
+{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %}
+{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}
+
+{%- set ns = namespace(last_user_idx = -1) %}
+{%- set loop_messages = messages %}
+{%- for m in loop_messages %}
+  {%- if m["role"] == "user" %}
+    {%- set ns.last_user_idx = loop.index0 %}
+  {%- endif %}
+{%- endfor %}
+
+{%- if messages[0]["role"] == "system" %}
+    {%- if messages[0]["content"] is string %}
+        {%- set system_message = messages[0]["content"] %}
+    {%- else %}
+        {%- set ns = namespace(system_text="") %}
+        {%- for part in messages[0]["content"] %}
+            {%- if 'text' in part %}
+                {%- set ns.system_text = ns.system_text + part.text %}
+            {%- endif %}
+        {%- endfor %}
+        {%- set system_message = ns.system_text %}
+    {%- endif %}
+
+    {%- set loop_messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+    {%- set loop_messages = messages %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = [] %}
+{%- endif %}
+{# Recompute last_user_idx relative to loop_messages after handling system #}
+{%- set ns = namespace(last_user_idx = -1) %}
+{%- for m in loop_messages %}
+  {%- if m["role"] == "user" %}
+    {%- set ns.last_user_idx = loop.index0 %}
+  {%- endif %}
+{%- endfor %}
+{%- if system_message is defined %}
+    {{- "<|im_start|>system
+" + system_message }}
+{%- else %}
+    {%- if tools is iterable and tools | length > 0 %}
+        {{- "<|im_start|>system
+" }}
+    {%- endif %}
+{%- endif %}
+{%- if tools is iterable and tools | length > 0 %}
+    {%- if system_message is defined and system_message | length > 0 %}
+        {{- "
+
+" }}
+    {%- endif %}
+    {{- "# Tools
+
+You have access to the following functions:
+
+" }}
+    {{- "<tools>" }}
+    {%- for tool in tools %}
+        {%- if tool.function is defined %}
+            {%- set tool = tool.function %}
+        {%- endif %}
+        {{- "
+<function>
+<name>" ~ tool.name ~ "</name>" }}
+        {%- if tool.description is defined %}
+            {{- '
+<description>' ~ (tool.description | trim) ~ '</description>' }}
+        {%- endif %}
+        {{- '
+<parameters>' }}
+        {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
+            {%- for param_name, param_fields in tool.parameters.properties|items %}
+                {{- '
+<parameter>' }}
+                {{- '
+<name>' ~ param_name ~ '</name>' }}
+                {%- if param_fields.type is defined %}
+                    {{- '
+<type>' ~ (param_fields.type | string) ~ '</type>' }}
+                {%- endif %}
+                {%- if param_fields.description is defined %}
+                    {{- '
+<description>' ~ (param_fields.description | trim) ~ '</description>' }}
+                {%- endif %}
+                {%- if param_fields.enum is defined %}
+                    {{- '
+<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}
+                {%- endif %}
+                {%- set handled_keys = ['name', 'type', 'description', 'enum'] %}
+                {{- render_extra_keys(param_fields, handled_keys) }}
+                {{- '
+</parameter>' }}
+            {%- endfor %}
+        {%- endif %}
+        {% set handled_keys = ['type', 'properties', 'required'] %}
+        {{- render_extra_keys(tool.parameters, handled_keys) }}
+        {%- if tool.parameters is defined and tool.parameters.required is defined %}
+            {{- '
+<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}
+        {%- endif %}
+        {{- '
+</parameters>' }}
+        {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
+        {{- render_extra_keys(tool, handled_keys) }}
+        {{- '
+</function>' }}
+    {%- endfor %}
+    {{- "
+</tools>" }}
+
+    {{- '
+
+If you choose to call a function ONLY reply in the following format with NO suffix:
+
+<tool_call>
+<function=example_function_name>
+<parameter=example_parameter_1>
+value_1
+</parameter>
+<parameter=example_parameter_2>
+This is the value for the second parameter
+that can span
+multiple lines
+</parameter>
+</function>
+</tool_call>
+
+<IMPORTANT>
+Reminder:
+- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags
+- Required parameters MUST be specified
+- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after
+- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls
+</IMPORTANT>' }}
+{%- endif %}
+
+
+{%- if system_message is defined %}
+    {{- '<|im_end|>
+' }}
+{%- else %}
+    {%- if tools is iterable and tools | length > 0 %}
+        {{- '<|im_end|>
+' }}
+    {%- endif %}
+{%- endif %}
+
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- for message in loop_messages %}
+    {%- if message.role == "assistant" %}
+        {# Add reasoning content in to content field for unified processing below. #}
+        {%- set text_content = namespace(text_content="") -%}
+        {%- if message.content is string %}
+            {%- set text_content.text_content = message.content %}
+        {%- else %}
+            {%- for content in message.content %}
+                {%- if 'text' in content %}
+                    {%- set text_content.text_content = content.text %}
+                    {%- break %}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}
+            {%- set content = "<think>
+" ~ message.reasoning_content ~ "
+</think>
+" ~ (text_content.text_content | default('', true)) %}
+        {%- else %}
+            {%- set content = text_content.text_content | default('', true) %}
+            {# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}
+            {%- if '<think>' not in content and '</think>' not in content %}
+                {%- set content = "<think></think>" ~ content %}
+            {%- else %}
+                {%- set content = content %}
+            {%- endif -%}
+        {%- endif %}
+        {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
+            {# Assistant message has tool calls. #}
+            {{- '<|im_start|>assistant
+' }}
+                {%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
+                {%- if content is string and content | trim | length > 0 %}
+                    {%- if include_content %}
+                        {{- (content | trim) ~ '
+' -}}
+                    {%- else %}
+                        {%- set c = (content | string) %}
+                        {%- if '</think>' in c %}
+                            {# Keep only content after the last closing think. Also generation prompt causes this. #}
+                            {%- set c = c.split('</think>')[-1] %}
+                        {%- elif '<think>' in c %}
+                            {# If <think> was opened but never closed, drop the trailing think segment #}
+                            {%- set c = c.split('<think>')[0] %}
+                        {%- endif %}
+                        {%- set c = "<think></think>" ~ c | trim %}
+                        {%- if c | length > 0 %}
+                            {{- c ~ '
+' -}}
+                        {%- endif %}
+                    {%- endif %}
+                {%- else %}
+                    {{- "<think></think>" -}}
+                {%- endif %}
+                {%- for tool_call in message.tool_calls %}
+                    {%- if tool_call.function is defined %}
+                        {%- set tool_call = tool_call.function %}
+                    {%- endif %}
+                    {{- '<tool_call>
+<function=' ~ tool_call.name ~ '>
+' -}}
+                        {%- if tool_call.arguments is defined %}
+                            {%- for args_name, args_value in tool_call.arguments|items %}
+                                {{- '<parameter=' ~ args_name ~ '>
+' -}}
+                                    {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                                {{- args_value ~ '
+</parameter>
+' -}}
+                            {%- endfor %}
+                        {%- endif %}
+                    {{- '</function>
+</tool_call>
+' -}}
+                {%- endfor %}
+                {{- '<|im_end|>
+' }}
+        {%- else %}
+            {# Assistant message doesn't have tool calls. #}
+            {%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
+                {{- '<|im_start|>assistant
+' ~ (content | default('', true) | string | trim) ~ '<|im_end|>
+' }}
+            {%- else %}
+                {%- set c = (content | default('', true) | string) %}
+                {%- if '<think>' in c and '</think>' in c %}
+                    {%- set c = "<think></think>" ~ c.split('</think>')[-1] %}
+                {%- endif %}
+                {%- set c = c | trim %}
+                {%- if c | length > 0 %}
+                    {{- '<|im_start|>assistant
+' ~ c ~ '<|im_end|>
+' }}
+                {%- else %}
+                    {{- '<|im_start|>assistant
+<|im_end|>
+' }}
+                {%- endif %}
+            {%- endif %}
+        {%- endif %}
+    {%- elif message.role == "user" or message.role == "system" %}
+        {{- '<|im_start|>' + message.role + '
+' }}
+        {%- if message.content is string %}
+            {{- message.content }}
+        {%- else %}
+            {%- for content in message.content %}
+                {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                    {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
+                    <|vision_start|><|image_pad|><|vision_end|>
+                {%- elif content.type == 'video' or 'video' in content %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                    {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
+                    <|vision_start|><|video_pad|><|vision_end|>
+                {%- elif 'text' in content %}
+                    {{- content.text }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>
+' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user
+' }}
+        {%- endif %}
+        {{- '<tool_response>
+' }}
+        {{- message.content }}
+        {{- '
+</tool_response>
+' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>
+' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>
+' }}
+        {%- endif %}
+    {%- else %}
+        {{- '<|im_start|>' + message.role + '
+' + message.content + '<|im_end|>
+' }}
+    {%- endif %}
+{%- endfor %}
+
+{%- if add_generation_prompt %}
+    {%- if enable_thinking %}
+        {{- '<|im_start|>assistant
+<think>
+' }}
+    {%- else %}
+        {{- '<|im_start|>assistant
+<think></think>' }}
+    {%- endif %}
+{%- endif %}

cosmos3_edge_libero_full_finetune_bs2048/tokenizer/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

cosmos3_edge_libero_full_finetune_bs2048/tokenizer/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4dc692a99dca6d92a44e9529ffb53541eb79bb260eec7281bc51268c05d7311c
+size 17077500

cosmos3_edge_libero_full_finetune_bs2048/tokenizer/tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "is_local": true,
+  "local_files_only": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 262144,
+  "pad_token": "<|im_end|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<unk>"
+}

cosmos3_edge_libero_full_finetune_bs2048/vlm_backbone_config.json

@@ -0,0 +1,41 @@
+{
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "enable_mrope": true,
+  "enable_rope": true,
+  "eos_token_id": 11,
+  "head_dim": 128,
+  "hidden_dropout": 0.0,
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 9216,
+  "layer_norm_epsilon": 1e-05,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "mlp_hidden_act": "relu2",
+  "model_type": "nemotron_3_dense_vl_text",
+  "mrope_section": [
+    24,
+    20,
+    20
+  ],
+  "num_attention_heads": 16,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "num_logits_to_keep": 1,
+  "packed_attention_backend": "flash2",
+  "pad_token_id": 11,
+  "residual_in_fp32": false,
+  "rope_parameters": {
+    "rope_theta": 100000000.0,
+    "rope_type": "default"
+  },
+  "rope_theta": 100000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.3.0",
+  "use_cache": true,
+  "use_und_k_norm_for_gen": true,
+  "vocab_size": 131072
+}

cosmos3_nano_libero_full_finetune_bs2048/checkpoints/step-005000-epoch-22-loss=0.3622.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:418c6bb0215ad0598f6a16c6633a91a0381c905ada647d6d93dc7bb87f915277
+size 63511447112

cosmos3_nano_libero_full_finetune_bs2048/config.json

@@ -0,0 +1,1740 @@
+{
+  "_action_dim": 10,
+  "_action_horizon": 16,
+  "_action_lr": 0.00025,
+  "_base_lr": 5e-05,
+  "_cfg_dropout_rate": 0.1,
+  "_ckpt_root": "./checkpoints",
+  "_conditioning_fps": 20.0,
+  "_cosmos3_nano_ckpt": "./checkpoints/Cosmos3-Nano",
+  "_cosmos3_nano_name_mapping": {
+    ".self_attn.k_norm.": ".self_attn.norm_k.",
+    ".self_attn.k_norm_moe_gen.": ".self_attn.norm_added_k.",
+    ".self_attn.k_proj.": ".self_attn.to_k.",
+    ".self_attn.k_proj_moe_gen.": ".self_attn.add_k_proj.",
+    ".self_attn.o_proj.": ".self_attn.to_out.",
+    ".self_attn.o_proj_moe_gen.": ".self_attn.to_add_out.",
+    ".self_attn.q_norm.": ".self_attn.norm_q.",
+    ".self_attn.q_norm_moe_gen.": ".self_attn.norm_added_q.",
+    ".self_attn.q_proj.": ".self_attn.to_q.",
+    ".self_attn.q_proj_moe_gen.": ".self_attn.add_q_proj.",
+    ".self_attn.v_proj.": ".self_attn.to_v.",
+    ".self_attn.v_proj_moe_gen.": ".self_attn.add_v_proj.",
+    "action_in_proj.": "action_proj_in.",
+    "action_modality_embed": "action_modality_embed",
+    "action_out_proj.": "action_proj_out.",
+    "time_embedder.mlp.0.": "time_embedder.linear_1.",
+    "time_embedder.mlp.2.": "time_embedder.linear_2.",
+    "vision_in_proj.projector.": "proj_in.",
+    "vision_out_proj.projector.": "proj_out.",
+    "vlm_backbone.lm_head.weight": "lm_head.weight",
+    "vlm_backbone.model.language_model.embed_tokens.weight": "embed_tokens.weight",
+    "vlm_backbone.model.language_model.layers.": "layers.",
+    "vlm_backbone.model.language_model.norm.weight": "norm.weight",
+    "vlm_backbone.model.language_model.norm_moe_gen.weight": "norm_moe_gen.weight",
+    "vlm_backbone.model.visual.blocks.": "blocks.",
+    "vlm_backbone.model.visual.deepstack_merger_list.": "deepstack_merger_list.",
+    "vlm_backbone.model.visual.merger.": "merger.",
+    "vlm_backbone.model.visual.patch_embed.": "patch_embed.",
+    "vlm_backbone.model.visual.pos_embed.": "pos_embed."
+  },
+  "_cosmos3_nano_special_tokens": {
+    "end_of_generation": 151653,
+    "eos_token_id": 151645,
+    "start_of_generation": 151652
+  },
+  "_cosmos3_nano_tokenizer": {
+    "model_max_length": 4096,
+    "model_path": "./checkpoints/Cosmos3-Nano/text_tokenizer",
+    "padding_side": "right",
+    "trust_remote_code": true,
+    "type": "PretrainedTokenizer"
+  },
+  "_cosmos3_nano_transformer": "./checkpoints/Cosmos3-Nano/transformer",
+  "_cosmos3_nano_vision_encoder": "./checkpoints/Cosmos3-Nano/vision_encoder",
+  "_cosmos3_nano_vlm_config": {
+    "image_token_id": 151655,
+    "model_type": "qwen3_vl",
+    "text_config": {
+      "attention_bias": false,
+      "attention_dropout": 0.0,
+      "bos_token_id": 151643,
+      "eos_token_id": 151645,
+      "head_dim": 128,
+      "hidden_act": "silu",
+      "hidden_size": 4096,
+      "intermediate_size": 12288,
+      "layer_types": [
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention",
+        "full_attention"
+      ],
+      "max_position_embeddings": 262144,
+      "model_type": "qwen3_vl_text",
+      "num_attention_heads": 32,
+      "num_hidden_layers": 36,
+      "num_key_value_heads": 8,
+      "pad_token_id": 0,
+      "rms_norm_eps": 1e-06,
+      "rope_scaling": {
+        "mrope_interleaved": true,
+        "mrope_section": [
+          24,
+          20,
+          20
+        ],
+        "rope_type": "default"
+      },
+      "rope_theta": 5000000,
+      "tie_word_embeddings": false,
+      "vocab_size": 151936
+    },
+    "tie_word_embeddings": false,
+    "video_token_id": 151656,
+    "vision_config": {
+      "deepstack_visual_indexes": [
+        8,
+        16,
+        24
+      ],
+      "depth": 27,
+      "hidden_act": "gelu_pytorch_tanh",
+      "hidden_size": 1152,
+      "in_channels": 3,
+      "initializer_range": 0.02,
+      "intermediate_size": 4304,
+      "model_type": "qwen3_vl",
+      "num_heads": 16,
+      "num_position_embeddings": 2304,
+      "out_hidden_size": 4096,
+      "patch_size": 16,
+      "spatial_merge_size": 2,
+      "temporal_patch_size": 2
+    },
+    "vision_end_token_id": 151653,
+    "vision_start_token_id": 151652,
+    "vocab_size": 151936
+  },
+  "_data_root_path": [
+    "./datasets/libero_spatial_no_noops_lerobotv2.1",
+    "./datasets/libero_spatial_no_noops_lerobotv2.1",
+    "./datasets/libero_object_no_noops_lerobotv2.1",
+    "./datasets/libero_object_no_noops_lerobotv2.1",
+    "./datasets/libero_goal_no_noops_lerobotv2.1",
+    "./datasets/libero_goal_no_noops_lerobotv2.1",
+    "./datasets/libero_10_no_noops_lerobotv2.1"
+  ],
+  "_frame_window_size": 17,
+  "_grad_accumulation_steps": 8,
+  "_image_height": 256,
+  "_image_width": 256,
+  "_libero_action_stats": {
+    "max": [
+      0.9375,
+      0.9375,
+      0.9375,
+      1.0,
+      0.368853,
+      0.341214,
+      0.356395,
+      1.0,
+      0.348251,
+      1.0
+    ],
+    "mean": [
+      0.050704,
+      0.097407,
+      -0.094833,
+      0.994873,
+      -0.004579,
+      -0.004288,
+      0.004389,
+      0.996104,
+      0.001109,
+      0.476725
+    ],
+    "min": [
+      -0.9375,
+      -0.9375,
+      -0.9375,
+      0.902028,
+      -0.356085,
+      -0.367416,
+      -0.370434,
+      0.921907,
+      -0.255,
+      0.0
+    ],
+    "q01": [
+      -0.723214,
+      -0.808929,
+      -0.9375,
+      0.934955,
+      -0.223431,
+      -0.189878,
+      -0.334735,
+      0.938516,
+      -0.107736,
+      0.0
+    ],
+    "q99": [
+      0.9375,
+      0.870536,
+      0.9375,
+      1.0,
+      0.331,
+      0.163153,
+      0.226216,
+      1.0,
+      0.127158,
+      1.0
+    ],
+    "std": [
+      0.333621,
+      0.387175,
+      0.45714,
+      0.010807,
+      0.077802,
+      0.063386,
+      0.078571,
+      0.009994,
+      0.038504,
+      0.49946
+    ]
+  },
+  "_max_action_dim": 64,
+  "_max_state_dim": 64,
+  "_max_steps": 5000,
+  "_per_device_batch_size": 16,
+  "_prepend_state_to_action": false,
+  "_save_iter_interval": 500,
+  "_statistic_name": "all_libero_no_noops",
+  "_transforms": [
+    {
+      "embodiment_id": 5,
+      "name_mappings": {
+        "actions": [
+          "actions"
+        ],
+        "observation.state": [
+          "states"
+        ]
+      },
+      "parquet_keys": [
+        "observation.state",
+        "timestamp",
+        "actions",
+        "info",
+        "stats",
+        "action_masks"
+      ],
+      "type": "ProcessParquetInputs",
+      "video_keys": [
+        "observation.images.image",
+        "observation.images.wrist_image"
+      ]
+    },
+    {
+      "type": "LiberoFramewiseActionToRot6D"
+    },
+    {
+      "action_metadata": {
+        "append_viewpoint": false,
+        "conditioning_fps": 20.0,
+        "frame_window_size": 17,
+        "video_height": 192,
+        "video_width": 320
+      },
+      "cfg_dropout_rate": 0.1,
+      "format_prompt_as_json": true,
+      "max_len": 512,
+      "tokenizer": {
+        "model_max_length": 4096,
+        "model_path": "./checkpoints/Cosmos3-Nano/text_tokenizer",
+        "padding_side": "right",
+        "trust_remote_code": true,
+        "type": "PretrainedTokenizer"
+      },
+      "type": "ProcessCosmos3Prompt"
+    },
+    {
+      "height": 256,
+      "type": "ResizeImages",
+      "width": 256
+    },
+    {
+      "type": "SimpleNormalizeImages"
+    },
+    {
+      "action_dim": 64,
+      "action_key": "action",
+      "action_norm_type": "quantile",
+      "state_dim": 64,
+      "state_key": "proprio",
+      "state_norm_type": "none",
+      "type": "NormalizeStatesAndActions"
+    },
+    {
+      "conditioning_fps": 20.0,
+      "frame_window_size": 17,
+      "mode": "wam",
+      "prepend_state_to_action": false,
+      "raw_action_dim": 10,
+      "type": "BuildCosmos3Sequence"
+    },
+    {
+      "frame_window_size": 17,
+      "num_views": 2,
+      "tile_direction": "horizontal",
+      "type": "PrepareVideo"
+    },
+    {
+      "height": 192,
+      "type": "ResizeAndReflectPad",
+      "width": 320
+    }
+  ],
+  "_video_height": 192,
+  "_video_width": 320,
+  "_vision_vae": {
+    "encode_exact_durations": [
+      17
+    ],
+    "pretrained_name_or_path": "./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth",
+    "type": "Cosmos3Wan22VAE"
+  },
+  "_wan22_vae_path": "./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth",
+  "eval": {
+    "dataset": {
+      "extra_tensor_keys": [
+        "conditioning_fps",
+        "prepend_state_to_action"
+      ],
+      "img_buffer_len": 1,
+      "transforms": [
+        {
+          "embodiment_id": 5,
+          "img_keys": [
+            "agentview_image",
+            "robot0_eye_in_hand_image"
+          ],
+          "resize_size": 256,
+          "type": "ProcessLiberoEvalInputs"
+        },
+        {
+          "conditioning_fps": 20.0,
+          "prepend_state_to_action": false,
+          "type": "SetCosmos3ActionMetadata"
+        },
+        {
+          "image_resize_strategy": "resize-crop",
+          "input_sizes": [
+            [
+              3,
+              256,
+              256
+            ],
+            [
+              3,
+              256,
+              256
+            ]
+          ],
+          "means": [
+            [
+              127.5,
+              127.5,
+              127.5
+            ],
+            [
+              127.5,
+              127.5,
+              127.5
+            ]
+          ],
+          "stds": [
+            [
+              127.5,
+              127.5,
+              127.5
+            ],
+            [
+              127.5,
+              127.5,
+              127.5
+            ]
+          ],
+          "type": "TransformImage"
+        },
+        {
+          "action_metadata": {
+            "append_viewpoint": false,
+            "conditioning_fps": 20.0,
+            "frame_window_size": 17,
+            "video_height": 192,
+            "video_width": 320
+          },
+          "cfg_dropout_rate": 0.0,
+          "format_prompt_as_json": true,
+          "max_len": 512,
+          "output_attention_mask_key": "lang_masks",
+          "output_key": "lang_tokens",
+          "tokenizer": {
+            "model_max_length": 4096,
+            "model_path": "./checkpoints/Cosmos3-Nano/text_tokenizer",
+            "padding_side": "right",
+            "trust_remote_code": true,
+            "type": "PretrainedTokenizer"
+          },
+          "type": "ProcessCosmos3Prompt"
+        },
+        {
+          "frame_window_size": 1,
+          "num_views": 2,
+          "tile_direction": "horizontal",
+          "type": "PrepareVideo"
+        },
+        {
+          "height": 192,
+          "type": "ResizeAndReflectPad",
+          "width": 320
+        }
+      ],
+      "type": "LiberoParquetEvalDataset"
+    },
+    "denormalize_action": {
+      "action_dim": 10,
+      "norm_type": "quantile_rot",
+      "type": "DenormalizeLiberoFramewiseRot6DAction"
+    },
+    "enable_mixed_precision_training": true,
+    "eval_chunk_size": 16,
+    "mixed_precision_dtype": "bf16",
+    "model_family": "cosmos3",
+    "norm_stats_key": "all_libero_no_noops",
+    "num_steps_wait": 10,
+    "num_trials_per_task": 50,
+    "seed": 7,
+    "task_suite_name": "libero_10",
+    "type": "LiberoEvalRunner"
+  },
+  "inference_model": {
+    "action_horizon": 16,
+    "action_in_proj": {
+      "input_size": 64,
+      "num_domains": 32,
+      "output_size": 4096,
+      "type": "DomainAwareLinear"
+    },
+    "action_out_proj": {
+      "input_size": 4096,
+      "num_domains": 32,
+      "output_size": 64,
+      "type": "DomainAwareLinear"
+    },
+    "base_fps": 24.0,
+    "enable_fps_modulation": true,
+    "enable_vision_loss": true,
+    "freeze_non_moe_vlm_backbone": true,
+    "freeze_vlm_backbone": false,
+    "latent_patch_size": 2,
+    "max_action_dim": 64,
+    "name_mapping": {
+      ".self_attn.k_norm.": ".self_attn.norm_k.",
+      ".self_attn.k_norm_moe_gen.": ".self_attn.norm_added_k.",
+      ".self_attn.k_proj.": ".self_attn.to_k.",
+      ".self_attn.k_proj_moe_gen.": ".self_attn.add_k_proj.",
+      ".self_attn.o_proj.": ".self_attn.to_out.",
+      ".self_attn.o_proj_moe_gen.": ".self_attn.to_add_out.",
+      ".self_attn.q_norm.": ".self_attn.norm_q.",
+      ".self_attn.q_norm_moe_gen.": ".self_attn.norm_added_q.",
+      ".self_attn.q_proj.": ".self_attn.to_q.",
+      ".self_attn.q_proj_moe_gen.": ".self_attn.add_q_proj.",
+      ".self_attn.v_proj.": ".self_attn.to_v.",
+      ".self_attn.v_proj_moe_gen.": ".self_attn.add_v_proj.",
+      "action_in_proj.": "action_proj_in.",
+      "action_modality_embed": "action_modality_embed",
+      "action_out_proj.": "action_proj_out.",
+      "time_embedder.mlp.0.": "time_embedder.linear_1.",
+      "time_embedder.mlp.2.": "time_embedder.linear_2.",
+      "vision_in_proj.projector.": "proj_in.",
+      "vision_out_proj.projector.": "proj_out.",
+      "vlm_backbone.lm_head.weight": "lm_head.weight",
+      "vlm_backbone.model.language_model.embed_tokens.weight": "embed_tokens.weight",
+      "vlm_backbone.model.language_model.layers.": "layers.",
+      "vlm_backbone.model.language_model.norm.weight": "norm.weight",
+      "vlm_backbone.model.language_model.norm_moe_gen.weight": "norm_moe_gen.weight",
+      "vlm_backbone.model.visual.blocks.": "blocks.",
+      "vlm_backbone.model.visual.deepstack_merger_list.": "deepstack_merger_list.",
+      "vlm_backbone.model.visual.merger.": "merger.",
+      "vlm_backbone.model.visual.patch_embed.": "patch_embed.",
+      "vlm_backbone.model.visual.pos_embed.": "pos_embed."
+    },
+    "num_embodiment_domains": 32,
+    "ori_action_dim": 10,
+    "packed_attention_backend": "flash2",
+    "position_embedding_type": "unified_3d_mrope",
+    "pretrained_name_or_path": "./checkpoints/Cosmos3-Nano/transformer",
+    "rectified_flow_inference_config": {
+      "num_steps": 30,
+      "num_train_timesteps": 1000,
+      "scheduler_type": "unipc",
+      "shift": 10.0,
+      "use_dynamic_shifting": false,
+      "use_karras_sigmas": false
+    },
+    "rectified_flow_training_config": {
+      "action_loss_weight": 10.0,
+      "high_sigma_ratio": 0.05,
+      "high_sigma_timesteps_max": 1000,
+      "high_sigma_timesteps_min": 995,
+      "independent_action_schedule": false,
+      "normalize_loss_by_active": false,
+      "shift": {
+        "256": 3,
+        "480": 5,
+        "720": 10
+      },
+      "shift_action": null,
+      "train_time_action_distribution": "logitnormal",
+      "train_time_image_distribution": "logitnormal",
+      "train_time_video_distribution": "waver",
+      "train_time_weight": "uniform",
+      "use_discrete_rf": false,
+      "use_dynamic_shift": false,
+      "use_high_sigma_strategy": false,
+      "use_high_sigma_strategy_action": false,
+      "vision_loss_weight": 10.0
+    },
+    "special_tokens": {
+      "end_of_generation": 151653,
+      "eos_token_id": 151645,
+      "start_of_generation": 151652
+    },
+    "timestep_scale": 0.001,
+    "type": "Cosmos3FlowMatching",
+    "unified_3d_mrope_reset_spatial_ids": true,
+    "unified_3d_mrope_temporal_modality_margin": 15000,
+    "vision_in_proj": {
+      "in_dim": 192,
+      "out_dim": 4096,
+      "type": "LinearProjector"
+    },
+    "vision_latent_dim": 48,
+    "vision_out_proj": {
+      "in_dim": 4096,
+      "out_dim": 192,
+      "type": "LinearProjector"
+    },
+    "vision_vae": {
+      "encode_exact_durations": [
+        17
+      ],
+      "pretrained_name_or_path": null,
+      "type": "Cosmos3Wan22VAE"
+    },
+    "vlm_backbone": {
+      "include_visual": false,
+      "skip_init_weights": true,
+      "type": "Cosmos3MoTBackbone",
+      "vision_encoder_path": "./checkpoints/Cosmos3-Nano/vision_encoder",
+      "vlm_config": {
+        "image_token_id": 151655,
+        "model_type": "qwen3_vl",
+        "text_config": {
+          "attention_bias": false,
+          "attention_dropout": 0.0,
+          "bos_token_id": 151643,
+          "eos_token_id": 151645,
+          "head_dim": 128,
+          "hidden_act": "silu",
+          "hidden_size": 4096,
+          "intermediate_size": 12288,
+          "layer_types": [
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention"
+          ],
+          "max_position_embeddings": 262144,
+          "model_type": "qwen3_vl_text",
+          "num_attention_heads": 32,
+          "num_hidden_layers": 36,
+          "num_key_value_heads": 8,
+          "pad_token_id": 0,
+          "rms_norm_eps": 1e-06,
+          "rope_scaling": {
+            "mrope_interleaved": true,
+            "mrope_section": [
+              24,
+              20,
+              20
+            ],
+            "rope_type": "default"
+          },
+          "rope_theta": 5000000,
+          "tie_word_embeddings": false,
+          "vocab_size": 151936
+        },
+        "tie_word_embeddings": false,
+        "video_token_id": 151656,
+        "vision_config": {
+          "deepstack_visual_indexes": [
+            8,
+            16,
+            24
+          ],
+          "depth": 27,
+          "hidden_act": "gelu_pytorch_tanh",
+          "hidden_size": 1152,
+          "in_channels": 3,
+          "initializer_range": 0.02,
+          "intermediate_size": 4304,
+          "model_type": "qwen3_vl",
+          "num_heads": 16,
+          "num_position_embeddings": 2304,
+          "out_hidden_size": 4096,
+          "patch_size": 16,
+          "spatial_merge_size": 2,
+          "temporal_patch_size": 2
+        },
+        "vision_end_token_id": 151653,
+        "vision_start_token_id": 151652,
+        "vocab_size": 151936
+      }
+    }
+  },
+  "model": {
+    "action_horizon": 16,
+    "action_in_proj": {
+      "input_size": 64,
+      "num_domains": 32,
+      "output_size": 4096,
+      "type": "DomainAwareLinear"
+    },
+    "action_out_proj": {
+      "input_size": 4096,
+      "num_domains": 32,
+      "output_size": 64,
+      "type": "DomainAwareLinear"
+    },
+    "base_fps": 24.0,
+    "enable_fps_modulation": true,
+    "enable_vision_loss": true,
+    "freeze_non_moe_vlm_backbone": true,
+    "freeze_vlm_backbone": false,
+    "latent_patch_size": 2,
+    "max_action_dim": 64,
+    "name_mapping": {
+      ".self_attn.k_norm.": ".self_attn.norm_k.",
+      ".self_attn.k_norm_moe_gen.": ".self_attn.norm_added_k.",
+      ".self_attn.k_proj.": ".self_attn.to_k.",
+      ".self_attn.k_proj_moe_gen.": ".self_attn.add_k_proj.",
+      ".self_attn.o_proj.": ".self_attn.to_out.",
+      ".self_attn.o_proj_moe_gen.": ".self_attn.to_add_out.",
+      ".self_attn.q_norm.": ".self_attn.norm_q.",
+      ".self_attn.q_norm_moe_gen.": ".self_attn.norm_added_q.",
+      ".self_attn.q_proj.": ".self_attn.to_q.",
+      ".self_attn.q_proj_moe_gen.": ".self_attn.add_q_proj.",
+      ".self_attn.v_proj.": ".self_attn.to_v.",
+      ".self_attn.v_proj_moe_gen.": ".self_attn.add_v_proj.",
+      "action_in_proj.": "action_proj_in.",
+      "action_modality_embed": "action_modality_embed",
+      "action_out_proj.": "action_proj_out.",
+      "time_embedder.mlp.0.": "time_embedder.linear_1.",
+      "time_embedder.mlp.2.": "time_embedder.linear_2.",
+      "vision_in_proj.projector.": "proj_in.",
+      "vision_out_proj.projector.": "proj_out.",
+      "vlm_backbone.lm_head.weight": "lm_head.weight",
+      "vlm_backbone.model.language_model.embed_tokens.weight": "embed_tokens.weight",
+      "vlm_backbone.model.language_model.layers.": "layers.",
+      "vlm_backbone.model.language_model.norm.weight": "norm.weight",
+      "vlm_backbone.model.language_model.norm_moe_gen.weight": "norm_moe_gen.weight",
+      "vlm_backbone.model.visual.blocks.": "blocks.",
+      "vlm_backbone.model.visual.deepstack_merger_list.": "deepstack_merger_list.",
+      "vlm_backbone.model.visual.merger.": "merger.",
+      "vlm_backbone.model.visual.patch_embed.": "patch_embed.",
+      "vlm_backbone.model.visual.pos_embed.": "pos_embed."
+    },
+    "num_embodiment_domains": 32,
+    "ori_action_dim": 10,
+    "packed_attention_backend": "flash2",
+    "position_embedding_type": "unified_3d_mrope",
+    "pretrained_name_or_path": "./checkpoints/Cosmos3-Nano/transformer",
+    "rectified_flow_inference_config": {
+      "num_steps": 30,
+      "num_train_timesteps": 1000,
+      "scheduler_type": "unipc",
+      "shift": 10.0,
+      "use_dynamic_shifting": false,
+      "use_karras_sigmas": false
+    },
+    "rectified_flow_training_config": {
+      "action_loss_weight": 10.0,
+      "high_sigma_ratio": 0.05,
+      "high_sigma_timesteps_max": 1000,
+      "high_sigma_timesteps_min": 995,
+      "independent_action_schedule": false,
+      "normalize_loss_by_active": false,
+      "shift": {
+        "256": 3,
+        "480": 5,
+        "720": 10
+      },
+      "shift_action": null,
+      "train_time_action_distribution": "logitnormal",
+      "train_time_image_distribution": "logitnormal",
+      "train_time_video_distribution": "waver",
+      "train_time_weight": "uniform",
+      "use_discrete_rf": false,
+      "use_dynamic_shift": false,
+      "use_high_sigma_strategy": false,
+      "use_high_sigma_strategy_action": false,
+      "vision_loss_weight": 10.0
+    },
+    "special_tokens": {
+      "end_of_generation": 151653,
+      "eos_token_id": 151645,
+      "start_of_generation": 151652
+    },
+    "timestep_scale": 0.001,
+    "type": "Cosmos3FlowMatching",
+    "unified_3d_mrope_reset_spatial_ids": true,
+    "unified_3d_mrope_temporal_modality_margin": 15000,
+    "vision_in_proj": {
+      "in_dim": 192,
+      "out_dim": 4096,
+      "type": "LinearProjector"
+    },
+    "vision_latent_dim": 48,
+    "vision_out_proj": {
+      "in_dim": 4096,
+      "out_dim": 192,
+      "type": "LinearProjector"
+    },
+    "vision_vae": {
+      "encode_exact_durations": [
+        17
+      ],
+      "pretrained_name_or_path": "./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth",
+      "type": "Cosmos3Wan22VAE"
+    },
+    "vlm_backbone": {
+      "include_visual": false,
+      "skip_init_weights": true,
+      "type": "Cosmos3MoTBackbone",
+      "vision_encoder_path": "./checkpoints/Cosmos3-Nano/vision_encoder",
+      "vlm_config": {
+        "image_token_id": 151655,
+        "model_type": "qwen3_vl",
+        "text_config": {
+          "attention_bias": false,
+          "attention_dropout": 0.0,
+          "bos_token_id": 151643,
+          "eos_token_id": 151645,
+          "head_dim": 128,
+          "hidden_act": "silu",
+          "hidden_size": 4096,
+          "intermediate_size": 12288,
+          "layer_types": [
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention",
+            "full_attention"
+          ],
+          "max_position_embeddings": 262144,
+          "model_type": "qwen3_vl_text",
+          "num_attention_heads": 32,
+          "num_hidden_layers": 36,
+          "num_key_value_heads": 8,
+          "pad_token_id": 0,
+          "rms_norm_eps": 1e-06,
+          "rope_scaling": {
+            "mrope_interleaved": true,
+            "mrope_section": [
+              24,
+              20,
+              20
+            ],
+            "rope_type": "default"
+          },
+          "rope_theta": 5000000,
+          "tie_word_embeddings": false,
+          "vocab_size": 151936
+        },
+        "tie_word_embeddings": false,
+        "video_token_id": 151656,
+        "vision_config": {
+          "deepstack_visual_indexes": [
+            8,
+            16,
+            24
+          ],
+          "depth": 27,
+          "hidden_act": "gelu_pytorch_tanh",
+          "hidden_size": 1152,
+          "in_channels": 3,
+          "initializer_range": 0.02,
+          "intermediate_size": 4304,
+          "model_type": "qwen3_vl",
+          "num_heads": 16,
+          "num_position_embeddings": 2304,
+          "out_hidden_size": 4096,
+          "patch_size": 16,
+          "spatial_merge_size": 2,
+          "temporal_patch_size": 2
+        },
+        "vision_end_token_id": 151653,
+        "vision_start_token_id": 151652,
+        "vocab_size": 151936
+      }
+    }
+  },
+  "runner": {
+    "change_key_name": false,
+    "collator": {
+      "list_keys": [
+        "sequence_plan"
+      ],
+      "meta_keys": [
+        "task_description",
+        "stats",
+        "info",
+        "timestamp"
+      ],
+      "pad_id": 0,
+      "sequence_keys": [
+        "text_token_ids"
+      ],
+      "tensor_keys": [
+        "images",
+        "actions",
+        "embodiment_ids",
+        "raw_action_dim",
+        "conditioning_fps",
+        "action_fps"
+      ],
+      "type": "Cosmos3Collator"
+    },
+    "enable_gradient_checkpointing": true,
+    "enable_mixed_precision_training": true,
+    "grad_accumulation_steps": 8,
+    "lr_scheduler": {
+      "cycle_length": 16000,
+      "type": "linear-warmup+linear-decay",
+      "warmup_steps": 500
+    },
+    "max_grad_norm": 1.0,
+    "max_keep_ckpts": 2,
+    "max_steps": 5000,
+    "metric": {
+      "active_trackers": [
+        "jsonl",
+        "wandb"
+      ],
+      "grad_accumulation_steps": 8,
+      "run_dir": "work_dirs",
+      "type": "VLAMetric",
+      "window_size": 1
+    },
+    "mixed_precision_dtype": "bf16",
+    "optimizer": {
+      "betas": [
+        0.9,
+        0.99
+      ],
+      "eps": 1e-08,
+      "exclude_1d_from_weight_decay": false,
+      "fused": true,
+      "lr": 5e-05,
+      "paramwise_learning_rate": {
+        "action_in_proj.": 0.00025,
+        "action_modality_embed": 0.00025,
+        "action_out_proj.": 0.00025
+      },
+      "type": "AdamW",
+      "weight_decay": 0.05
+    },
+    "sampler": null,
+    "save_iter_interval": 500,
+    "sharding_strategy": "full-shard",
+    "tokenizer": {
+      "model_max_length": 4096,
+      "model_path": "./checkpoints/Cosmos3-Nano/text_tokenizer",
+      "padding_side": "right",
+      "trust_remote_code": true,
+      "type": "PretrainedTokenizer"
+    },
+    "type": "FSDPTrainRunner"
+  },
+  "train_dataloader": {
+    "dataset": {
+      "datasets": [
+        {
+          "action_key": "action",
+          "action_window_size": 16,
+          "data_root_path": "./datasets/libero_spatial_no_noops_lerobotv2.1",
+          "frame_window_size": 17,
+          "require_full_window": true,
+          "statistic_name": "all_libero_no_noops",
+          "transforms": [
+            {
+              "embodiment_id": 5,
+              "name_mappings": {
+                "actions": [
+                  "actions"
+                ],
+                "observation.state": [
+                  "states"
+                ]
+              },
+              "parquet_keys": [
+                "observation.state",
+                "timestamp",
+                "actions",
+                "info",
+                "stats",
+                "action_masks"
+              ],
+              "type": "ProcessParquetInputs",
+              "video_keys": [
+                "observation.images.image",
+                "observation.images.wrist_image"
+              ]
+            },
+            {
+              "type": "LiberoFramewiseActionToRot6D"
+            },
+            {
+              "action_metadata": {
+                "append_viewpoint": false,
+                "conditioning_fps": 20.0,
+                "frame_window_size": 17,
+                "video_height": 192,
+                "video_width": 320
+              },
+              "cfg_dropout_rate": 0.1,
+              "format_prompt_as_json": true,
+              "max_len": 512,
+              "tokenizer": {
+                "model_max_length": 4096,
+                "model_path": "./checkpoints/Cosmos3-Nano/text_tokenizer",
+                "padding_side": "right",
+                "trust_remote_code": true,
+                "type": "PretrainedTokenizer"
+              },
+              "type": "ProcessCosmos3Prompt"
+            },
+            {
+              "height": 256,
+              "type": "ResizeImages",
+              "width": 256
+            },
+            {
+              "type": "SimpleNormalizeImages"
+            },
+            {
+              "action_dim": 64,
+              "action_key": "action",
+              "action_norm_type": "quantile",
+              "state_dim": 64,
+              "state_key": "proprio",
+              "state_norm_type": "none",
+              "type": "NormalizeStatesAndActions"
+            },
+            {
+              "conditioning_fps": 20.0,
+              "frame_window_size": 17,
+              "mode": "wam",
+              "prepend_state_to_action": false,
+              "raw_action_dim": 10,
+              "type": "BuildCosmos3Sequence"
+            },
+            {
+              "frame_window_size": 17,
+              "num_views": 2,
+              "tile_direction": "horizontal",
+              "type": "PrepareVideo"
+            },
+            {
+              "height": 192,
+              "type": "ResizeAndReflectPad",
+              "width": 320
+            }
+          ],
+          "type": "ParquetDataset",
+          "use_delta": false,
+          "window_start_idx": 0
+        },
+        {
+          "action_key": "action",
+          "action_window_size": 16,
+          "data_root_path": "./datasets/libero_spatial_no_noops_lerobotv2.1",
+          "frame_window_size": 17,
+          "require_full_window": true,
+          "statistic_name": "all_libero_no_noops",
+          "transforms": [
+            {
+              "embodiment_id": 5,
+              "name_mappings": {
+                "actions": [
+                  "actions"
+                ],
+                "observation.state": [
+                  "states"
+                ]
+              },
+              "parquet_keys": [
+                "observation.state",
+                "timestamp",
+                "actions",
+                "info",
+                "stats",
+                "action_masks"
+              ],
+              "type": "ProcessParquetInputs",
+              "video_keys": [
+                "observation.images.image",
+                "observation.images.wrist_image"
+              ]
+            },
+            {
+              "type": "LiberoFramewiseActionToRot6D"
+            },
+            {
+              "action_metadata": {
+                "append_viewpoint": false,
+                "conditioning_fps": 20.0,
+                "frame_window_size": 17,
+                "video_height": 192,
+                "video_width": 320
+              },
+              "cfg_dropout_rate": 0.1,
+              "format_prompt_as_json": true,
+              "max_len": 512,
+              "tokenizer": {
+                "model_max_length": 4096,
+                "model_path": "./checkpoints/Cosmos3-Nano/text_tokenizer",
+                "padding_side": "right",
+                "trust_remote_code": true,
+                "type": "PretrainedTokenizer"
+              },
+              "type": "ProcessCosmos3Prompt"
+            },
+            {
+              "height": 256,
+              "type": "ResizeImages",
+              "width": 256
+            },
+            {
+              "type": "SimpleNormalizeImages"
+            },
+            {
+              "action_dim": 64,
+              "action_key": "action",
+              "action_norm_type": "quantile",
+              "state_dim": 64,
+              "state_key": "proprio",
+              "state_norm_type": "none",
+              "type": "NormalizeStatesAndActions"
+            },
+            {
+              "conditioning_fps": 20.0,
+              "frame_window_size": 17,
+              "mode": "wam",
+              "prepend_state_to_action": false,
+              "raw_action_dim": 10,
+              "type": "BuildCosmos3Sequence"
+            },
+            {
+              "frame_window_size": 17,
+              "num_views": 2,
+              "tile_direction": "horizontal",
+              "type": "PrepareVideo"
+            },
+            {
+              "height": 192,
+              "type": "ResizeAndReflectPad",
+              "width": 320
+            }
+          ],
+          "type": "ParquetDataset",
+          "use_delta": false,
+          "window_start_idx": 0
+        },
+        {
+          "action_key": "action",
+          "action_window_size": 16,
+          "data_root_path": "./datasets/libero_object_no_noops_lerobotv2.1",
+          "frame_window_size": 17,
+          "require_full_window": true,
+          "statistic_name": "all_libero_no_noops",
+          "transforms": [
+            {
+              "embodiment_id": 5,
+              "name_mappings": {
+                "actions": [
+                  "actions"
+                ],
+                "observation.state": [
+                  "states"
+                ]
+              },
+              "parquet_keys": [
+                "observation.state",
+                "timestamp",
+                "actions",
+                "info",
+                "stats",
+                "action_masks"
+              ],
+              "type": "ProcessParquetInputs",
+              "video_keys": [
+                "observation.images.image",
+                "observation.images.wrist_image"
+              ]
+            },
+            {
+              "type": "LiberoFramewiseActionToRot6D"
+            },
+            {
+              "action_metadata": {
+                "append_viewpoint": false,
+                "conditioning_fps": 20.0,
+                "frame_window_size": 17,
+                "video_height": 192,
+                "video_width": 320
+              },
+              "cfg_dropout_rate": 0.1,
+              "format_prompt_as_json": true,
+              "max_len": 512,
+              "tokenizer": {
+                "model_max_length": 4096,
+                "model_path": "./checkpoints/Cosmos3-Nano/text_tokenizer",
+                "padding_side": "right",
+                "trust_remote_code": true,
+                "type": "PretrainedTokenizer"
+              },
+              "type": "ProcessCosmos3Prompt"
+            },
+            {
+              "height": 256,
+              "type": "ResizeImages",
+              "width": 256
+            },
+            {
+              "type": "SimpleNormalizeImages"
+            },
+            {
+              "action_dim": 64,
+              "action_key": "action",
+              "action_norm_type": "quantile",
+              "state_dim": 64,
+              "state_key": "proprio",
+              "state_norm_type": "none",
+              "type": "NormalizeStatesAndActions"
+            },
+            {
+              "conditioning_fps": 20.0,
+              "frame_window_size": 17,
+              "mode": "wam",
+              "prepend_state_to_action": false,
+              "raw_action_dim": 10,
+              "type": "BuildCosmos3Sequence"
+            },
+            {
+              "frame_window_size": 17,
+              "num_views": 2,
+              "tile_direction": "horizontal",
+              "type": "PrepareVideo"
+            },
+            {
+              "height": 192,
+              "type": "ResizeAndReflectPad",
+              "width": 320
+            }
+          ],
+          "type": "ParquetDataset",
+          "use_delta": false,
+          "window_start_idx": 0
+        },
+        {
+          "action_key": "action",
+          "action_window_size": 16,
+          "data_root_path": "./datasets/libero_object_no_noops_lerobotv2.1",
+          "frame_window_size": 17,
+          "require_full_window": true,
+          "statistic_name": "all_libero_no_noops",
+          "transforms": [
+            {
+              "embodiment_id": 5,
+              "name_mappings": {
+                "actions": [
+                  "actions"
+                ],
+                "observation.state": [
+                  "states"
+                ]
+              },
+              "parquet_keys": [
+                "observation.state",
+                "timestamp",
+                "actions",
+                "info",
+                "stats",
+                "action_masks"
+              ],
+              "type": "ProcessParquetInputs",
+              "video_keys": [
+                "observation.images.image",
+                "observation.images.wrist_image"
+              ]
+            },
+            {
+              "type": "LiberoFramewiseActionToRot6D"
+            },
+            {
+              "action_metadata": {
+                "append_viewpoint": false,
+                "conditioning_fps": 20.0,
+                "frame_window_size": 17,
+                "video_height": 192,
+                "video_width": 320
+              },
+              "cfg_dropout_rate": 0.1,
+              "format_prompt_as_json": true,
+              "max_len": 512,
+              "tokenizer": {
+                "model_max_length": 4096,
+                "model_path": "./checkpoints/Cosmos3-Nano/text_tokenizer",
+                "padding_side": "right",
+                "trust_remote_code": true,
+                "type": "PretrainedTokenizer"
+              },
+              "type": "ProcessCosmos3Prompt"
+            },
+            {
+              "height": 256,
+              "type": "ResizeImages",
+              "width": 256
+            },
+            {
+              "type": "SimpleNormalizeImages"
+            },
+            {
+              "action_dim": 64,
+              "action_key": "action",
+              "action_norm_type": "quantile",
+              "state_dim": 64,
+              "state_key": "proprio",
+              "state_norm_type": "none",
+              "type": "NormalizeStatesAndActions"
+            },
+            {
+              "conditioning_fps": 20.0,
+              "frame_window_size": 17,
+              "mode": "wam",
+              "prepend_state_to_action": false,
+              "raw_action_dim": 10,
+              "type": "BuildCosmos3Sequence"
+            },
+            {
+              "frame_window_size": 17,
+              "num_views": 2,
+              "tile_direction": "horizontal",
+              "type": "PrepareVideo"
+            },
+            {
+              "height": 192,
+              "type": "ResizeAndReflectPad",
+              "width": 320
+            }
+          ],
+          "type": "ParquetDataset",
+          "use_delta": false,
+          "window_start_idx": 0
+        },
+        {
+          "action_key": "action",
+          "action_window_size": 16,
+          "data_root_path": "./datasets/libero_goal_no_noops_lerobotv2.1",
+          "frame_window_size": 17,
+          "require_full_window": true,
+          "statistic_name": "all_libero_no_noops",
+          "transforms": [
+            {
+              "embodiment_id": 5,
+              "name_mappings": {
+                "actions": [
+                  "actions"
+                ],
+                "observation.state": [
+                  "states"
+                ]
+              },
+              "parquet_keys": [
+                "observation.state",
+                "timestamp",
+                "actions",
+                "info",
+                "stats",
+                "action_masks"
+              ],
+              "type": "ProcessParquetInputs",
+              "video_keys": [
+                "observation.images.image",
+                "observation.images.wrist_image"
+              ]
+            },
+            {
+              "type": "LiberoFramewiseActionToRot6D"
+            },
+            {
+              "action_metadata": {
+                "append_viewpoint": false,
+                "conditioning_fps": 20.0,
+                "frame_window_size": 17,
+                "video_height": 192,
+                "video_width": 320
+              },
+              "cfg_dropout_rate": 0.1,
+              "format_prompt_as_json": true,
+              "max_len": 512,
+              "tokenizer": {
+                "model_max_length": 4096,
+                "model_path": "./checkpoints/Cosmos3-Nano/text_tokenizer",
+                "padding_side": "right",
+                "trust_remote_code": true,
+                "type": "PretrainedTokenizer"
+              },
+              "type": "ProcessCosmos3Prompt"
+            },
+            {
+              "height": 256,
+              "type": "ResizeImages",
+              "width": 256
+            },
+            {
+              "type": "SimpleNormalizeImages"
+            },
+            {
+              "action_dim": 64,
+              "action_key": "action",
+              "action_norm_type": "quantile",
+              "state_dim": 64,
+              "state_key": "proprio",
+              "state_norm_type": "none",
+              "type": "NormalizeStatesAndActions"
+            },
+            {
+              "conditioning_fps": 20.0,
+              "frame_window_size": 17,
+              "mode": "wam",
+              "prepend_state_to_action": false,
+              "raw_action_dim": 10,
+              "type": "BuildCosmos3Sequence"
+            },
+            {
+              "frame_window_size": 17,
+              "num_views": 2,
+              "tile_direction": "horizontal",
+              "type": "PrepareVideo"
+            },
+            {
+              "height": 192,
+              "type": "ResizeAndReflectPad",
+              "width": 320
+            }
+          ],
+          "type": "ParquetDataset",
+          "use_delta": false,
+          "window_start_idx": 0
+        },
+        {
+          "action_key": "action",
+          "action_window_size": 16,
+          "data_root_path": "./datasets/libero_goal_no_noops_lerobotv2.1",
+          "frame_window_size": 17,
+          "require_full_window": true,
+          "statistic_name": "all_libero_no_noops",
+          "transforms": [
+            {
+              "embodiment_id": 5,
+              "name_mappings": {
+                "actions": [
+                  "actions"
+                ],
+                "observation.state": [
+                  "states"
+                ]
+              },
+              "parquet_keys": [
+                "observation.state",
+                "timestamp",
+                "actions",
+                "info",
+                "stats",
+                "action_masks"
+              ],
+              "type": "ProcessParquetInputs",
+              "video_keys": [
+                "observation.images.image",
+                "observation.images.wrist_image"
+              ]
+            },
+            {
+              "type": "LiberoFramewiseActionToRot6D"
+            },
+            {
+              "action_metadata": {
+                "append_viewpoint": false,
+                "conditioning_fps": 20.0,
+                "frame_window_size": 17,
+                "video_height": 192,
+                "video_width": 320
+              },
+              "cfg_dropout_rate": 0.1,
+              "format_prompt_as_json": true,
+              "max_len": 512,
+              "tokenizer": {
+                "model_max_length": 4096,
+                "model_path": "./checkpoints/Cosmos3-Nano/text_tokenizer",
+                "padding_side": "right",
+                "trust_remote_code": true,
+                "type": "PretrainedTokenizer"
+              },
+              "type": "ProcessCosmos3Prompt"
+            },
+            {
+              "height": 256,
+              "type": "ResizeImages",
+              "width": 256
+            },
+            {
+              "type": "SimpleNormalizeImages"
+            },
+            {
+              "action_dim": 64,
+              "action_key": "action",
+              "action_norm_type": "quantile",
+              "state_dim": 64,
+              "state_key": "proprio",
+              "state_norm_type": "none",
+              "type": "NormalizeStatesAndActions"
+            },
+            {
+              "conditioning_fps": 20.0,
+              "frame_window_size": 17,
+              "mode": "wam",
+              "prepend_state_to_action": false,
+              "raw_action_dim": 10,
+              "type": "BuildCosmos3Sequence"
+            },
+            {
+              "frame_window_size": 17,
+              "num_views": 2,
+              "tile_direction": "horizontal",
+              "type": "PrepareVideo"
+            },
+            {
+              "height": 192,
+              "type": "ResizeAndReflectPad",
+              "width": 320
+            }
+          ],
+          "type": "ParquetDataset",
+          "use_delta": false,
+          "window_start_idx": 0
+        },
+        {
+          "action_key": "action",
+          "action_window_size": 16,
+          "data_root_path": "./datasets/libero_10_no_noops_lerobotv2.1",
+          "frame_window_size": 17,
+          "require_full_window": true,
+          "statistic_name": "all_libero_no_noops",
+          "transforms": [
+            {
+              "embodiment_id": 5,
+              "name_mappings": {
+                "actions": [
+                  "actions"
+                ],
+                "observation.state": [
+                  "states"
+                ]
+              },
+              "parquet_keys": [
+                "observation.state",
+                "timestamp",
+                "actions",
+                "info",
+                "stats",
+                "action_masks"
+              ],
+              "type": "ProcessParquetInputs",
+              "video_keys": [
+                "observation.images.image",
+                "observation.images.wrist_image"
+              ]
+            },
+            {
+              "type": "LiberoFramewiseActionToRot6D"
+            },
+            {
+              "action_metadata": {
+                "append_viewpoint": false,
+                "conditioning_fps": 20.0,
+                "frame_window_size": 17,
+                "video_height": 192,
+                "video_width": 320
+              },
+              "cfg_dropout_rate": 0.1,
+              "format_prompt_as_json": true,
+              "max_len": 512,
+              "tokenizer": {
+                "model_max_length": 4096,
+                "model_path": "./checkpoints/Cosmos3-Nano/text_tokenizer",
+                "padding_side": "right",
+                "trust_remote_code": true,
+                "type": "PretrainedTokenizer"
+              },
+              "type": "ProcessCosmos3Prompt"
+            },
+            {
+              "height": 256,
+              "type": "ResizeImages",
+              "width": 256
+            },
+            {
+              "type": "SimpleNormalizeImages"
+            },
+            {
+              "action_dim": 64,
+              "action_key": "action",
+              "action_norm_type": "quantile",
+              "state_dim": 64,
+              "state_key": "proprio",
+              "state_norm_type": "none",
+              "type": "NormalizeStatesAndActions"
+            },
+            {
+              "conditioning_fps": 20.0,
+              "frame_window_size": 17,
+              "mode": "wam",
+              "prepend_state_to_action": false,
+              "raw_action_dim": 10,
+              "type": "BuildCosmos3Sequence"
+            },
+            {
+              "frame_window_size": 17,
+              "num_views": 2,
+              "tile_direction": "horizontal",
+              "type": "PrepareVideo"
+            },
+            {
+              "height": 192,
+              "type": "ResizeAndReflectPad",
+              "width": 320
+            }
+          ],
+          "type": "ParquetDataset",
+          "use_delta": false,
+          "window_start_idx": 0
+        }
+      ],
+      "name_mappings": {
+        "action": [
+          "action"
+        ],
+        "observation.state": [
+          "proprio"
+        ]
+      },
+      "statistic_keys": [
+        "observation.state",
+        "timestamp",
+        "action"
+      ],
+      "statistic_name": "all_libero_no_noops",
+      "statistics_overrides": {
+        "all_libero_no_noops": {
+          "action": {
+            "max": [
+              0.9375,
+              0.9375,
+              0.9375,
+              1.0,
+              0.368853,
+              0.341214,
+              0.356395,
+              1.0,
+              0.348251,
+              1.0
+            ],
+            "mean": [
+              0.050704,
+              0.097407,
+              -0.094833,
+              0.994873,
+              -0.004579,
+              -0.004288,
+              0.004389,
+              0.996104,
+              0.001109,
+              0.476725
+            ],
+            "min": [
+              -0.9375,
+              -0.9375,
+              -0.9375,
+              0.902028,
+              -0.356085,
+              -0.367416,
+              -0.370434,
+              0.921907,
+              -0.255,
+              0.0
+            ],
+            "q01": [
+              -0.723214,
+              -0.808929,
+              -0.9375,
+              0.934955,
+              -0.223431,
+              -0.189878,
+              -0.334735,
+              0.938516,
+              -0.107736,
+              0.0
+            ],
+            "q99": [
+              0.9375,
+              0.870536,
+              0.9375,
+              1.0,
+              0.331,
+              0.163153,
+              0.226216,
+              1.0,
+              0.127158,
+              1.0
+            ],
+            "std": [
+              0.333621,
+              0.387175,
+              0.45714,
+              0.010807,
+              0.077802,
+              0.063386,
+              0.078571,
+              0.009994,
+              0.038504,
+              0.49946
+            ]
+          }
+        }
+      },
+      "type": "DistributedRepeatingDataset"
+    },
+    "per_device_batch_size": 16,
+    "per_device_num_workers": 4
+  }
+}

cosmos3_nano_libero_full_finetune_bs2048/config.yaml

@@ -0,0 +1,1392 @@
+_action_dim: 10
+_action_horizon: 16
+_action_lr: 0.00025
+_base_lr: 5.0e-05
+_cfg_dropout_rate: 0.1
+_ckpt_root: ./checkpoints
+_conditioning_fps: 20.0
+_cosmos3_nano_ckpt: ./checkpoints/Cosmos3-Nano
+_cosmos3_nano_name_mapping:
+  .self_attn.k_norm.: .self_attn.norm_k.
+  .self_attn.k_norm_moe_gen.: .self_attn.norm_added_k.
+  .self_attn.k_proj.: .self_attn.to_k.
+  .self_attn.k_proj_moe_gen.: .self_attn.add_k_proj.
+  .self_attn.o_proj.: .self_attn.to_out.
+  .self_attn.o_proj_moe_gen.: .self_attn.to_add_out.
+  .self_attn.q_norm.: .self_attn.norm_q.
+  .self_attn.q_norm_moe_gen.: .self_attn.norm_added_q.
+  .self_attn.q_proj.: .self_attn.to_q.
+  .self_attn.q_proj_moe_gen.: .self_attn.add_q_proj.
+  .self_attn.v_proj.: .self_attn.to_v.
+  .self_attn.v_proj_moe_gen.: .self_attn.add_v_proj.
+  action_in_proj.: action_proj_in.
+  action_modality_embed: action_modality_embed
+  action_out_proj.: action_proj_out.
+  time_embedder.mlp.0.: time_embedder.linear_1.
+  time_embedder.mlp.2.: time_embedder.linear_2.
+  vision_in_proj.projector.: proj_in.
+  vision_out_proj.projector.: proj_out.
+  vlm_backbone.lm_head.weight: lm_head.weight
+  vlm_backbone.model.language_model.embed_tokens.weight: embed_tokens.weight
+  vlm_backbone.model.language_model.layers.: layers.
+  vlm_backbone.model.language_model.norm.weight: norm.weight
+  vlm_backbone.model.language_model.norm_moe_gen.weight: norm_moe_gen.weight
+  vlm_backbone.model.visual.blocks.: blocks.
+  vlm_backbone.model.visual.deepstack_merger_list.: deepstack_merger_list.
+  vlm_backbone.model.visual.merger.: merger.
+  vlm_backbone.model.visual.patch_embed.: patch_embed.
+  vlm_backbone.model.visual.pos_embed.: pos_embed.
+_cosmos3_nano_special_tokens:
+  end_of_generation: 151653
+  eos_token_id: 151645
+  start_of_generation: 151652
+_cosmos3_nano_tokenizer:
+  model_max_length: 4096
+  model_path: ./checkpoints/Cosmos3-Nano/text_tokenizer
+  padding_side: right
+  trust_remote_code: true
+  type: PretrainedTokenizer
+_cosmos3_nano_transformer: ./checkpoints/Cosmos3-Nano/transformer
+_cosmos3_nano_vision_encoder: ./checkpoints/Cosmos3-Nano/vision_encoder
+_cosmos3_nano_vlm_config:
+  image_token_id: 151655
+  model_type: qwen3_vl
+  text_config:
+    attention_bias: false
+    attention_dropout: 0.0
+    bos_token_id: 151643
+    eos_token_id: 151645
+    head_dim: 128
+    hidden_act: silu
+    hidden_size: 4096
+    intermediate_size: 12288
+    layer_types:
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    - full_attention
+    max_position_embeddings: 262144
+    model_type: qwen3_vl_text
+    num_attention_heads: 32
+    num_hidden_layers: 36
+    num_key_value_heads: 8
+    pad_token_id: 0
+    rms_norm_eps: 1.0e-06
+    rope_scaling:
+      mrope_interleaved: true
+      mrope_section:
+      - 24
+      - 20
+      - 20
+      rope_type: default
+    rope_theta: 5000000
+    tie_word_embeddings: false
+    vocab_size: 151936
+  tie_word_embeddings: false
+  video_token_id: 151656
+  vision_config:
+    deepstack_visual_indexes:
+    - 8
+    - 16
+    - 24
+    depth: 27
+    hidden_act: gelu_pytorch_tanh
+    hidden_size: 1152
+    in_channels: 3
+    initializer_range: 0.02
+    intermediate_size: 4304
+    model_type: qwen3_vl
+    num_heads: 16
+    num_position_embeddings: 2304
+    out_hidden_size: 4096
+    patch_size: 16
+    spatial_merge_size: 2
+    temporal_patch_size: 2
+  vision_end_token_id: 151653
+  vision_start_token_id: 151652
+  vocab_size: 151936
+_data_root_path:
+- ./datasets/libero_spatial_no_noops_lerobotv2.1
+- ./datasets/libero_spatial_no_noops_lerobotv2.1
+- ./datasets/libero_object_no_noops_lerobotv2.1
+- ./datasets/libero_object_no_noops_lerobotv2.1
+- ./datasets/libero_goal_no_noops_lerobotv2.1
+- ./datasets/libero_goal_no_noops_lerobotv2.1
+- ./datasets/libero_10_no_noops_lerobotv2.1
+_frame_window_size: 17
+_grad_accumulation_steps: 8
+_image_height: 256
+_image_width: 256
+_libero_action_stats:
+  max:
+  - 0.9375
+  - 0.9375
+  - 0.9375
+  - 1.0
+  - 0.368853
+  - 0.341214
+  - 0.356395
+  - 1.0
+  - 0.348251
+  - 1.0
+  mean:
+  - 0.050704
+  - 0.097407
+  - -0.094833
+  - 0.994873
+  - -0.004579
+  - -0.004288
+  - 0.004389
+  - 0.996104
+  - 0.001109
+  - 0.476725
+  min:
+  - -0.9375
+  - -0.9375
+  - -0.9375
+  - 0.902028
+  - -0.356085
+  - -0.367416
+  - -0.370434
+  - 0.921907
+  - -0.255
+  - 0.0
+  q01:
+  - -0.723214
+  - -0.808929
+  - -0.9375
+  - 0.934955
+  - -0.223431
+  - -0.189878
+  - -0.334735
+  - 0.938516
+  - -0.107736
+  - 0.0
+  q99:
+  - 0.9375
+  - 0.870536
+  - 0.9375
+  - 1.0
+  - 0.331
+  - 0.163153
+  - 0.226216
+  - 1.0
+  - 0.127158
+  - 1.0
+  std:
+  - 0.333621
+  - 0.387175
+  - 0.45714
+  - 0.010807
+  - 0.077802
+  - 0.063386
+  - 0.078571
+  - 0.009994
+  - 0.038504
+  - 0.49946
+_max_action_dim: 64
+_max_state_dim: 64
+_max_steps: 5000
+_per_device_batch_size: 16
+_prepend_state_to_action: false
+_save_iter_interval: 500
+_statistic_name: all_libero_no_noops
+_transforms:
+- embodiment_id: 5
+  name_mappings:
+    actions:
+    - actions
+    observation.state:
+    - states
+  parquet_keys:
+  - observation.state
+  - timestamp
+  - actions
+  - info
+  - stats
+  - action_masks
+  type: ProcessParquetInputs
+  video_keys:
+  - observation.images.image
+  - observation.images.wrist_image
+- type: LiberoFramewiseActionToRot6D
+- action_metadata:
+    append_viewpoint: false
+    conditioning_fps: 20.0
+    frame_window_size: 17
+    video_height: 192
+    video_width: 320
+  cfg_dropout_rate: 0.1
+  format_prompt_as_json: true
+  max_len: 512
+  tokenizer:
+    model_max_length: 4096
+    model_path: ./checkpoints/Cosmos3-Nano/text_tokenizer
+    padding_side: right
+    trust_remote_code: true
+    type: PretrainedTokenizer
+  type: ProcessCosmos3Prompt
+- height: 256
+  type: ResizeImages
+  width: 256
+- type: SimpleNormalizeImages
+- action_dim: 64
+  action_key: action
+  action_norm_type: quantile
+  state_dim: 64
+  state_key: proprio
+  state_norm_type: none
+  type: NormalizeStatesAndActions
+- conditioning_fps: 20.0
+  frame_window_size: 17
+  mode: wam
+  prepend_state_to_action: false
+  raw_action_dim: 10
+  type: BuildCosmos3Sequence
+- frame_window_size: 17
+  num_views: 2
+  tile_direction: horizontal
+  type: PrepareVideo
+- height: 192
+  type: ResizeAndReflectPad
+  width: 320
+_video_height: 192
+_video_width: 320
+_vision_vae:
+  encode_exact_durations:
+  - 17
+  pretrained_name_or_path: ./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
+  type: Cosmos3Wan22VAE
+_wan22_vae_path: ./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
+eval:
+  dataset:
+    extra_tensor_keys:
+    - conditioning_fps
+    - prepend_state_to_action
+    img_buffer_len: 1
+    transforms:
+    - embodiment_id: 5
+      img_keys:
+      - agentview_image
+      - robot0_eye_in_hand_image
+      resize_size: 256
+      type: ProcessLiberoEvalInputs
+    - conditioning_fps: 20.0
+      prepend_state_to_action: false
+      type: SetCosmos3ActionMetadata
+    - image_resize_strategy: resize-crop
+      input_sizes:
+      - - 3
+        - 256
+        - 256
+      - - 3
+        - 256
+        - 256
+      means:
+      - - 127.5
+        - 127.5
+        - 127.5
+      - - 127.5
+        - 127.5
+        - 127.5
+      stds:
+      - - 127.5
+        - 127.5
+        - 127.5
+      - - 127.5
+        - 127.5
+        - 127.5
+      type: TransformImage
+    - action_metadata:
+        append_viewpoint: false
+        conditioning_fps: 20.0
+        frame_window_size: 17
+        video_height: 192
+        video_width: 320
+      cfg_dropout_rate: 0.0
+      format_prompt_as_json: true
+      max_len: 512
+      output_attention_mask_key: lang_masks
+      output_key: lang_tokens
+      tokenizer:
+        model_max_length: 4096
+        model_path: ./checkpoints/Cosmos3-Nano/text_tokenizer
+        padding_side: right
+        trust_remote_code: true
+        type: PretrainedTokenizer
+      type: ProcessCosmos3Prompt
+    - frame_window_size: 1
+      num_views: 2
+      tile_direction: horizontal
+      type: PrepareVideo
+    - height: 192
+      type: ResizeAndReflectPad
+      width: 320
+    type: LiberoParquetEvalDataset
+  denormalize_action:
+    action_dim: 10
+    norm_type: quantile_rot
+    type: DenormalizeLiberoFramewiseRot6DAction
+  enable_mixed_precision_training: true
+  eval_chunk_size: 16
+  mixed_precision_dtype: bf16
+  model_family: cosmos3
+  norm_stats_key: all_libero_no_noops
+  num_steps_wait: 10
+  num_trials_per_task: 50
+  seed: 7
+  task_suite_name: libero_10
+  type: LiberoEvalRunner
+inference_model:
+  action_horizon: 16
+  action_in_proj:
+    input_size: 64
+    num_domains: 32
+    output_size: 4096
+    type: DomainAwareLinear
+  action_out_proj:
+    input_size: 4096
+    num_domains: 32
+    output_size: 64
+    type: DomainAwareLinear
+  base_fps: 24.0
+  enable_fps_modulation: true
+  enable_vision_loss: true
+  freeze_non_moe_vlm_backbone: true
+  freeze_vlm_backbone: false
+  latent_patch_size: 2
+  max_action_dim: 64
+  name_mapping:
+    .self_attn.k_norm.: .self_attn.norm_k.
+    .self_attn.k_norm_moe_gen.: .self_attn.norm_added_k.
+    .self_attn.k_proj.: .self_attn.to_k.
+    .self_attn.k_proj_moe_gen.: .self_attn.add_k_proj.
+    .self_attn.o_proj.: .self_attn.to_out.
+    .self_attn.o_proj_moe_gen.: .self_attn.to_add_out.
+    .self_attn.q_norm.: .self_attn.norm_q.
+    .self_attn.q_norm_moe_gen.: .self_attn.norm_added_q.
+    .self_attn.q_proj.: .self_attn.to_q.
+    .self_attn.q_proj_moe_gen.: .self_attn.add_q_proj.
+    .self_attn.v_proj.: .self_attn.to_v.
+    .self_attn.v_proj_moe_gen.: .self_attn.add_v_proj.
+    action_in_proj.: action_proj_in.
+    action_modality_embed: action_modality_embed
+    action_out_proj.: action_proj_out.
+    time_embedder.mlp.0.: time_embedder.linear_1.
+    time_embedder.mlp.2.: time_embedder.linear_2.
+    vision_in_proj.projector.: proj_in.
+    vision_out_proj.projector.: proj_out.
+    vlm_backbone.lm_head.weight: lm_head.weight
+    vlm_backbone.model.language_model.embed_tokens.weight: embed_tokens.weight
+    vlm_backbone.model.language_model.layers.: layers.
+    vlm_backbone.model.language_model.norm.weight: norm.weight
+    vlm_backbone.model.language_model.norm_moe_gen.weight: norm_moe_gen.weight
+    vlm_backbone.model.visual.blocks.: blocks.
+    vlm_backbone.model.visual.deepstack_merger_list.: deepstack_merger_list.
+    vlm_backbone.model.visual.merger.: merger.
+    vlm_backbone.model.visual.patch_embed.: patch_embed.
+    vlm_backbone.model.visual.pos_embed.: pos_embed.
+  num_embodiment_domains: 32
+  ori_action_dim: 10
+  packed_attention_backend: flash2
+  position_embedding_type: unified_3d_mrope
+  pretrained_name_or_path: ./checkpoints/Cosmos3-Nano/transformer
+  rectified_flow_inference_config:
+    num_steps: 30
+    num_train_timesteps: 1000
+    scheduler_type: unipc
+    shift: 10.0
+    use_dynamic_shifting: false
+    use_karras_sigmas: false
+  rectified_flow_training_config:
+    action_loss_weight: 10.0
+    high_sigma_ratio: 0.05
+    high_sigma_timesteps_max: 1000
+    high_sigma_timesteps_min: 995
+    independent_action_schedule: false
+    normalize_loss_by_active: false
+    shift:
+      '256': 3
+      '480': 5
+      '720': 10
+    shift_action: null
+    train_time_action_distribution: logitnormal
+    train_time_image_distribution: logitnormal
+    train_time_video_distribution: waver
+    train_time_weight: uniform
+    use_discrete_rf: false
+    use_dynamic_shift: false
+    use_high_sigma_strategy: false
+    use_high_sigma_strategy_action: false
+    vision_loss_weight: 10.0
+  special_tokens:
+    end_of_generation: 151653
+    eos_token_id: 151645
+    start_of_generation: 151652
+  timestep_scale: 0.001
+  type: Cosmos3FlowMatching
+  unified_3d_mrope_reset_spatial_ids: true
+  unified_3d_mrope_temporal_modality_margin: 15000
+  vision_in_proj:
+    in_dim: 192
+    out_dim: 4096
+    type: LinearProjector
+  vision_latent_dim: 48
+  vision_out_proj:
+    in_dim: 4096
+    out_dim: 192
+    type: LinearProjector
+  vision_vae:
+    encode_exact_durations:
+    - 17
+    pretrained_name_or_path: null
+    type: Cosmos3Wan22VAE
+  vlm_backbone:
+    include_visual: false
+    skip_init_weights: true
+    type: Cosmos3MoTBackbone
+    vision_encoder_path: ./checkpoints/Cosmos3-Nano/vision_encoder
+    vlm_config:
+      image_token_id: 151655
+      model_type: qwen3_vl
+      text_config:
+        attention_bias: false
+        attention_dropout: 0.0
+        bos_token_id: 151643
+        eos_token_id: 151645
+        head_dim: 128
+        hidden_act: silu
+        hidden_size: 4096
+        intermediate_size: 12288
+        layer_types:
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        max_position_embeddings: 262144
+        model_type: qwen3_vl_text
+        num_attention_heads: 32
+        num_hidden_layers: 36
+        num_key_value_heads: 8
+        pad_token_id: 0
+        rms_norm_eps: 1.0e-06
+        rope_scaling:
+          mrope_interleaved: true
+          mrope_section:
+          - 24
+          - 20
+          - 20
+          rope_type: default
+        rope_theta: 5000000
+        tie_word_embeddings: false
+        vocab_size: 151936
+      tie_word_embeddings: false
+      video_token_id: 151656
+      vision_config:
+        deepstack_visual_indexes:
+        - 8
+        - 16
+        - 24
+        depth: 27
+        hidden_act: gelu_pytorch_tanh
+        hidden_size: 1152
+        in_channels: 3
+        initializer_range: 0.02
+        intermediate_size: 4304
+        model_type: qwen3_vl
+        num_heads: 16
+        num_position_embeddings: 2304
+        out_hidden_size: 4096
+        patch_size: 16
+        spatial_merge_size: 2
+        temporal_patch_size: 2
+      vision_end_token_id: 151653
+      vision_start_token_id: 151652
+      vocab_size: 151936
+model:
+  action_horizon: 16
+  action_in_proj:
+    input_size: 64
+    num_domains: 32
+    output_size: 4096
+    type: DomainAwareLinear
+  action_out_proj:
+    input_size: 4096
+    num_domains: 32
+    output_size: 64
+    type: DomainAwareLinear
+  base_fps: 24.0
+  enable_fps_modulation: true
+  enable_vision_loss: true
+  freeze_non_moe_vlm_backbone: true
+  freeze_vlm_backbone: false
+  latent_patch_size: 2
+  max_action_dim: 64
+  name_mapping:
+    .self_attn.k_norm.: .self_attn.norm_k.
+    .self_attn.k_norm_moe_gen.: .self_attn.norm_added_k.
+    .self_attn.k_proj.: .self_attn.to_k.
+    .self_attn.k_proj_moe_gen.: .self_attn.add_k_proj.
+    .self_attn.o_proj.: .self_attn.to_out.
+    .self_attn.o_proj_moe_gen.: .self_attn.to_add_out.
+    .self_attn.q_norm.: .self_attn.norm_q.
+    .self_attn.q_norm_moe_gen.: .self_attn.norm_added_q.
+    .self_attn.q_proj.: .self_attn.to_q.
+    .self_attn.q_proj_moe_gen.: .self_attn.add_q_proj.
+    .self_attn.v_proj.: .self_attn.to_v.
+    .self_attn.v_proj_moe_gen.: .self_attn.add_v_proj.
+    action_in_proj.: action_proj_in.
+    action_modality_embed: action_modality_embed
+    action_out_proj.: action_proj_out.
+    time_embedder.mlp.0.: time_embedder.linear_1.
+    time_embedder.mlp.2.: time_embedder.linear_2.
+    vision_in_proj.projector.: proj_in.
+    vision_out_proj.projector.: proj_out.
+    vlm_backbone.lm_head.weight: lm_head.weight
+    vlm_backbone.model.language_model.embed_tokens.weight: embed_tokens.weight
+    vlm_backbone.model.language_model.layers.: layers.
+    vlm_backbone.model.language_model.norm.weight: norm.weight
+    vlm_backbone.model.language_model.norm_moe_gen.weight: norm_moe_gen.weight
+    vlm_backbone.model.visual.blocks.: blocks.
+    vlm_backbone.model.visual.deepstack_merger_list.: deepstack_merger_list.
+    vlm_backbone.model.visual.merger.: merger.
+    vlm_backbone.model.visual.patch_embed.: patch_embed.
+    vlm_backbone.model.visual.pos_embed.: pos_embed.
+  num_embodiment_domains: 32
+  ori_action_dim: 10
+  packed_attention_backend: flash2
+  position_embedding_type: unified_3d_mrope
+  pretrained_name_or_path: ./checkpoints/Cosmos3-Nano/transformer
+  rectified_flow_inference_config:
+    num_steps: 30
+    num_train_timesteps: 1000
+    scheduler_type: unipc
+    shift: 10.0
+    use_dynamic_shifting: false
+    use_karras_sigmas: false
+  rectified_flow_training_config:
+    action_loss_weight: 10.0
+    high_sigma_ratio: 0.05
+    high_sigma_timesteps_max: 1000
+    high_sigma_timesteps_min: 995
+    independent_action_schedule: false
+    normalize_loss_by_active: false
+    shift:
+      '256': 3
+      '480': 5
+      '720': 10
+    shift_action: null
+    train_time_action_distribution: logitnormal
+    train_time_image_distribution: logitnormal
+    train_time_video_distribution: waver
+    train_time_weight: uniform
+    use_discrete_rf: false
+    use_dynamic_shift: false
+    use_high_sigma_strategy: false
+    use_high_sigma_strategy_action: false
+    vision_loss_weight: 10.0
+  special_tokens:
+    end_of_generation: 151653
+    eos_token_id: 151645
+    start_of_generation: 151652
+  timestep_scale: 0.001
+  type: Cosmos3FlowMatching
+  unified_3d_mrope_reset_spatial_ids: true
+  unified_3d_mrope_temporal_modality_margin: 15000
+  vision_in_proj:
+    in_dim: 192
+    out_dim: 4096
+    type: LinearProjector
+  vision_latent_dim: 48
+  vision_out_proj:
+    in_dim: 4096
+    out_dim: 192
+    type: LinearProjector
+  vision_vae:
+    encode_exact_durations:
+    - 17
+    pretrained_name_or_path: ./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
+    type: Cosmos3Wan22VAE
+  vlm_backbone:
+    include_visual: false
+    skip_init_weights: true
+    type: Cosmos3MoTBackbone
+    vision_encoder_path: ./checkpoints/Cosmos3-Nano/vision_encoder
+    vlm_config:
+      image_token_id: 151655
+      model_type: qwen3_vl
+      text_config:
+        attention_bias: false
+        attention_dropout: 0.0
+        bos_token_id: 151643
+        eos_token_id: 151645
+        head_dim: 128
+        hidden_act: silu
+        hidden_size: 4096
+        intermediate_size: 12288
+        layer_types:
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        - full_attention
+        max_position_embeddings: 262144
+        model_type: qwen3_vl_text
+        num_attention_heads: 32
+        num_hidden_layers: 36
+        num_key_value_heads: 8
+        pad_token_id: 0
+        rms_norm_eps: 1.0e-06
+        rope_scaling:
+          mrope_interleaved: true
+          mrope_section:
+          - 24
+          - 20
+          - 20
+          rope_type: default
+        rope_theta: 5000000
+        tie_word_embeddings: false
+        vocab_size: 151936
+      tie_word_embeddings: false
+      video_token_id: 151656
+      vision_config:
+        deepstack_visual_indexes:
+        - 8
+        - 16
+        - 24
+        depth: 27
+        hidden_act: gelu_pytorch_tanh
+        hidden_size: 1152
+        in_channels: 3
+        initializer_range: 0.02
+        intermediate_size: 4304
+        model_type: qwen3_vl
+        num_heads: 16
+        num_position_embeddings: 2304
+        out_hidden_size: 4096
+        patch_size: 16
+        spatial_merge_size: 2
+        temporal_patch_size: 2
+      vision_end_token_id: 151653
+      vision_start_token_id: 151652
+      vocab_size: 151936
+runner:
+  change_key_name: false
+  collator:
+    list_keys:
+    - sequence_plan
+    meta_keys:
+    - task_description
+    - stats
+    - info
+    - timestamp
+    pad_id: 0
+    sequence_keys:
+    - text_token_ids
+    tensor_keys:
+    - images
+    - actions
+    - embodiment_ids
+    - raw_action_dim
+    - conditioning_fps
+    - action_fps
+    type: Cosmos3Collator
+  enable_gradient_checkpointing: true
+  enable_mixed_precision_training: true
+  grad_accumulation_steps: 8
+  lr_scheduler:
+    cycle_length: 16000
+    type: linear-warmup+linear-decay
+    warmup_steps: 500
+  max_grad_norm: 1.0
+  max_keep_ckpts: 2
+  max_steps: 5000
+  metric:
+    active_trackers:
+    - jsonl
+    - wandb
+    grad_accumulation_steps: 8
+    run_dir: work_dirs
+    type: VLAMetric
+    window_size: 1
+  mixed_precision_dtype: bf16
+  optimizer:
+    betas:
+    - 0.9
+    - 0.99
+    eps: 1.0e-08
+    exclude_1d_from_weight_decay: false
+    fused: true
+    lr: 5.0e-05
+    paramwise_learning_rate:
+      action_in_proj.: 0.00025
+      action_modality_embed: 0.00025
+      action_out_proj.: 0.00025
+    type: AdamW
+    weight_decay: 0.05
+  sampler: null
+  save_iter_interval: 500
+  sharding_strategy: full-shard
+  tokenizer:
+    model_max_length: 4096
+    model_path: ./checkpoints/Cosmos3-Nano/text_tokenizer
+    padding_side: right
+    trust_remote_code: true
+    type: PretrainedTokenizer
+  type: FSDPTrainRunner
+train_dataloader:
+  dataset:
+    datasets:
+    - action_key: action
+      action_window_size: 16
+      data_root_path: ./datasets/libero_spatial_no_noops_lerobotv2.1
+      frame_window_size: 17
+      require_full_window: true
+      statistic_name: all_libero_no_noops
+      transforms:
+      - embodiment_id: 5
+        name_mappings:
+          actions:
+          - actions
+          observation.state:
+          - states
+        parquet_keys:
+        - observation.state
+        - timestamp
+        - actions
+        - info
+        - stats
+        - action_masks
+        type: ProcessParquetInputs
+        video_keys:
+        - observation.images.image
+        - observation.images.wrist_image
+      - type: LiberoFramewiseActionToRot6D
+      - action_metadata:
+          append_viewpoint: false
+          conditioning_fps: 20.0
+          frame_window_size: 17
+          video_height: 192
+          video_width: 320
+        cfg_dropout_rate: 0.1
+        format_prompt_as_json: true
+        max_len: 512
+        tokenizer:
+          model_max_length: 4096
+          model_path: ./checkpoints/Cosmos3-Nano/text_tokenizer
+          padding_side: right
+          trust_remote_code: true
+          type: PretrainedTokenizer
+        type: ProcessCosmos3Prompt
+      - height: 256
+        type: ResizeImages
+        width: 256
+      - type: SimpleNormalizeImages
+      - action_dim: 64
+        action_key: action
+        action_norm_type: quantile
+        state_dim: 64
+        state_key: proprio
+        state_norm_type: none
+        type: NormalizeStatesAndActions
+      - conditioning_fps: 20.0
+        frame_window_size: 17
+        mode: wam
+        prepend_state_to_action: false
+        raw_action_dim: 10
+        type: BuildCosmos3Sequence
+      - frame_window_size: 17
+        num_views: 2
+        tile_direction: horizontal
+        type: PrepareVideo
+      - height: 192
+        type: ResizeAndReflectPad
+        width: 320
+      type: ParquetDataset
+      use_delta: false
+      window_start_idx: 0
+    - action_key: action
+      action_window_size: 16
+      data_root_path: ./datasets/libero_spatial_no_noops_lerobotv2.1
+      frame_window_size: 17
+      require_full_window: true
+      statistic_name: all_libero_no_noops
+      transforms:
+      - embodiment_id: 5
+        name_mappings:
+          actions:
+          - actions
+          observation.state:
+          - states
+        parquet_keys:
+        - observation.state
+        - timestamp
+        - actions
+        - info
+        - stats
+        - action_masks
+        type: ProcessParquetInputs
+        video_keys:
+        - observation.images.image
+        - observation.images.wrist_image
+      - type: LiberoFramewiseActionToRot6D
+      - action_metadata:
+          append_viewpoint: false
+          conditioning_fps: 20.0
+          frame_window_size: 17
+          video_height: 192
+          video_width: 320
+        cfg_dropout_rate: 0.1
+        format_prompt_as_json: true
+        max_len: 512
+        tokenizer:
+          model_max_length: 4096
+          model_path: ./checkpoints/Cosmos3-Nano/text_tokenizer
+          padding_side: right
+          trust_remote_code: true
+          type: PretrainedTokenizer
+        type: ProcessCosmos3Prompt
+      - height: 256
+        type: ResizeImages
+        width: 256
+      - type: SimpleNormalizeImages
+      - action_dim: 64
+        action_key: action
+        action_norm_type: quantile
+        state_dim: 64
+        state_key: proprio
+        state_norm_type: none
+        type: NormalizeStatesAndActions
+      - conditioning_fps: 20.0
+        frame_window_size: 17
+        mode: wam
+        prepend_state_to_action: false
+        raw_action_dim: 10
+        type: BuildCosmos3Sequence
+      - frame_window_size: 17
+        num_views: 2
+        tile_direction: horizontal
+        type: PrepareVideo
+      - height: 192
+        type: ResizeAndReflectPad
+        width: 320
+      type: ParquetDataset
+      use_delta: false
+      window_start_idx: 0
+    - action_key: action
+      action_window_size: 16
+      data_root_path: ./datasets/libero_object_no_noops_lerobotv2.1
+      frame_window_size: 17
+      require_full_window: true
+      statistic_name: all_libero_no_noops
+      transforms:
+      - embodiment_id: 5
+        name_mappings:
+          actions:
+          - actions
+          observation.state:
+          - states
+        parquet_keys:
+        - observation.state
+        - timestamp
+        - actions
+        - info
+        - stats
+        - action_masks
+        type: ProcessParquetInputs
+        video_keys:
+        - observation.images.image
+        - observation.images.wrist_image
+      - type: LiberoFramewiseActionToRot6D
+      - action_metadata:
+          append_viewpoint: false
+          conditioning_fps: 20.0
+          frame_window_size: 17
+          video_height: 192
+          video_width: 320
+        cfg_dropout_rate: 0.1
+        format_prompt_as_json: true
+        max_len: 512
+        tokenizer:
+          model_max_length: 4096
+          model_path: ./checkpoints/Cosmos3-Nano/text_tokenizer
+          padding_side: right
+          trust_remote_code: true
+          type: PretrainedTokenizer
+        type: ProcessCosmos3Prompt
+      - height: 256
+        type: ResizeImages
+        width: 256
+      - type: SimpleNormalizeImages
+      - action_dim: 64
+        action_key: action
+        action_norm_type: quantile
+        state_dim: 64
+        state_key: proprio
+        state_norm_type: none
+        type: NormalizeStatesAndActions
+      - conditioning_fps: 20.0
+        frame_window_size: 17
+        mode: wam
+        prepend_state_to_action: false
+        raw_action_dim: 10
+        type: BuildCosmos3Sequence
+      - frame_window_size: 17
+        num_views: 2
+        tile_direction: horizontal
+        type: PrepareVideo
+      - height: 192
+        type: ResizeAndReflectPad
+        width: 320
+      type: ParquetDataset
+      use_delta: false
+      window_start_idx: 0
+    - action_key: action
+      action_window_size: 16
+      data_root_path: ./datasets/libero_object_no_noops_lerobotv2.1
+      frame_window_size: 17
+      require_full_window: true
+      statistic_name: all_libero_no_noops
+      transforms:
+      - embodiment_id: 5
+        name_mappings:
+          actions:
+          - actions
+          observation.state:
+          - states
+        parquet_keys:
+        - observation.state
+        - timestamp
+        - actions
+        - info
+        - stats
+        - action_masks
+        type: ProcessParquetInputs
+        video_keys:
+        - observation.images.image
+        - observation.images.wrist_image
+      - type: LiberoFramewiseActionToRot6D
+      - action_metadata:
+          append_viewpoint: false
+          conditioning_fps: 20.0
+          frame_window_size: 17
+          video_height: 192
+          video_width: 320
+        cfg_dropout_rate: 0.1
+        format_prompt_as_json: true
+        max_len: 512
+        tokenizer:
+          model_max_length: 4096
+          model_path: ./checkpoints/Cosmos3-Nano/text_tokenizer
+          padding_side: right
+          trust_remote_code: true
+          type: PretrainedTokenizer
+        type: ProcessCosmos3Prompt
+      - height: 256
+        type: ResizeImages
+        width: 256
+      - type: SimpleNormalizeImages
+      - action_dim: 64
+        action_key: action
+        action_norm_type: quantile
+        state_dim: 64
+        state_key: proprio
+        state_norm_type: none
+        type: NormalizeStatesAndActions
+      - conditioning_fps: 20.0
+        frame_window_size: 17
+        mode: wam
+        prepend_state_to_action: false
+        raw_action_dim: 10
+        type: BuildCosmos3Sequence
+      - frame_window_size: 17
+        num_views: 2
+        tile_direction: horizontal
+        type: PrepareVideo
+      - height: 192
+        type: ResizeAndReflectPad
+        width: 320
+      type: ParquetDataset
+      use_delta: false
+      window_start_idx: 0
+    - action_key: action
+      action_window_size: 16
+      data_root_path: ./datasets/libero_goal_no_noops_lerobotv2.1
+      frame_window_size: 17
+      require_full_window: true
+      statistic_name: all_libero_no_noops
+      transforms:
+      - embodiment_id: 5
+        name_mappings:
+          actions:
+          - actions
+          observation.state:
+          - states
+        parquet_keys:
+        - observation.state
+        - timestamp
+        - actions
+        - info
+        - stats
+        - action_masks
+        type: ProcessParquetInputs
+        video_keys:
+        - observation.images.image
+        - observation.images.wrist_image
+      - type: LiberoFramewiseActionToRot6D
+      - action_metadata:
+          append_viewpoint: false
+          conditioning_fps: 20.0
+          frame_window_size: 17
+          video_height: 192
+          video_width: 320
+        cfg_dropout_rate: 0.1
+        format_prompt_as_json: true
+        max_len: 512
+        tokenizer:
+          model_max_length: 4096
+          model_path: ./checkpoints/Cosmos3-Nano/text_tokenizer
+          padding_side: right
+          trust_remote_code: true
+          type: PretrainedTokenizer
+        type: ProcessCosmos3Prompt
+      - height: 256
+        type: ResizeImages
+        width: 256
+      - type: SimpleNormalizeImages
+      - action_dim: 64
+        action_key: action
+        action_norm_type: quantile
+        state_dim: 64
+        state_key: proprio
+        state_norm_type: none
+        type: NormalizeStatesAndActions
+      - conditioning_fps: 20.0
+        frame_window_size: 17
+        mode: wam
+        prepend_state_to_action: false
+        raw_action_dim: 10
+        type: BuildCosmos3Sequence
+      - frame_window_size: 17
+        num_views: 2
+        tile_direction: horizontal
+        type: PrepareVideo
+      - height: 192
+        type: ResizeAndReflectPad
+        width: 320
+      type: ParquetDataset
+      use_delta: false
+      window_start_idx: 0
+    - action_key: action
+      action_window_size: 16
+      data_root_path: ./datasets/libero_goal_no_noops_lerobotv2.1
+      frame_window_size: 17
+      require_full_window: true
+      statistic_name: all_libero_no_noops
+      transforms:
+      - embodiment_id: 5
+        name_mappings:
+          actions:
+          - actions
+          observation.state:
+          - states
+        parquet_keys:
+        - observation.state
+        - timestamp
+        - actions
+        - info
+        - stats
+        - action_masks
+        type: ProcessParquetInputs
+        video_keys:
+        - observation.images.image
+        - observation.images.wrist_image
+      - type: LiberoFramewiseActionToRot6D
+      - action_metadata:
+          append_viewpoint: false
+          conditioning_fps: 20.0
+          frame_window_size: 17
+          video_height: 192
+          video_width: 320
+        cfg_dropout_rate: 0.1
+        format_prompt_as_json: true
+        max_len: 512
+        tokenizer:
+          model_max_length: 4096
+          model_path: ./checkpoints/Cosmos3-Nano/text_tokenizer
+          padding_side: right
+          trust_remote_code: true
+          type: PretrainedTokenizer
+        type: ProcessCosmos3Prompt
+      - height: 256
+        type: ResizeImages
+        width: 256
+      - type: SimpleNormalizeImages
+      - action_dim: 64
+        action_key: action
+        action_norm_type: quantile
+        state_dim: 64
+        state_key: proprio
+        state_norm_type: none
+        type: NormalizeStatesAndActions
+      - conditioning_fps: 20.0
+        frame_window_size: 17
+        mode: wam
+        prepend_state_to_action: false
+        raw_action_dim: 10
+        type: BuildCosmos3Sequence
+      - frame_window_size: 17
+        num_views: 2
+        tile_direction: horizontal
+        type: PrepareVideo
+      - height: 192
+        type: ResizeAndReflectPad
+        width: 320
+      type: ParquetDataset
+      use_delta: false
+      window_start_idx: 0
+    - action_key: action
+      action_window_size: 16
+      data_root_path: ./datasets/libero_10_no_noops_lerobotv2.1
+      frame_window_size: 17
+      require_full_window: true
+      statistic_name: all_libero_no_noops
+      transforms:
+      - embodiment_id: 5
+        name_mappings:
+          actions:
+          - actions
+          observation.state:
+          - states
+        parquet_keys:
+        - observation.state
+        - timestamp
+        - actions
+        - info
+        - stats
+        - action_masks
+        type: ProcessParquetInputs
+        video_keys:
+        - observation.images.image
+        - observation.images.wrist_image
+      - type: LiberoFramewiseActionToRot6D
+      - action_metadata:
+          append_viewpoint: false
+          conditioning_fps: 20.0
+          frame_window_size: 17
+          video_height: 192
+          video_width: 320
+        cfg_dropout_rate: 0.1
+        format_prompt_as_json: true
+        max_len: 512
+        tokenizer:
+          model_max_length: 4096
+          model_path: ./checkpoints/Cosmos3-Nano/text_tokenizer
+          padding_side: right
+          trust_remote_code: true
+          type: PretrainedTokenizer
+        type: ProcessCosmos3Prompt
+      - height: 256
+        type: ResizeImages
+        width: 256
+      - type: SimpleNormalizeImages
+      - action_dim: 64
+        action_key: action
+        action_norm_type: quantile
+        state_dim: 64
+        state_key: proprio
+        state_norm_type: none
+        type: NormalizeStatesAndActions
+      - conditioning_fps: 20.0
+        frame_window_size: 17
+        mode: wam
+        prepend_state_to_action: false
+        raw_action_dim: 10
+        type: BuildCosmos3Sequence
+      - frame_window_size: 17
+        num_views: 2
+        tile_direction: horizontal
+        type: PrepareVideo
+      - height: 192
+        type: ResizeAndReflectPad
+        width: 320
+      type: ParquetDataset
+      use_delta: false
+      window_start_idx: 0
+    name_mappings:
+      action:
+      - action
+      observation.state:
+      - proprio
+    statistic_keys:
+    - observation.state
+    - timestamp
+    - action
+    statistic_name: all_libero_no_noops
+    statistics_overrides:
+      all_libero_no_noops:
+        action:
+          max:
+          - 0.9375
+          - 0.9375
+          - 0.9375
+          - 1.0
+          - 0.368853
+          - 0.341214
+          - 0.356395
+          - 1.0
+          - 0.348251
+          - 1.0
+          mean:
+          - 0.050704
+          - 0.097407
+          - -0.094833
+          - 0.994873
+          - -0.004579
+          - -0.004288
+          - 0.004389
+          - 0.996104
+          - 0.001109
+          - 0.476725
+          min:
+          - -0.9375
+          - -0.9375
+          - -0.9375
+          - 0.902028
+          - -0.356085
+          - -0.367416
+          - -0.370434
+          - 0.921907
+          - -0.255
+          - 0.0
+          q01:
+          - -0.723214
+          - -0.808929
+          - -0.9375
+          - 0.934955
+          - -0.223431
+          - -0.189878
+          - -0.334735
+          - 0.938516
+          - -0.107736
+          - 0.0
+          q99:
+          - 0.9375
+          - 0.870536
+          - 0.9375
+          - 1.0
+          - 0.331
+          - 0.163153
+          - 0.226216
+          - 1.0
+          - 0.127158
+          - 1.0
+          std:
+          - 0.333621
+          - 0.387175
+          - 0.45714
+          - 0.010807
+          - 0.077802
+          - 0.063386
+          - 0.078571
+          - 0.009994
+          - 0.038504
+          - 0.49946
+    type: DistributedRepeatingDataset
+  per_device_batch_size: 16
+  per_device_num_workers: 4

cosmos3_nano_libero_full_finetune_bs2048/dataset_statistics.json

@@ -0,0 +1,138 @@
+{
+  "all_libero_no_noops": {
+    "proprio": {
+      "mean": [
+        -0.047496414237850684,
+        0.03435983471608021,
+        0.7514258672095461,
+        2.983973905120257,
+        -0.1419395165888572,
+        -0.118218935315545,
+        0.026695325676034158,
+        -0.026933706126891162
+      ],
+      "std": [
+        0.10441071071075952,
+        0.15324089039592545,
+        0.39886824990682457,
+        0.34338260700499407,
+        0.8068781715283699,
+        0.31379284611306424,
+        0.014319374319652562,
+        0.014208265894281714
+      ],
+      "min": [
+        -0.48278069496154785,
+        -0.3309336006641388,
+        0.008128181099891663,
+        1.002794623374939,
+        -3.6312508583068848,
+        -1.842738389968872,
+        -0.005453015677630901,
+        -0.042015016078948975
+      ],
+      "max": [
+        0.2103137969970703,
+        0.3904264271259308,
+        1.472778081893921,
+        3.7248642444610596,
+        3.5618896484375,
+        1.3863215446472168,
+        0.04232141748070717,
+        0.0013126095291227102
+      ],
+      "q01": null,
+      "q99": null
+    },
+    "timestamp": {
+      "mean": [
+        4.252653686389771
+      ],
+      "std": [
+        3.2577109230601815
+      ],
+      "min": [
+        0.0
+      ],
+      "max": [
+        25.2
+      ],
+      "q01": null,
+      "q99": null
+    },
+    "action": {
+      "mean": [
+        0.050704,
+        0.097407,
+        -0.094833,
+        0.994873,
+        -0.004579,
+        -0.004288,
+        0.004389,
+        0.996104,
+        0.001109,
+        0.476725
+      ],
+      "std": [
+        0.333621,
+        0.387175,
+        0.45714,
+        0.010807,
+        0.077802,
+        0.063386,
+        0.078571,
+        0.009994,
+        0.038504,
+        0.49946
+      ],
+      "min": [
+        -0.9375,
+        -0.9375,
+        -0.9375,
+        0.902028,
+        -0.356085,
+        -0.367416,
+        -0.370434,
+        0.921907,
+        -0.255,
+        0.0
+      ],
+      "max": [
+        0.9375,
+        0.9375,
+        0.9375,
+        1.0,
+        0.368853,
+        0.341214,
+        0.356395,
+        1.0,
+        0.348251,
+        1.0
+      ],
+      "q01": [
+        -0.723214,
+        -0.808929,
+        -0.9375,
+        0.934955,
+        -0.223431,
+        -0.189878,
+        -0.334735,
+        0.938516,
+        -0.107736,
+        0.0
+      ],
+      "q99": [
+        0.9375,
+        0.870536,
+        0.9375,
+        1.0,
+        0.331,
+        0.163153,
+        0.226216,
+        1.0,
+        0.127158,
+        1.0
+      ]
+    }
+  }
+}

cosmos3_nano_libero_full_finetune_bs2048/run-metrics.jsonl

@@ -0,0 +1 @@
+{"hparams": "{'_ckpt_root': './checkpoints', '_cosmos3_nano_ckpt': './checkpoints/Cosmos3-Nano', '_cosmos3_nano_transformer': './checkpoints/Cosmos3-Nano/transformer', '_cosmos3_nano_vision_encoder': './checkpoints/Cosmos3-Nano/vision_encoder', '_cosmos3_nano_tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, '_wan22_vae_path': './checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth', '_vision_vae': {'type': 'Cosmos3Wan22VAE', 'pretrained_name_or_path': './checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth', 'encode_exact_durations': [17]}, '_action_dim': 10, '_max_action_dim': 64, '_max_state_dim': 64, '_action_horizon': 16, '_frame_window_size': 17, '_prepend_state_to_action': False, '_image_height': 256, '_image_width': 256, '_video_height': 192, '_video_width': 320, '_conditioning_fps': 20.0, '_libero_action_stats': {'mean': [0.050704, 0.097407, -0.094833, 0.994873, -0.004579, -0.004288, 0.004389, 0.996104, 0.001109, 0.476725], 'std': [0.333621, 0.387175, 0.45714, 0.010807, 0.077802, 0.063386, 0.078571, 0.009994, 0.038504, 0.49946], 'min': [-0.9375, -0.9375, -0.9375, 0.902028, -0.356085, -0.367416, -0.370434, 0.921907, -0.255, 0.0], 'max': [0.9375, 0.9375, 0.9375, 1.0, 0.368853, 0.341214, 0.356395, 1.0, 0.348251, 1.0], 'q01': [-0.723214, -0.808929, -0.9375, 0.934955, -0.223431, -0.189878, -0.334735, 0.938516, -0.107736, 0.0], 'q99': [0.9375, 0.870536, 0.9375, 1.0, 0.331, 0.163153, 0.226216, 1.0, 0.127158, 1.0]}, '_cfg_dropout_rate': 0.1, '_base_lr': 5e-05, '_action_lr': 0.00025, '_per_device_batch_size': 16, '_grad_accumulation_steps': 8, '_max_steps': 5000, '_save_iter_interval': 500, '_data_root_path': ['./datasets/libero_spatial_no_noops_lerobotv2.1', './datasets/libero_spatial_no_noops_lerobotv2.1', './datasets/libero_object_no_noops_lerobotv2.1', './datasets/libero_object_no_noops_lerobotv2.1', './datasets/libero_goal_no_noops_lerobotv2.1', './datasets/libero_goal_no_noops_lerobotv2.1', './datasets/libero_10_no_noops_lerobotv2.1'], '_statistic_name': 'all_libero_no_noops', '_cosmos3_nano_special_tokens': {'eos_token_id': 151645, 'start_of_generation': 151652, 'end_of_generation': 151653}, '_cosmos3_nano_vlm_config': {'model_type': 'qwen3_vl', 'vocab_size': 151936, 'tie_word_embeddings': False, 'image_token_id': 151655, 'video_token_id': 151656, 'vision_start_token_id': 151652, 'vision_end_token_id': 151653, 'text_config': {'model_type': 'qwen3_vl_text', 'vocab_size': 151936, 'hidden_size': 4096, 'intermediate_size': 12288, 'num_hidden_layers': 36, 'num_attention_heads': 32, 'num_key_value_heads': 8, 'head_dim': 128, 'hidden_act': 'silu', 'max_position_embeddings': 262144, 'rms_norm_eps': 1e-06, 'rope_theta': 5000000, 'attention_bias': False, 'attention_dropout': 0.0, 'bos_token_id': 151643, 'eos_token_id': 151645, 'pad_token_id': 0, 'tie_word_embeddings': False, 'rope_scaling': {'rope_type': 'default', 'mrope_interleaved': True, 'mrope_section': [24, 20, 20]}, 'layer_types': ['full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention']}, 'vision_config': {'model_type': 'qwen3_vl', 'hidden_size': 1152, 'hidden_act': 'gelu_pytorch_tanh', 'intermediate_size': 4304, 'depth': 27, 'num_heads': 16, 'in_channels': 3, 'initializer_range': 0.02, 'out_hidden_size': 4096, 'patch_size': 16, 'spatial_merge_size': 2, 'temporal_patch_size': 2, 'num_position_embeddings': 2304, 'deepstack_visual_indexes': [8, 16, 24]}}, '_cosmos3_nano_name_mapping': {'action_modality_embed': 'action_modality_embed', 'vlm_backbone.model.language_model.embed_tokens.weight': 'embed_tokens.weight', 'vlm_backbone.lm_head.weight': 'lm_head.weight', 'vlm_backbone.model.language_model.norm.weight': 'norm.weight', 'vlm_backbone.model.language_model.norm_moe_gen.weight': 'norm_moe_gen.weight', 'vlm_backbone.model.language_model.layers.': 'layers.', 'vision_in_proj.projector.': 'proj_in.', 'vision_out_proj.projector.': 'proj_out.', 'time_embedder.mlp.0.': 'time_embedder.linear_1.', 'time_embedder.mlp.2.': 'time_embedder.linear_2.', 'action_in_proj.': 'action_proj_in.', 'action_out_proj.': 'action_proj_out.', '.self_attn.q_proj.': '.self_attn.to_q.', '.self_attn.k_proj.': '.self_attn.to_k.', '.self_attn.v_proj.': '.self_attn.to_v.', '.self_attn.o_proj.': '.self_attn.to_out.', '.self_attn.q_proj_moe_gen.': '.self_attn.add_q_proj.', '.self_attn.k_proj_moe_gen.': '.self_attn.add_k_proj.', '.self_attn.v_proj_moe_gen.': '.self_attn.add_v_proj.', '.self_attn.o_proj_moe_gen.': '.self_attn.to_add_out.', '.self_attn.q_norm.': '.self_attn.norm_q.', '.self_attn.k_norm.': '.self_attn.norm_k.', '.self_attn.q_norm_moe_gen.': '.self_attn.norm_added_q.', '.self_attn.k_norm_moe_gen.': '.self_attn.norm_added_k.', 'vlm_backbone.model.visual.patch_embed.': 'patch_embed.', 'vlm_backbone.model.visual.blocks.': 'blocks.', 'vlm_backbone.model.visual.pos_embed.': 'pos_embed.', 'vlm_backbone.model.visual.merger.': 'merger.', 'vlm_backbone.model.visual.deepstack_merger_list.': 'deepstack_merger_list.'}, 'model': {'type': 'Cosmos3FlowMatching', 'vlm_backbone': {'type': 'Cosmos3MoTBackbone', 'vlm_config': {'model_type': 'qwen3_vl', 'vocab_size': 151936, 'tie_word_embeddings': False, 'image_token_id': 151655, 'video_token_id': 151656, 'vision_start_token_id': 151652, 'vision_end_token_id': 151653, 'text_config': {'model_type': 'qwen3_vl_text', 'vocab_size': 151936, 'hidden_size': 4096, 'intermediate_size': 12288, 'num_hidden_layers': 36, 'num_attention_heads': 32, 'num_key_value_heads': 8, 'head_dim': 128, 'hidden_act': 'silu', 'max_position_embeddings': 262144, 'rms_norm_eps': 1e-06, 'rope_theta': 5000000, 'attention_bias': False, 'attention_dropout': 0.0, 'bos_token_id': 151643, 'eos_token_id': 151645, 'pad_token_id': 0, 'tie_word_embeddings': False, 'rope_scaling': {'rope_type': 'default', 'mrope_interleaved': True, 'mrope_section': [24, 20, 20]}, 'layer_types': ['full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention']}, 'vision_config': {'model_type': 'qwen3_vl', 'hidden_size': 1152, 'hidden_act': 'gelu_pytorch_tanh', 'intermediate_size': 4304, 'depth': 27, 'num_heads': 16, 'in_channels': 3, 'initializer_range': 0.02, 'out_hidden_size': 4096, 'patch_size': 16, 'spatial_merge_size': 2, 'temporal_patch_size': 2, 'num_position_embeddings': 2304, 'deepstack_visual_indexes': [8, 16, 24]}}, 'include_visual': False, 'vision_encoder_path': './checkpoints/Cosmos3-Nano/vision_encoder', 'skip_init_weights': True}, 'vision_latent_dim': 48, 'latent_patch_size': 2, 'max_action_dim': 64, 'num_embodiment_domains': 32, 'vision_in_proj': {'type': 'LinearProjector', 'in_dim': 192, 'out_dim': 4096}, 'vision_out_proj': {'type': 'LinearProjector', 'in_dim': 4096, 'out_dim': 192}, 'action_in_proj': {'type': 'DomainAwareLinear', 'input_size': 64, 'output_size': 4096, 'num_domains': 32}, 'action_out_proj': {'type': 'DomainAwareLinear', 'input_size': 4096, 'output_size': 64, 'num_domains': 32}, 'rectified_flow_training_config': {'shift': {'256': 3, '480': 5, '720': 10}, 'use_dynamic_shift': False, 'train_time_image_distribution': 'logitnormal', 'train_time_video_distribution': 'waver', 'train_time_action_distribution': 'logitnormal', 'train_time_weight': 'uniform', 'vision_loss_weight': 10.0, 'independent_action_schedule': False, 'shift_action': None, 'use_high_sigma_strategy': False, 'high_sigma_ratio': 0.05, 'high_sigma_timesteps_min': 995, 'high_sigma_timesteps_max': 1000, 'use_high_sigma_strategy_action': False, 'use_discrete_rf': False, 'normalize_loss_by_active': False, 'action_loss_weight': 10.0}, 'rectified_flow_inference_config': {'num_train_timesteps': 1000, 'scheduler_type': 'unipc', 'num_steps': 30, 'shift': 10.0, 'use_dynamic_shifting': False, 'use_karras_sigmas': False}, 'timestep_scale': 0.001, 'packed_attention_backend': 'flash2', 'position_embedding_type': 'unified_3d_mrope', 'unified_3d_mrope_reset_spatial_ids': True, 'unified_3d_mrope_temporal_modality_margin': 15000, 'enable_fps_modulation': True, 'base_fps': 24.0, 'special_tokens': {'eos_token_id': 151645, 'start_of_generation': 151652, 'end_of_generation': 151653}, 'pretrained_name_or_path': './checkpoints/Cosmos3-Nano/transformer', 'name_mapping': {'action_modality_embed': 'action_modality_embed', 'vlm_backbone.model.language_model.embed_tokens.weight': 'embed_tokens.weight', 'vlm_backbone.lm_head.weight': 'lm_head.weight', 'vlm_backbone.model.language_model.norm.weight': 'norm.weight', 'vlm_backbone.model.language_model.norm_moe_gen.weight': 'norm_moe_gen.weight', 'vlm_backbone.model.language_model.layers.': 'layers.', 'vision_in_proj.projector.': 'proj_in.', 'vision_out_proj.projector.': 'proj_out.', 'time_embedder.mlp.0.': 'time_embedder.linear_1.', 'time_embedder.mlp.2.': 'time_embedder.linear_2.', 'action_in_proj.': 'action_proj_in.', 'action_out_proj.': 'action_proj_out.', '.self_attn.q_proj.': '.self_attn.to_q.', '.self_attn.k_proj.': '.self_attn.to_k.', '.self_attn.v_proj.': '.self_attn.to_v.', '.self_attn.o_proj.': '.self_attn.to_out.', '.self_attn.q_proj_moe_gen.': '.self_attn.add_q_proj.', '.self_attn.k_proj_moe_gen.': '.self_attn.add_k_proj.', '.self_attn.v_proj_moe_gen.': '.self_attn.add_v_proj.', '.self_attn.o_proj_moe_gen.': '.self_attn.to_add_out.', '.self_attn.q_norm.': '.self_attn.norm_q.', '.self_attn.k_norm.': '.self_attn.norm_k.', '.self_attn.q_norm_moe_gen.': '.self_attn.norm_added_q.', '.self_attn.k_norm_moe_gen.': '.self_attn.norm_added_k.', 'vlm_backbone.model.visual.patch_embed.': 'patch_embed.', 'vlm_backbone.model.visual.blocks.': 'blocks.', 'vlm_backbone.model.visual.pos_embed.': 'pos_embed.', 'vlm_backbone.model.visual.merger.': 'merger.', 'vlm_backbone.model.visual.deepstack_merger_list.': 'deepstack_merger_list.'}, 'vision_vae': {'type': 'Cosmos3Wan22VAE', 'pretrained_name_or_path': './checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth', 'encode_exact_durations': [17]}, 'ori_action_dim': 10, 'action_horizon': 16, 'freeze_vlm_backbone': False, 'freeze_non_moe_vlm_backbone': True, 'enable_vision_loss': True}, 'inference_model': {'type': 'Cosmos3FlowMatching', 'vlm_backbone': {'type': 'Cosmos3MoTBackbone', 'vlm_config': {'model_type': 'qwen3_vl', 'vocab_size': 151936, 'tie_word_embeddings': False, 'image_token_id': 151655, 'video_token_id': 151656, 'vision_start_token_id': 151652, 'vision_end_token_id': 151653, 'text_config': {'model_type': 'qwen3_vl_text', 'vocab_size': 151936, 'hidden_size': 4096, 'intermediate_size': 12288, 'num_hidden_layers': 36, 'num_attention_heads': 32, 'num_key_value_heads': 8, 'head_dim': 128, 'hidden_act': 'silu', 'max_position_embeddings': 262144, 'rms_norm_eps': 1e-06, 'rope_theta': 5000000, 'attention_bias': False, 'attention_dropout': 0.0, 'bos_token_id': 151643, 'eos_token_id': 151645, 'pad_token_id': 0, 'tie_word_embeddings': False, 'rope_scaling': {'rope_type': 'default', 'mrope_interleaved': True, 'mrope_section': [24, 20, 20]}, 'layer_types': ['full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention']}, 'vision_config': {'model_type': 'qwen3_vl', 'hidden_size': 1152, 'hidden_act': 'gelu_pytorch_tanh', 'intermediate_size': 4304, 'depth': 27, 'num_heads': 16, 'in_channels': 3, 'initializer_range': 0.02, 'out_hidden_size': 4096, 'patch_size': 16, 'spatial_merge_size': 2, 'temporal_patch_size': 2, 'num_position_embeddings': 2304, 'deepstack_visual_indexes': [8, 16, 24]}}, 'include_visual': False, 'vision_encoder_path': './checkpoints/Cosmos3-Nano/vision_encoder', 'skip_init_weights': True}, 'vision_latent_dim': 48, 'latent_patch_size': 2, 'max_action_dim': 64, 'num_embodiment_domains': 32, 'vision_in_proj': {'type': 'LinearProjector', 'in_dim': 192, 'out_dim': 4096}, 'vision_out_proj': {'type': 'LinearProjector', 'in_dim': 4096, 'out_dim': 192}, 'action_in_proj': {'type': 'DomainAwareLinear', 'input_size': 64, 'output_size': 4096, 'num_domains': 32}, 'action_out_proj': {'type': 'DomainAwareLinear', 'input_size': 4096, 'output_size': 64, 'num_domains': 32}, 'rectified_flow_training_config': {'shift': {'256': 3, '480': 5, '720': 10}, 'use_dynamic_shift': False, 'train_time_image_distribution': 'logitnormal', 'train_time_video_distribution': 'waver', 'train_time_action_distribution': 'logitnormal', 'train_time_weight': 'uniform', 'vision_loss_weight': 10.0, 'independent_action_schedule': False, 'shift_action': None, 'use_high_sigma_strategy': False, 'high_sigma_ratio': 0.05, 'high_sigma_timesteps_min': 995, 'high_sigma_timesteps_max': 1000, 'use_high_sigma_strategy_action': False, 'use_discrete_rf': False, 'normalize_loss_by_active': False, 'action_loss_weight': 10.0}, 'rectified_flow_inference_config': {'num_train_timesteps': 1000, 'scheduler_type': 'unipc', 'num_steps': 30, 'shift': 10.0, 'use_dynamic_shifting': False, 'use_karras_sigmas': False}, 'timestep_scale': 0.001, 'packed_attention_backend': 'flash2', 'position_embedding_type': 'unified_3d_mrope', 'unified_3d_mrope_reset_spatial_ids': True, 'unified_3d_mrope_temporal_modality_margin': 15000, 'enable_fps_modulation': True, 'base_fps': 24.0, 'special_tokens': {'eos_token_id': 151645, 'start_of_generation': 151652, 'end_of_generation': 151653}, 'pretrained_name_or_path': './checkpoints/Cosmos3-Nano/transformer', 'name_mapping': {'action_modality_embed': 'action_modality_embed', 'vlm_backbone.model.language_model.embed_tokens.weight': 'embed_tokens.weight', 'vlm_backbone.lm_head.weight': 'lm_head.weight', 'vlm_backbone.model.language_model.norm.weight': 'norm.weight', 'vlm_backbone.model.language_model.norm_moe_gen.weight': 'norm_moe_gen.weight', 'vlm_backbone.model.language_model.layers.': 'layers.', 'vision_in_proj.projector.': 'proj_in.', 'vision_out_proj.projector.': 'proj_out.', 'time_embedder.mlp.0.': 'time_embedder.linear_1.', 'time_embedder.mlp.2.': 'time_embedder.linear_2.', 'action_in_proj.': 'action_proj_in.', 'action_out_proj.': 'action_proj_out.', '.self_attn.q_proj.': '.self_attn.to_q.', '.self_attn.k_proj.': '.self_attn.to_k.', '.self_attn.v_proj.': '.self_attn.to_v.', '.self_attn.o_proj.': '.self_attn.to_out.', '.self_attn.q_proj_moe_gen.': '.self_attn.add_q_proj.', '.self_attn.k_proj_moe_gen.': '.self_attn.add_k_proj.', '.self_attn.v_proj_moe_gen.': '.self_attn.add_v_proj.', '.self_attn.o_proj_moe_gen.': '.self_attn.to_add_out.', '.self_attn.q_norm.': '.self_attn.norm_q.', '.self_attn.k_norm.': '.self_attn.norm_k.', '.self_attn.q_norm_moe_gen.': '.self_attn.norm_added_q.', '.self_attn.k_norm_moe_gen.': '.self_attn.norm_added_k.', 'vlm_backbone.model.visual.patch_embed.': 'patch_embed.', 'vlm_backbone.model.visual.blocks.': 'blocks.', 'vlm_backbone.model.visual.pos_embed.': 'pos_embed.', 'vlm_backbone.model.visual.merger.': 'merger.', 'vlm_backbone.model.visual.deepstack_merger_list.': 'deepstack_merger_list.'}, 'vision_vae': {'type': 'Cosmos3Wan22VAE', 'pretrained_name_or_path': None, 'encode_exact_durations': [17]}, 'ori_action_dim': 10, 'action_horizon': 16, 'freeze_vlm_backbone': False, 'freeze_non_moe_vlm_backbone': True, 'enable_vision_loss': True}, '_transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'embodiment_id': 5}, {'type': 'LiberoFramewiseActionToRot6D'}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}}, {'type': 'ResizeImages', 'height': 256, 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'type': 'NormalizeStatesAndActions', 'action_dim': 64, 'state_dim': 64, 'state_key': 'proprio', 'action_key': 'action', 'state_norm_type': 'none', 'action_norm_type': 'quantile'}, {'type': 'BuildCosmos3Sequence', 'raw_action_dim': 10, 'mode': 'wam', 'frame_window_size': 17, 'prepend_state_to_action': False, 'conditioning_fps': 20.0}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 17, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}], 'train_dataloader': {'per_device_batch_size': 16, 'per_device_num_workers': 4, 'dataset': {'type': 'DistributedRepeatingDataset', 'name_mappings': {'observation.state': ['proprio'], 'action': ['action']}, 'statistic_keys': ['observation.state', 'timestamp', 'action'], 'statistic_name': 'all_libero_no_noops', 'statistics_overrides': {'all_libero_no_noops': {'action': {'mean': [0.050704, 0.097407, -0.094833, 0.994873, -0.004579, -0.004288, 0.004389, 0.996104, 0.001109, 0.476725], 'std': [0.333621, 0.387175, 0.45714, 0.010807, 0.077802, 0.063386, 0.078571, 0.009994, 0.038504, 0.49946], 'min': [-0.9375, -0.9375, -0.9375, 0.902028, -0.356085, -0.367416, -0.370434, 0.921907, -0.255, 0.0], 'max': [0.9375, 0.9375, 0.9375, 1.0, 0.368853, 0.341214, 0.356395, 1.0, 0.348251, 1.0], 'q01': [-0.723214, -0.808929, -0.9375, 0.934955, -0.223431, -0.189878, -0.334735, 0.938516, -0.107736, 0.0], 'q99': [0.9375, 0.870536, 0.9375, 1.0, 0.331, 0.163153, 0.226216, 1.0, 0.127158, 1.0]}}}, 'datasets': [{'type': 'ParquetDataset', 'data_root_path': './datasets/libero_spatial_no_noops_lerobotv2.1', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'embodiment_id': 5}, {'type': 'LiberoFramewiseActionToRot6D'}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}}, {'type': 'ResizeImages', 'height': 256, 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'type': 'NormalizeStatesAndActions', 'action_dim': 64, 'state_dim': 64, 'state_key': 'proprio', 'action_key': 'action', 'state_norm_type': 'none', 'action_norm_type': 'quantile'}, {'type': 'BuildCosmos3Sequence', 'raw_action_dim': 10, 'mode': 'wam', 'frame_window_size': 17, 'prepend_state_to_action': False, 'conditioning_fps': 20.0}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 17, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}], 'action_window_size': 16, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'all_libero_no_noops', 'window_start_idx': 0, 'frame_window_size': 17, 'require_full_window': True}, {'type': 'ParquetDataset', 'data_root_path': './datasets/libero_spatial_no_noops_lerobotv2.1', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'embodiment_id': 5}, {'type': 'LiberoFramewiseActionToRot6D'}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}}, {'type': 'ResizeImages', 'height': 256, 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'type': 'NormalizeStatesAndActions', 'action_dim': 64, 'state_dim': 64, 'state_key': 'proprio', 'action_key': 'action', 'state_norm_type': 'none', 'action_norm_type': 'quantile'}, {'type': 'BuildCosmos3Sequence', 'raw_action_dim': 10, 'mode': 'wam', 'frame_window_size': 17, 'prepend_state_to_action': False, 'conditioning_fps': 20.0}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 17, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}], 'action_window_size': 16, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'all_libero_no_noops', 'window_start_idx': 0, 'frame_window_size': 17, 'require_full_window': True}, {'type': 'ParquetDataset', 'data_root_path': './datasets/libero_object_no_noops_lerobotv2.1', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'embodiment_id': 5}, {'type': 'LiberoFramewiseActionToRot6D'}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}}, {'type': 'ResizeImages', 'height': 256, 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'type': 'NormalizeStatesAndActions', 'action_dim': 64, 'state_dim': 64, 'state_key': 'proprio', 'action_key': 'action', 'state_norm_type': 'none', 'action_norm_type': 'quantile'}, {'type': 'BuildCosmos3Sequence', 'raw_action_dim': 10, 'mode': 'wam', 'frame_window_size': 17, 'prepend_state_to_action': False, 'conditioning_fps': 20.0}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 17, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}], 'action_window_size': 16, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'all_libero_no_noops', 'window_start_idx': 0, 'frame_window_size': 17, 'require_full_window': True}, {'type': 'ParquetDataset', 'data_root_path': './datasets/libero_object_no_noops_lerobotv2.1', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'embodiment_id': 5}, {'type': 'LiberoFramewiseActionToRot6D'}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}}, {'type': 'ResizeImages', 'height': 256, 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'type': 'NormalizeStatesAndActions', 'action_dim': 64, 'state_dim': 64, 'state_key': 'proprio', 'action_key': 'action', 'state_norm_type': 'none', 'action_norm_type': 'quantile'}, {'type': 'BuildCosmos3Sequence', 'raw_action_dim': 10, 'mode': 'wam', 'frame_window_size': 17, 'prepend_state_to_action': False, 'conditioning_fps': 20.0}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 17, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}], 'action_window_size': 16, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'all_libero_no_noops', 'window_start_idx': 0, 'frame_window_size': 17, 'require_full_window': True}, {'type': 'ParquetDataset', 'data_root_path': './datasets/libero_goal_no_noops_lerobotv2.1', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'embodiment_id': 5}, {'type': 'LiberoFramewiseActionToRot6D'}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}}, {'type': 'ResizeImages', 'height': 256, 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'type': 'NormalizeStatesAndActions', 'action_dim': 64, 'state_dim': 64, 'state_key': 'proprio', 'action_key': 'action', 'state_norm_type': 'none', 'action_norm_type': 'quantile'}, {'type': 'BuildCosmos3Sequence', 'raw_action_dim': 10, 'mode': 'wam', 'frame_window_size': 17, 'prepend_state_to_action': False, 'conditioning_fps': 20.0}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 17, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}], 'action_window_size': 16, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'all_libero_no_noops', 'window_start_idx': 0, 'frame_window_size': 17, 'require_full_window': True}, {'type': 'ParquetDataset', 'data_root_path': './datasets/libero_goal_no_noops_lerobotv2.1', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'embodiment_id': 5}, {'type': 'LiberoFramewiseActionToRot6D'}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}}, {'type': 'ResizeImages', 'height': 256, 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'type': 'NormalizeStatesAndActions', 'action_dim': 64, 'state_dim': 64, 'state_key': 'proprio', 'action_key': 'action', 'state_norm_type': 'none', 'action_norm_type': 'quantile'}, {'type': 'BuildCosmos3Sequence', 'raw_action_dim': 10, 'mode': 'wam', 'frame_window_size': 17, 'prepend_state_to_action': False, 'conditioning_fps': 20.0}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 17, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}], 'action_window_size': 16, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'all_libero_no_noops', 'window_start_idx': 0, 'frame_window_size': 17, 'require_full_window': True}, {'type': 'ParquetDataset', 'data_root_path': './datasets/libero_10_no_noops_lerobotv2.1', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'embodiment_id': 5}, {'type': 'LiberoFramewiseActionToRot6D'}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}}, {'type': 'ResizeImages', 'height': 256, 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'type': 'NormalizeStatesAndActions', 'action_dim': 64, 'state_dim': 64, 'state_key': 'proprio', 'action_key': 'action', 'state_norm_type': 'none', 'action_norm_type': 'quantile'}, {'type': 'BuildCosmos3Sequence', 'raw_action_dim': 10, 'mode': 'wam', 'frame_window_size': 17, 'prepend_state_to_action': False, 'conditioning_fps': 20.0}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 17, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}], 'action_window_size': 16, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'all_libero_no_noops', 'window_start_idx': 0, 'frame_window_size': 17, 'require_full_window': True}]}}, 'runner': {'type': 'FSDPTrainRunner', 'max_steps': 5000, 'save_iter_interval': 500, 'max_keep_ckpts': 2, 'optimizer': {'lr': 5e-05, 'type': 'AdamW', 'weight_decay': 0.05, 'betas': (0.9, 0.99), 'eps': 1e-08, 'fused': True, 'exclude_1d_from_weight_decay': False, 'paramwise_learning_rate': {'action_in_proj.': 0.00025, 'action_out_proj.': 0.00025, 'action_modality_embed': 0.00025}}, 'max_grad_norm': 1.0, 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'collator': {'type': 'Cosmos3Collator', 'tensor_keys': ['images', 'actions', 'embodiment_ids', 'raw_action_dim', 'conditioning_fps', 'action_fps'], 'sequence_keys': ['text_token_ids'], 'list_keys': ['sequence_plan'], 'meta_keys': ['task_description', 'stats', 'info', 'timestamp'], 'pad_id': 0}, 'sampler': None, 'grad_accumulation_steps': 8, 'metric': {'type': 'VLAMetric', 'active_trackers': ('jsonl', 'wandb'), 'run_dir': '/mnt/data/cpfs/users/jikun/wk_dir/cosmos3/libero_align_16_9_padding', 'grad_accumulation_steps': 8, 'window_size': 1}, 'lr_scheduler': {'type': 'linear-warmup+linear-decay', 'warmup_steps': 500, 'cycle_length': 16000}, 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'mixed_precision_dtype': 'bf16', 'sharding_strategy': 'full-shard', 'change_key_name': False, 'cfg': Config (path: configs/cosmos3/cosmos3nano_libero_full_finetune.py): {'_ckpt_root': './checkpoints', '_cosmos3_nano_ckpt': './checkpoints/Cosmos3-Nano', '_cosmos3_nano_transformer': './checkpoints/Cosmos3-Nano/transformer', '_cosmos3_nano_vision_encoder': './checkpoints/Cosmos3-Nano/vision_encoder', '_cosmos3_nano_tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, '_wan22_vae_path': './checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth', '_vision_vae': {'type': 'Cosmos3Wan22VAE', 'pretrained_name_or_path': './checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth', 'encode_exact_durations': [17]}, '_action_dim': 10, '_max_action_dim': 64, '_max_state_dim': 64, '_action_horizon': 16, '_frame_window_size': 17, '_prepend_state_to_action': False, '_image_height': 256, '_image_width': 256, '_video_height': 192, '_video_width': 320, '_conditioning_fps': 20.0, '_libero_action_stats': {'mean': [0.050704, 0.097407, -0.094833, 0.994873, -0.004579, -0.004288, 0.004389, 0.996104, 0.001109, 0.476725], 'std': [0.333621, 0.387175, 0.45714, 0.010807, 0.077802, 0.063386, 0.078571, 0.009994, 0.038504, 0.49946], 'min': [-0.9375, -0.9375, -0.9375, 0.902028, -0.356085, -0.367416, -0.370434, 0.921907, -0.255, 0.0], 'max': [0.9375, 0.9375, 0.9375, 1.0, 0.368853, 0.341214, 0.356395, 1.0, 0.348251, 1.0], 'q01': [-0.723214, -0.808929, -0.9375, 0.934955, -0.223431, -0.189878, -0.334735, 0.938516, -0.107736, 0.0], 'q99': [0.9375, 0.870536, 0.9375, 1.0, 0.331, 0.163153, 0.226216, 1.0, 0.127158, 1.0]}, '_cfg_dropout_rate': 0.1, '_base_lr': 5e-05, '_action_lr': 0.00025, '_per_device_batch_size': 16, '_grad_accumulation_steps': 8, '_max_steps': 5000, '_save_iter_interval': 500, '_data_root_path': ['./datasets/libero_spatial_no_noops_lerobotv2.1', './datasets/libero_spatial_no_noops_lerobotv2.1', './datasets/libero_object_no_noops_lerobotv2.1', './datasets/libero_object_no_noops_lerobotv2.1', './datasets/libero_goal_no_noops_lerobotv2.1', './datasets/libero_goal_no_noops_lerobotv2.1', './datasets/libero_10_no_noops_lerobotv2.1'], '_statistic_name': 'all_libero_no_noops', '_cosmos3_nano_special_tokens': {'eos_token_id': 151645, 'start_of_generation': 151652, 'end_of_generation': 151653}, '_cosmos3_nano_vlm_config': {'model_type': 'qwen3_vl', 'vocab_size': 151936, 'tie_word_embeddings': False, 'image_token_id': 151655, 'video_token_id': 151656, 'vision_start_token_id': 151652, 'vision_end_token_id': 151653, 'text_config': {'model_type': 'qwen3_vl_text', 'vocab_size': 151936, 'hidden_size': 4096, 'intermediate_size': 12288, 'num_hidden_layers': 36, 'num_attention_heads': 32, 'num_key_value_heads': 8, 'head_dim': 128, 'hidden_act': 'silu', 'max_position_embeddings': 262144, 'rms_norm_eps': 1e-06, 'rope_theta': 5000000, 'attention_bias': False, 'attention_dropout': 0.0, 'bos_token_id': 151643, 'eos_token_id': 151645, 'pad_token_id': 0, 'tie_word_embeddings': False, 'rope_scaling': {'rope_type': 'default', 'mrope_interleaved': True, 'mrope_section': [24, 20, 20]}, 'layer_types': ['full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention']}, 'vision_config': {'model_type': 'qwen3_vl', 'hidden_size': 1152, 'hidden_act': 'gelu_pytorch_tanh', 'intermediate_size': 4304, 'depth': 27, 'num_heads': 16, 'in_channels': 3, 'initializer_range': 0.02, 'out_hidden_size': 4096, 'patch_size': 16, 'spatial_merge_size': 2, 'temporal_patch_size': 2, 'num_position_embeddings': 2304, 'deepstack_visual_indexes': [8, 16, 24]}}, '_cosmos3_nano_name_mapping': {'action_modality_embed': 'action_modality_embed', 'vlm_backbone.model.language_model.embed_tokens.weight': 'embed_tokens.weight', 'vlm_backbone.lm_head.weight': 'lm_head.weight', 'vlm_backbone.model.language_model.norm.weight': 'norm.weight', 'vlm_backbone.model.language_model.norm_moe_gen.weight': 'norm_moe_gen.weight', 'vlm_backbone.model.language_model.layers.': 'layers.', 'vision_in_proj.projector.': 'proj_in.', 'vision_out_proj.projector.': 'proj_out.', 'time_embedder.mlp.0.': 'time_embedder.linear_1.', 'time_embedder.mlp.2.': 'time_embedder.linear_2.', 'action_in_proj.': 'action_proj_in.', 'action_out_proj.': 'action_proj_out.', '.self_attn.q_proj.': '.self_attn.to_q.', '.self_attn.k_proj.': '.self_attn.to_k.', '.self_attn.v_proj.': '.self_attn.to_v.', '.self_attn.o_proj.': '.self_attn.to_out.', '.self_attn.q_proj_moe_gen.': '.self_attn.add_q_proj.', '.self_attn.k_proj_moe_gen.': '.self_attn.add_k_proj.', '.self_attn.v_proj_moe_gen.': '.self_attn.add_v_proj.', '.self_attn.o_proj_moe_gen.': '.self_attn.to_add_out.', '.self_attn.q_norm.': '.self_attn.norm_q.', '.self_attn.k_norm.': '.self_attn.norm_k.', '.self_attn.q_norm_moe_gen.': '.self_attn.norm_added_q.', '.self_attn.k_norm_moe_gen.': '.self_attn.norm_added_k.', 'vlm_backbone.model.visual.patch_embed.': 'patch_embed.', 'vlm_backbone.model.visual.blocks.': 'blocks.', 'vlm_backbone.model.visual.pos_embed.': 'pos_embed.', 'vlm_backbone.model.visual.merger.': 'merger.', 'vlm_backbone.model.visual.deepstack_merger_list.': 'deepstack_merger_list.'}, 'model': {'type': 'Cosmos3FlowMatching', 'vlm_backbone': {'type': 'Cosmos3MoTBackbone', 'vlm_config': {'model_type': 'qwen3_vl', 'vocab_size': 151936, 'tie_word_embeddings': False, 'image_token_id': 151655, 'video_token_id': 151656, 'vision_start_token_id': 151652, 'vision_end_token_id': 151653, 'text_config': {'model_type': 'qwen3_vl_text', 'vocab_size': 151936, 'hidden_size': 4096, 'intermediate_size': 12288, 'num_hidden_layers': 36, 'num_attention_heads': 32, 'num_key_value_heads': 8, 'head_dim': 128, 'hidden_act': 'silu', 'max_position_embeddings': 262144, 'rms_norm_eps': 1e-06, 'rope_theta': 5000000, 'attention_bias': False, 'attention_dropout': 0.0, 'bos_token_id': 151643, 'eos_token_id': 151645, 'pad_token_id': 0, 'tie_word_embeddings': False, 'rope_scaling': {'rope_type': 'default', 'mrope_interleaved': True, 'mrope_section': [24, 20, 20]}, 'layer_types': ['full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention']}, 'vision_config': {'model_type': 'qwen3_vl', 'hidden_size': 1152, 'hidden_act': 'gelu_pytorch_tanh', 'intermediate_size': 4304, 'depth': 27, 'num_heads': 16, 'in_channels': 3, 'initializer_range': 0.02, 'out_hidden_size': 4096, 'patch_size': 16, 'spatial_merge_size': 2, 'temporal_patch_size': 2, 'num_position_embeddings': 2304, 'deepstack_visual_indexes': [8, 16, 24]}}, 'include_visual': False, 'vision_encoder_path': './checkpoints/Cosmos3-Nano/vision_encoder', 'skip_init_weights': True}, 'vision_latent_dim': 48, 'latent_patch_size': 2, 'max_action_dim': 64, 'num_embodiment_domains': 32, 'vision_in_proj': {'type': 'LinearProjector', 'in_dim': 192, 'out_dim': 4096}, 'vision_out_proj': {'type': 'LinearProjector', 'in_dim': 4096, 'out_dim': 192}, 'action_in_proj': {'type': 'DomainAwareLinear', 'input_size': 64, 'output_size': 4096, 'num_domains': 32}, 'action_out_proj': {'type': 'DomainAwareLinear', 'input_size': 4096, 'output_size': 64, 'num_domains': 32}, 'rectified_flow_training_config': {'shift': {'256': 3, '480': 5, '720': 10}, 'use_dynamic_shift': False, 'train_time_image_distribution': 'logitnormal', 'train_time_video_distribution': 'waver', 'train_time_action_distribution': 'logitnormal', 'train_time_weight': 'uniform', 'vision_loss_weight': 10.0, 'independent_action_schedule': False, 'shift_action': None, 'use_high_sigma_strategy': False, 'high_sigma_ratio': 0.05, 'high_sigma_timesteps_min': 995, 'high_sigma_timesteps_max': 1000, 'use_high_sigma_strategy_action': False, 'use_discrete_rf': False, 'normalize_loss_by_active': False, 'action_loss_weight': 10.0}, 'rectified_flow_inference_config': {'num_train_timesteps': 1000, 'scheduler_type': 'unipc', 'num_steps': 30, 'shift': 10.0, 'use_dynamic_shifting': False, 'use_karras_sigmas': False}, 'timestep_scale': 0.001, 'packed_attention_backend': 'flash2', 'position_embedding_type': 'unified_3d_mrope', 'unified_3d_mrope_reset_spatial_ids': True, 'unified_3d_mrope_temporal_modality_margin': 15000, 'enable_fps_modulation': True, 'base_fps': 24.0, 'special_tokens': {'eos_token_id': 151645, 'start_of_generation': 151652, 'end_of_generation': 151653}, 'pretrained_name_or_path': './checkpoints/Cosmos3-Nano/transformer', 'name_mapping': {'action_modality_embed': 'action_modality_embed', 'vlm_backbone.model.language_model.embed_tokens.weight': 'embed_tokens.weight', 'vlm_backbone.lm_head.weight': 'lm_head.weight', 'vlm_backbone.model.language_model.norm.weight': 'norm.weight', 'vlm_backbone.model.language_model.norm_moe_gen.weight': 'norm_moe_gen.weight', 'vlm_backbone.model.language_model.layers.': 'layers.', 'vision_in_proj.projector.': 'proj_in.', 'vision_out_proj.projector.': 'proj_out.', 'time_embedder.mlp.0.': 'time_embedder.linear_1.', 'time_embedder.mlp.2.': 'time_embedder.linear_2.', 'action_in_proj.': 'action_proj_in.', 'action_out_proj.': 'action_proj_out.', '.self_attn.q_proj.': '.self_attn.to_q.', '.self_attn.k_proj.': '.self_attn.to_k.', '.self_attn.v_proj.': '.self_attn.to_v.', '.self_attn.o_proj.': '.self_attn.to_out.', '.self_attn.q_proj_moe_gen.': '.self_attn.add_q_proj.', '.self_attn.k_proj_moe_gen.': '.self_attn.add_k_proj.', '.self_attn.v_proj_moe_gen.': '.self_attn.add_v_proj.', '.self_attn.o_proj_moe_gen.': '.self_attn.to_add_out.', '.self_attn.q_norm.': '.self_attn.norm_q.', '.self_attn.k_norm.': '.self_attn.norm_k.', '.self_attn.q_norm_moe_gen.': '.self_attn.norm_added_q.', '.self_attn.k_norm_moe_gen.': '.self_attn.norm_added_k.', 'vlm_backbone.model.visual.patch_embed.': 'patch_embed.', 'vlm_backbone.model.visual.blocks.': 'blocks.', 'vlm_backbone.model.visual.pos_embed.': 'pos_embed.', 'vlm_backbone.model.visual.merger.': 'merger.', 'vlm_backbone.model.visual.deepstack_merger_list.': 'deepstack_merger_list.'}, 'vision_vae': {'type': 'Cosmos3Wan22VAE', 'pretrained_name_or_path': './checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth', 'encode_exact_durations': [17]}, 'ori_action_dim': 10, 'action_horizon': 16, 'freeze_vlm_backbone': False, 'freeze_non_moe_vlm_backbone': True, 'enable_vision_loss': True}, 'inference_model': {'type': 'Cosmos3FlowMatching', 'vlm_backbone': {'type': 'Cosmos3MoTBackbone', 'vlm_config': {'model_type': 'qwen3_vl', 'vocab_size': 151936, 'tie_word_embeddings': False, 'image_token_id': 151655, 'video_token_id': 151656, 'vision_start_token_id': 151652, 'vision_end_token_id': 151653, 'text_config': {'model_type': 'qwen3_vl_text', 'vocab_size': 151936, 'hidden_size': 4096, 'intermediate_size': 12288, 'num_hidden_layers': 36, 'num_attention_heads': 32, 'num_key_value_heads': 8, 'head_dim': 128, 'hidden_act': 'silu', 'max_position_embeddings': 262144, 'rms_norm_eps': 1e-06, 'rope_theta': 5000000, 'attention_bias': False, 'attention_dropout': 0.0, 'bos_token_id': 151643, 'eos_token_id': 151645, 'pad_token_id': 0, 'tie_word_embeddings': False, 'rope_scaling': {'rope_type': 'default', 'mrope_interleaved': True, 'mrope_section': [24, 20, 20]}, 'layer_types': ['full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention']}, 'vision_config': {'model_type': 'qwen3_vl', 'hidden_size': 1152, 'hidden_act': 'gelu_pytorch_tanh', 'intermediate_size': 4304, 'depth': 27, 'num_heads': 16, 'in_channels': 3, 'initializer_range': 0.02, 'out_hidden_size': 4096, 'patch_size': 16, 'spatial_merge_size': 2, 'temporal_patch_size': 2, 'num_position_embeddings': 2304, 'deepstack_visual_indexes': [8, 16, 24]}}, 'include_visual': False, 'vision_encoder_path': './checkpoints/Cosmos3-Nano/vision_encoder', 'skip_init_weights': True}, 'vision_latent_dim': 48, 'latent_patch_size': 2, 'max_action_dim': 64, 'num_embodiment_domains': 32, 'vision_in_proj': {'type': 'LinearProjector', 'in_dim': 192, 'out_dim': 4096}, 'vision_out_proj': {'type': 'LinearProjector', 'in_dim': 4096, 'out_dim': 192}, 'action_in_proj': {'type': 'DomainAwareLinear', 'input_size': 64, 'output_size': 4096, 'num_domains': 32}, 'action_out_proj': {'type': 'DomainAwareLinear', 'input_size': 4096, 'output_size': 64, 'num_domains': 32}, 'rectified_flow_training_config': {'shift': {'256': 3, '480': 5, '720': 10}, 'use_dynamic_shift': False, 'train_time_image_distribution': 'logitnormal', 'train_time_video_distribution': 'waver', 'train_time_action_distribution': 'logitnormal', 'train_time_weight': 'uniform', 'vision_loss_weight': 10.0, 'independent_action_schedule': False, 'shift_action': None, 'use_high_sigma_strategy': False, 'high_sigma_ratio': 0.05, 'high_sigma_timesteps_min': 995, 'high_sigma_timesteps_max': 1000, 'use_high_sigma_strategy_action': False, 'use_discrete_rf': False, 'normalize_loss_by_active': False, 'action_loss_weight': 10.0}, 'rectified_flow_inference_config': {'num_train_timesteps': 1000, 'scheduler_type': 'unipc', 'num_steps': 30, 'shift': 10.0, 'use_dynamic_shifting': False, 'use_karras_sigmas': False}, 'timestep_scale': 0.001, 'packed_attention_backend': 'flash2', 'position_embedding_type': 'unified_3d_mrope', 'unified_3d_mrope_reset_spatial_ids': True, 'unified_3d_mrope_temporal_modality_margin': 15000, 'enable_fps_modulation': True, 'base_fps': 24.0, 'special_tokens': {'eos_token_id': 151645, 'start_of_generation': 151652, 'end_of_generation': 151653}, 'pretrained_name_or_path': './checkpoints/Cosmos3-Nano/transformer', 'name_mapping': {'action_modality_embed': 'action_modality_embed', 'vlm_backbone.model.language_model.embed_tokens.weight': 'embed_tokens.weight', 'vlm_backbone.lm_head.weight': 'lm_head.weight', 'vlm_backbone.model.language_model.norm.weight': 'norm.weight', 'vlm_backbone.model.language_model.norm_moe_gen.weight': 'norm_moe_gen.weight', 'vlm_backbone.model.language_model.layers.': 'layers.', 'vision_in_proj.projector.': 'proj_in.', 'vision_out_proj.projector.': 'proj_out.', 'time_embedder.mlp.0.': 'time_embedder.linear_1.', 'time_embedder.mlp.2.': 'time_embedder.linear_2.', 'action_in_proj.': 'action_proj_in.', 'action_out_proj.': 'action_proj_out.', '.self_attn.q_proj.': '.self_attn.to_q.', '.self_attn.k_proj.': '.self_attn.to_k.', '.self_attn.v_proj.': '.self_attn.to_v.', '.self_attn.o_proj.': '.self_attn.to_out.', '.self_attn.q_proj_moe_gen.': '.self_attn.add_q_proj.', '.self_attn.k_proj_moe_gen.': '.self_attn.add_k_proj.', '.self_attn.v_proj_moe_gen.': '.self_attn.add_v_proj.', '.self_attn.o_proj_moe_gen.': '.self_attn.to_add_out.', '.self_attn.q_norm.': '.self_attn.norm_q.', '.self_attn.k_norm.': '.self_attn.norm_k.', '.self_attn.q_norm_moe_gen.': '.self_attn.norm_added_q.', '.self_attn.k_norm_moe_gen.': '.self_attn.norm_added_k.', 'vlm_backbone.model.visual.patch_embed.': 'patch_embed.', 'vlm_backbone.model.visual.blocks.': 'blocks.', 'vlm_backbone.model.visual.pos_embed.': 'pos_embed.', 'vlm_backbone.model.visual.merger.': 'merger.', 'vlm_backbone.model.visual.deepstack_merger_list.': 'deepstack_merger_list.'}, 'vision_vae': {'type': 'Cosmos3Wan22VAE', 'pretrained_name_or_path': None, 'encode_exact_durations': [17]}, 'ori_action_dim': 10, 'action_horizon': 16, 'freeze_vlm_backbone': False, 'freeze_non_moe_vlm_backbone': True, 'enable_vision_loss': True}, '_transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'embodiment_id': 5}, {'type': 'LiberoFramewiseActionToRot6D'}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}}, {'type': 'ResizeImages', 'height': 256, 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'type': 'NormalizeStatesAndActions', 'action_dim': 64, 'state_dim': 64, 'state_key': 'proprio', 'action_key': 'action', 'state_norm_type': 'none', 'action_norm_type': 'quantile'}, {'type': 'BuildCosmos3Sequence', 'raw_action_dim': 10, 'mode': 'wam', 'frame_window_size': 17, 'prepend_state_to_action': False, 'conditioning_fps': 20.0}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 17, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}], 'train_dataloader': {'per_device_batch_size': 16, 'per_device_num_workers': 4, 'dataset': {'type': 'DistributedRepeatingDataset', 'name_mappings': {'observation.state': ['proprio'], 'action': ['action']}, 'statistic_keys': ['observation.state', 'timestamp', 'action'], 'statistic_name': 'all_libero_no_noops', 'statistics_overrides': {'all_libero_no_noops': {'action': {'mean': [0.050704, 0.097407, -0.094833, 0.994873, -0.004579, -0.004288, 0.004389, 0.996104, 0.001109, 0.476725], 'std': [0.333621, 0.387175, 0.45714, 0.010807, 0.077802, 0.063386, 0.078571, 0.009994, 0.038504, 0.49946], 'min': [-0.9375, -0.9375, -0.9375, 0.902028, -0.356085, -0.367416, -0.370434, 0.921907, -0.255, 0.0], 'max': [0.9375, 0.9375, 0.9375, 1.0, 0.368853, 0.341214, 0.356395, 1.0, 0.348251, 1.0], 'q01': [-0.723214, -0.808929, -0.9375, 0.934955, -0.223431, -0.189878, -0.334735, 0.938516, -0.107736, 0.0], 'q99': [0.9375, 0.870536, 0.9375, 1.0, 0.331, 0.163153, 0.226216, 1.0, 0.127158, 1.0]}}}, 'datasets': [{'type': 'ParquetDataset', 'data_root_path': './datasets/libero_spatial_no_noops_lerobotv2.1', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'embodiment_id': 5}, {'type': 'LiberoFramewiseActionToRot6D'}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}}, {'type': 'ResizeImages', 'height': 256, 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'type': 'NormalizeStatesAndActions', 'action_dim': 64, 'state_dim': 64, 'state_key': 'proprio', 'action_key': 'action', 'state_norm_type': 'none', 'action_norm_type': 'quantile'}, {'type': 'BuildCosmos3Sequence', 'raw_action_dim': 10, 'mode': 'wam', 'frame_window_size': 17, 'prepend_state_to_action': False, 'conditioning_fps': 20.0}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 17, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}], 'action_window_size': 16, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'all_libero_no_noops', 'window_start_idx': 0, 'frame_window_size': 17, 'require_full_window': True}, {'type': 'ParquetDataset', 'data_root_path': './datasets/libero_spatial_no_noops_lerobotv2.1', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'embodiment_id': 5}, {'type': 'LiberoFramewiseActionToRot6D'}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}}, {'type': 'ResizeImages', 'height': 256, 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'type': 'NormalizeStatesAndActions', 'action_dim': 64, 'state_dim': 64, 'state_key': 'proprio', 'action_key': 'action', 'state_norm_type': 'none', 'action_norm_type': 'quantile'}, {'type': 'BuildCosmos3Sequence', 'raw_action_dim': 10, 'mode': 'wam', 'frame_window_size': 17, 'prepend_state_to_action': False, 'conditioning_fps': 20.0}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 17, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}], 'action_window_size': 16, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'all_libero_no_noops', 'window_start_idx': 0, 'frame_window_size': 17, 'require_full_window': True}, {'type': 'ParquetDataset', 'data_root_path': './datasets/libero_object_no_noops_lerobotv2.1', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'embodiment_id': 5}, {'type': 'LiberoFramewiseActionToRot6D'}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}}, {'type': 'ResizeImages', 'height': 256, 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'type': 'NormalizeStatesAndActions', 'action_dim': 64, 'state_dim': 64, 'state_key': 'proprio', 'action_key': 'action', 'state_norm_type': 'none', 'action_norm_type': 'quantile'}, {'type': 'BuildCosmos3Sequence', 'raw_action_dim': 10, 'mode': 'wam', 'frame_window_size': 17, 'prepend_state_to_action': False, 'conditioning_fps': 20.0}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 17, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}], 'action_window_size': 16, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'all_libero_no_noops', 'window_start_idx': 0, 'frame_window_size': 17, 'require_full_window': True}, {'type': 'ParquetDataset', 'data_root_path': './datasets/libero_object_no_noops_lerobotv2.1', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'embodiment_id': 5}, {'type': 'LiberoFramewiseActionToRot6D'}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}}, {'type': 'ResizeImages', 'height': 256, 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'type': 'NormalizeStatesAndActions', 'action_dim': 64, 'state_dim': 64, 'state_key': 'proprio', 'action_key': 'action', 'state_norm_type': 'none', 'action_norm_type': 'quantile'}, {'type': 'BuildCosmos3Sequence', 'raw_action_dim': 10, 'mode': 'wam', 'frame_window_size': 17, 'prepend_state_to_action': False, 'conditioning_fps': 20.0}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 17, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}], 'action_window_size': 16, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'all_libero_no_noops', 'window_start_idx': 0, 'frame_window_size': 17, 'require_full_window': True}, {'type': 'ParquetDataset', 'data_root_path': './datasets/libero_goal_no_noops_lerobotv2.1', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'embodiment_id': 5}, {'type': 'LiberoFramewiseActionToRot6D'}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}}, {'type': 'ResizeImages', 'height': 256, 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'type': 'NormalizeStatesAndActions', 'action_dim': 64, 'state_dim': 64, 'state_key': 'proprio', 'action_key': 'action', 'state_norm_type': 'none', 'action_norm_type': 'quantile'}, {'type': 'BuildCosmos3Sequence', 'raw_action_dim': 10, 'mode': 'wam', 'frame_window_size': 17, 'prepend_state_to_action': False, 'conditioning_fps': 20.0}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 17, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}], 'action_window_size': 16, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'all_libero_no_noops', 'window_start_idx': 0, 'frame_window_size': 17, 'require_full_window': True}, {'type': 'ParquetDataset', 'data_root_path': './datasets/libero_goal_no_noops_lerobotv2.1', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'embodiment_id': 5}, {'type': 'LiberoFramewiseActionToRot6D'}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}}, {'type': 'ResizeImages', 'height': 256, 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'type': 'NormalizeStatesAndActions', 'action_dim': 64, 'state_dim': 64, 'state_key': 'proprio', 'action_key': 'action', 'state_norm_type': 'none', 'action_norm_type': 'quantile'}, {'type': 'BuildCosmos3Sequence', 'raw_action_dim': 10, 'mode': 'wam', 'frame_window_size': 17, 'prepend_state_to_action': False, 'conditioning_fps': 20.0}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 17, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}], 'action_window_size': 16, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'all_libero_no_noops', 'window_start_idx': 0, 'frame_window_size': 17, 'require_full_window': True}, {'type': 'ParquetDataset', 'data_root_path': './datasets/libero_10_no_noops_lerobotv2.1', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'embodiment_id': 5}, {'type': 'LiberoFramewiseActionToRot6D'}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.1, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}}, {'type': 'ResizeImages', 'height': 256, 'width': 256}, {'type': 'SimpleNormalizeImages'}, {'type': 'NormalizeStatesAndActions', 'action_dim': 64, 'state_dim': 64, 'state_key': 'proprio', 'action_key': 'action', 'state_norm_type': 'none', 'action_norm_type': 'quantile'}, {'type': 'BuildCosmos3Sequence', 'raw_action_dim': 10, 'mode': 'wam', 'frame_window_size': 17, 'prepend_state_to_action': False, 'conditioning_fps': 20.0}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 17, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}], 'action_window_size': 16, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'all_libero_no_noops', 'window_start_idx': 0, 'frame_window_size': 17, 'require_full_window': True}]}}, 'runner': {'type': 'FSDPTrainRunner', 'max_steps': 5000, 'save_iter_interval': 500, 'max_keep_ckpts': 2, 'optimizer': {'lr': 5e-05, 'type': 'AdamW', 'weight_decay': 0.05, 'betas': (0.9, 0.99), 'eps': 1e-08, 'fused': True, 'exclude_1d_from_weight_decay': False, 'paramwise_learning_rate': {'action_in_proj.': 0.00025, 'action_out_proj.': 0.00025, 'action_modality_embed': 0.00025}}, 'max_grad_norm': 1.0, 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'collator': {'type': 'Cosmos3Collator', 'tensor_keys': ['images', 'actions', 'embodiment_ids', 'raw_action_dim', 'conditioning_fps', 'action_fps'], 'sequence_keys': ['text_token_ids'], 'list_keys': ['sequence_plan'], 'meta_keys': ['task_description', 'stats', 'info', 'timestamp'], 'pad_id': 0}, 'sampler': None, 'grad_accumulation_steps': 8, 'metric': {'type': 'VLAMetric', 'active_trackers': ('jsonl', 'wandb'), 'run_dir': '/mnt/data/cpfs/users/jikun/wk_dir/cosmos3/libero_align_16_9_padding', 'grad_accumulation_steps': 8, 'window_size': 1}, 'lr_scheduler': {'type': 'linear-warmup+linear-decay', 'warmup_steps': 500, 'cycle_length': 16000}, 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'mixed_precision_dtype': 'bf16', 'sharding_strategy': 'full-shard', 'change_key_name': False, 'cfg': Config (path: configs/cosmos3/cosmos3nano_libero_full_finetune.py): {...}, 'args': Namespace(config='configs/cosmos3/cosmos3nano_libero_full_finetune.py', work_dir='/mnt/data/cpfs/users/jikun/wk_dir/cosmos3/libero_align_16_9_padding', cfg_options=None, eval_after_train=True, resume_from=None)}, 'eval': {'type': 'LiberoEvalRunner', 'task_suite_name': 'libero_10', 'model_family': 'cosmos3', 'norm_stats_key': 'all_libero_no_noops', 'eval_chunk_size': 16, 'num_trials_per_task': 50, 'num_steps_wait': 10, 'seed': 7, 'enable_mixed_precision_training': True, 'mixed_precision_dtype': 'bf16', 'dataset': {'type': 'LiberoParquetEvalDataset', 'img_buffer_len': 1, 'extra_tensor_keys': ['conditioning_fps', 'prepend_state_to_action'], 'transforms': [{'type': 'ProcessLiberoEvalInputs', 'img_keys': ['agentview_image', 'robot0_eye_in_hand_image'], 'resize_size': 256, 'embodiment_id': 5}, {'type': 'SetCosmos3ActionMetadata', 'conditioning_fps': 20.0, 'prepend_state_to_action': False}, {'type': 'TransformImage', 'image_resize_strategy': 'resize-crop', 'input_sizes': [[3, 256, 256], [3, 256, 256]], 'means': [[127.5, 127.5, 127.5], [127.5, 127.5, 127.5]], 'stds': [[127.5, 127.5, 127.5], [127.5, 127.5, 127.5]]}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.0, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}, 'output_key': 'lang_tokens', 'output_attention_mask_key': 'lang_masks'}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 1, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}]}, 'denormalize_action': {'type': 'DenormalizeLiberoFramewiseRot6DAction', 'norm_type': 'quantile_rot', 'action_dim': 10}}}, 'args': Namespace(config='configs/cosmos3/cosmos3nano_libero_full_finetune.py', work_dir='/mnt/data/cpfs/users/jikun/wk_dir/cosmos3/libero_align_16_9_padding', cfg_options=None, eval_after_train=True, resume_from=None)}, 'eval': {'type': 'LiberoEvalRunner', 'task_suite_name': 'libero_10', 'model_family': 'cosmos3', 'norm_stats_key': 'all_libero_no_noops', 'eval_chunk_size': 16, 'num_trials_per_task': 50, 'num_steps_wait': 10, 'seed': 7, 'enable_mixed_precision_training': True, 'mixed_precision_dtype': 'bf16', 'dataset': {'type': 'LiberoParquetEvalDataset', 'img_buffer_len': 1, 'extra_tensor_keys': ['conditioning_fps', 'prepend_state_to_action'], 'transforms': [{'type': 'ProcessLiberoEvalInputs', 'img_keys': ['agentview_image', 'robot0_eye_in_hand_image'], 'resize_size': 256, 'embodiment_id': 5}, {'type': 'SetCosmos3ActionMetadata', 'conditioning_fps': 20.0, 'prepend_state_to_action': False}, {'type': 'TransformImage', 'image_resize_strategy': 'resize-crop', 'input_sizes': [[3, 256, 256], [3, 256, 256]], 'means': [[127.5, 127.5, 127.5], [127.5, 127.5, 127.5]], 'stds': [[127.5, 127.5, 127.5], [127.5, 127.5, 127.5]]}, {'type': 'ProcessCosmos3Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos3-Nano/text_tokenizer', 'model_max_length': 4096, 'padding_side': 'right', 'trust_remote_code': True}, 'max_len': 512, 'cfg_dropout_rate': 0.0, 'format_prompt_as_json': True, 'action_metadata': {'append_viewpoint': False, 'frame_window_size': 17, 'conditioning_fps': 20.0, 'video_height': 192, 'video_width': 320}, 'output_key': 'lang_tokens', 'output_attention_mask_key': 'lang_masks'}, {'type': 'PrepareVideo', 'num_views': 2, 'frame_window_size': 1, 'tile_direction': 'horizontal'}, {'type': 'ResizeAndReflectPad', 'height': 192, 'width': 320}]}, 'denormalize_action': {'type': 'DenormalizeLiberoFramewiseRot6DAction', 'norm_type': 'quantile_rot', 'action_dim': 10}}}", "run_id": "cosmos3nano_libero_full_finetune_2026_08_14_18_30_47"}

cosmos3_nano_libero_full_finetune_bs2048/tokenizer/added_tokens.json

@@ -0,0 +1,28 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

cosmos3_nano_libero_full_finetune_bs2048/tokenizer/chat_template.jinja

@@ -0,0 +1,120 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {%- if messages[0].content is string %}
+            {{- messages[0].content }}
+        {%- else %}
+            {%- for content in messages[0].content %}
+                {%- if 'text' in content %}
+                    {{- content.text }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {{- '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' }}
+        {%- if messages[0].content is string %}
+            {{- messages[0].content }}
+        {%- else %}
+            {%- for content in messages[0].content %}
+                {%- if 'text' in content %}
+                    {{- content.text }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- for message in messages %}
+    {%- if message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' }}
+        {%- if message.content is string %}
+            {{- message.content }}
+        {%- else %}
+            {%- for content in message.content %}
+                {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                    {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
+                    <|vision_start|><|image_pad|><|vision_end|>
+                {%- elif content.type == 'video' or 'video' in content %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                    {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
+                    <|vision_start|><|video_pad|><|vision_end|>
+                {%- elif 'text' in content %}
+                    {{- content.text }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role + '\n' }}
+        {%- if message.content is string %}
+            {{- message.content }}
+        {%- else %}
+            {%- for content_item in message.content %}
+                {%- if 'text' in content_item %}
+                    {{- content_item.text }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and message.content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {%- if message.content is string %}
+            {{- message.content }}
+        {%- else %}
+            {%- for content in message.content %}
+                {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                    {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
+                    <|vision_start|><|image_pad|><|vision_end|>
+                {%- elif content.type == 'video' or 'video' in content %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                    {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
+                    <|vision_start|><|video_pad|><|vision_end|>
+                {%- elif 'text' in content %}
+                    {{- content.text }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

cosmos3_nano_libero_full_finetune_bs2048/tokenizer/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

cosmos3_nano_libero_full_finetune_bs2048/tokenizer/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

cosmos3_nano_libero_full_finetune_bs2048/tokenizer/tokenizer_config.json

@@ -0,0 +1,239 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 262144,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

cosmos3_nano_libero_full_finetune_bs2048/vlm_backbone_config.json

@@ -0,0 +1,100 @@
+{
+  "image_token_id": 151655,
+  "model_type": "qwen3_vl",
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 151643,
+    "eos_token_id": 151645,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 4096,
+    "initializer_range": 0.02,
+    "intermediate_size": 12288,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 262144,
+    "model_type": "qwen3_vl_text",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 36,
+    "num_key_value_heads": 8,
+    "packed_attention_backend": "flash2",
+    "pad_token_id": 0,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "mrope_interleaved": true,
+      "mrope_section": [
+        24,
+        20,
+        20
+      ],
+      "rope_theta": 5000000,
+      "rope_type": "default"
+    },
+    "tie_word_embeddings": false,
+    "use_cache": true,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.3.0",
+  "video_token_id": 151656,
+  "vision_config": {
+    "deepstack_visual_indexes": [
+      8,
+      16,
+      24
+    ],
+    "depth": 27,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "in_channels": 3,
+    "initializer_range": 0.02,
+    "intermediate_size": 4304,
+    "model_type": "qwen3_vl",
+    "num_heads": 16,
+    "num_position_embeddings": 2304,
+    "out_hidden_size": 4096,
+    "patch_size": 16,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2
+  },
+  "vision_end_token_id": 151653,
+  "vision_start_token_id": 151652,
+  "vocab_size": 151936
+}

dit4dit_libero_all_full_finetune_bc071270ff2_bs256/checkpoints/step-120000-epoch-147-loss=0.0439.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:828f5700102fa9f5ac54f2f8cca015263a74ef7dd75a187759e3a1042b625a7f
+size 25727358750

dit4dit_libero_all_full_finetune_bc071270ff2_bs256/config.json

@@ -0,0 +1,1359 @@
+{
+  "_action_dim": 8,
+  "_action_horizon": 8,
+  "_action_norm_mask": [
+    true,
+    true,
+    true,
+    true,
+    true,
+    true,
+    false
+  ],
+  "_ckpt_root": "./checkpoints",
+  "_cosmos_base_model": "./checkpoints/Cosmos-Predict2.5-2B",
+  "_cosmos_revision": "diffusers/base/post-trained",
+  "_cosmos_tokenizer": {
+    "model_max_length": 512,
+    "model_path": "./checkpoints/Cosmos-Predict2.5-2B/tokenizer",
+    "type": "PretrainedTokenizer"
+  },
+  "_dit4dit_parquet_dataset": {
+    "action_key": "action",
+    "action_window_size": 8,
+    "frame_window_size": 9,
+    "statistic_name": "franka",
+    "transforms": [
+      {
+        "frame_stride": 2,
+        "name_mappings": {
+          "actions": [
+            "actions"
+          ],
+          "observation.state": [
+            "states"
+          ]
+        },
+        "parquet_keys": [
+          "observation.state",
+          "timestamp",
+          "actions",
+          "info",
+          "stats",
+          "action_masks"
+        ],
+        "type": "ProcessParquetInputs",
+        "video_keys": [
+          "observation.images.image",
+          "observation.images.wrist_image"
+        ]
+      },
+      {
+        "input_keys": [
+          "prompt",
+          "task_description",
+          "lang",
+          "instruction",
+          "instructions",
+          "text"
+        ],
+        "output_key": "prompt",
+        "remove_source_keys": true,
+        "type": "CanonicalizePrompt"
+      },
+      {
+        "input_key": "prompt",
+        "remove_input_key": true,
+        "tokenizer": {
+          "model_max_length": 512,
+          "model_path": "./checkpoints/Cosmos-Predict2.5-2B/tokenizer",
+          "type": "PretrainedTokenizer"
+        },
+        "type": "ProcessCosmos25Prompt"
+      },
+      {
+        "backend": "torch",
+        "height": 224,
+        "output_layout": "nchw",
+        "scale_divisor": 255.0,
+        "type": "ResizeImages",
+        "width": 224
+      },
+      {
+        "frame_stride": 1,
+        "keep_time_dim": true,
+        "key": "images",
+        "num_views": 2,
+        "type": "ConcatImagesHorizontally",
+        "views_first": true
+      },
+      {
+        "backend": "torch",
+        "interleave": true,
+        "keys": [
+          "states"
+        ],
+        "target_dims": {
+          "states": 16
+        },
+        "type": "SinCosKeys"
+      },
+      {
+        "action_dim": null,
+        "action_key": "action",
+        "action_norm_mask": [
+          true,
+          true,
+          true,
+          true,
+          true,
+          true,
+          false
+        ],
+        "action_norm_type": "min_max",
+        "clip_norm": false,
+        "norm_type": "min_max",
+        "normalization_epsilon": 0.0,
+        "output_dtype": "float16",
+        "preserve_input_dtype": true,
+        "state_dim": null,
+        "state_key": "proprio",
+        "state_norm_type": "none",
+        "type": "NormalizeStatesAndActions"
+      },
+      {
+        "action_dim": 8,
+        "type": "PadActionsAndActionMasks",
+        "valid_action_dim": 7
+      },
+      {
+        "action_dim": 8,
+        "action_horizon": 8,
+        "mark_all_action_steps_valid": true,
+        "state_dim": 16,
+        "type": "PrepareDiT4DiTInputs",
+        "valid_action_dim": 7
+      }
+    ],
+    "type": "ParquetDataset",
+    "use_delta": false,
+    "window_start_idx": 0
+  },
+  "_dit4dit_train_transforms": [
+    {
+      "frame_stride": 2,
+      "name_mappings": {
+        "actions": [
+          "actions"
+        ],
+        "observation.state": [
+          "states"
+        ]
+      },
+      "parquet_keys": [
+        "observation.state",
+        "timestamp",
+        "actions",
+        "info",
+        "stats",
+        "action_masks"
+      ],
+      "type": "ProcessParquetInputs",
+      "video_keys": [
+        "observation.images.image",
+        "observation.images.wrist_image"
+      ]
+    },
+    {
+      "input_keys": [
+        "prompt",
+        "task_description",
+        "lang",
+        "instruction",
+        "instructions",
+        "text"
+      ],
+      "output_key": "prompt",
+      "remove_source_keys": true,
+      "type": "CanonicalizePrompt"
+    },
+    {
+      "input_key": "prompt",
+      "remove_input_key": true,
+      "tokenizer": {
+        "model_max_length": 512,
+        "model_path": "./checkpoints/Cosmos-Predict2.5-2B/tokenizer",
+        "type": "PretrainedTokenizer"
+      },
+      "type": "ProcessCosmos25Prompt"
+    },
+    {
+      "backend": "torch",
+      "height": 224,
+      "output_layout": "nchw",
+      "scale_divisor": 255.0,
+      "type": "ResizeImages",
+      "width": 224
+    },
+    {
+      "frame_stride": 1,
+      "keep_time_dim": true,
+      "key": "images",
+      "num_views": 2,
+      "type": "ConcatImagesHorizontally",
+      "views_first": true
+    },
+    {
+      "backend": "torch",
+      "interleave": true,
+      "keys": [
+        "states"
+      ],
+      "target_dims": {
+        "states": 16
+      },
+      "type": "SinCosKeys"
+    },
+    {
+      "action_dim": null,
+      "action_key": "action",
+      "action_norm_mask": [
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        false
+      ],
+      "action_norm_type": "min_max",
+      "clip_norm": false,
+      "norm_type": "min_max",
+      "normalization_epsilon": 0.0,
+      "output_dtype": "float16",
+      "preserve_input_dtype": true,
+      "state_dim": null,
+      "state_key": "proprio",
+      "state_norm_type": "none",
+      "type": "NormalizeStatesAndActions"
+    },
+    {
+      "action_dim": 8,
+      "type": "PadActionsAndActionMasks",
+      "valid_action_dim": 7
+    },
+    {
+      "action_dim": 8,
+      "action_horizon": 8,
+      "mark_all_action_steps_valid": true,
+      "state_dim": 16,
+      "type": "PrepareDiT4DiTInputs",
+      "valid_action_dim": 7
+    }
+  ],
+  "_frame_window_size": 9,
+  "_image_frame_stride": 2,
+  "_image_size": 224,
+  "_libero_data_roots": [
+    "./datasets/libero_object_no_noops_lerobotv2.1",
+    "./datasets/libero_goal_no_noops_lerobotv2.1",
+    "./datasets/libero_spatial_no_noops_lerobotv2.1",
+    "./datasets/libero_10_no_noops_lerobotv2.1"
+  ],
+  "_official_dataset_statistics": {
+    "franka": {
+      "action": {
+        "mask": [
+          true,
+          true,
+          true,
+          true,
+          true,
+          true,
+          false
+        ],
+        "max": [
+          0.9375,
+          0.9375,
+          0.9375,
+          0.3557142913341522,
+          0.375,
+          0.375,
+          1.0
+        ],
+        "mean": [
+          0.07237596483901143,
+          0.08987006871029735,
+          -0.10144743137061596,
+          -0.00045383188989944756,
+          0.006273590726777911,
+          -0.003878799732774496,
+          0.524486355483532
+        ],
+        "min": [
+          -0.9375,
+          -0.9375,
+          -0.9375,
+          -0.2582142949104309,
+          -0.375,
+          -0.3675000071525574,
+          0.0
+        ],
+        "q01": [
+          -0.8785714507102966,
+          -0.8758928775787354,
+          -0.9375,
+          -0.1510714292526245,
+          -0.20678570866584778,
+          -0.2742857038974762,
+          0.0
+        ],
+        "q99": [
+          0.9375,
+          0.9107142686843872,
+          0.9375,
+          0.20357142388820648,
+          0.26357144117355347,
+          0.375,
+          1.0
+        ],
+        "std": [
+          0.3498823308902479,
+          0.37794140366375184,
+          0.460084266976933,
+          0.0403885784928603,
+          0.06616144248501059,
+          0.07763074391911857,
+          0.4994683356809767
+        ]
+      },
+      "num_trajectories": 1693,
+      "num_transitions": 273465,
+      "state": {
+        "max": [
+          0.21031762659549713,
+          0.39128610491752625,
+          1.3660105466842651,
+          3.6714255809783936,
+          3.560650587081909,
+          1.386339545249939,
+          0.04233968257904053,
+          0.0013633022317662835
+        ],
+        "mean": [
+          -0.04889854742214084,
+          0.03689368185587227,
+          0.7890402488410473,
+          2.9771945476531982,
+          -0.1417286954820156,
+          -0.11769362539052963,
+          0.026436020154505968,
+          -0.02665513101965189
+        ],
+        "min": [
+          -0.4828203022480011,
+          -0.3255046010017395,
+          0.008128180168569088,
+          0.35277295112609863,
+          -3.641430377960205,
+          -1.842738389968872,
+          -0.0013586411951109767,
+          -0.042040832340717316
+        ],
+        "q01": [
+          -0.42401049643754957,
+          -0.2838300323486328,
+          0.009925739830359817,
+          1.3085840785503386,
+          -2.886677579879761,
+          -1.1599004411697387,
+          0.001503719249740243,
+          -0.040336399003863335
+        ],
+        "q99": [
+          0.1530261474847791,
+          0.3629165390133857,
+          1.2910678112506866,
+          3.303542451858519,
+          2.7496529006957933,
+          0.6893712210655194,
+          0.040610933862626555,
+          -0.0015016929572448147
+        ],
+        "std": [
+          0.10639013941746686,
+          0.15115733130675715,
+          0.38406895599530033,
+          0.3530238395244304,
+          0.8227341427331599,
+          0.32357567121520087,
+          0.014583991652936385,
+          0.014467005007200339
+        ]
+      }
+    }
+  },
+  "_official_dit4dit_ckpt": "./checkpoints/dit4dit-model/dit4dit_libero/final_model/pytorch_model.pt",
+  "_ori_action_dim": 7,
+  "_prompt_input_keys": [
+    "prompt",
+    "task_description",
+    "lang",
+    "instruction",
+    "instructions",
+    "text"
+  ],
+  "_state_dim": 16,
+  "eval": {
+    "dataset": {
+      "img_buffer_len": 1,
+      "require_lang_tokens": true,
+      "transforms": [
+        {
+          "input_keys": [
+            "prompt",
+            "task_description",
+            "lang",
+            "instruction",
+            "instructions",
+            "text"
+          ],
+          "output_key": "prompt",
+          "remove_source_keys": true,
+          "type": "CanonicalizePrompt"
+        },
+        {
+          "input_key": "prompt",
+          "remove_input_key": true,
+          "tokenizer": {
+            "model_max_length": 512,
+            "model_path": "./checkpoints/Cosmos-Predict2.5-2B/tokenizer",
+            "type": "PretrainedTokenizer"
+          },
+          "type": "ProcessCosmos25Prompt"
+        },
+        {
+          "image_size": 224,
+          "img_keys": [
+            "agentview_image",
+            "robot0_eye_in_hand_image"
+          ],
+          "type": "ProcessDiT4DiTLiberoEvalInputs"
+        },
+        {
+          "gripper_key": "robot0_gripper_qpos",
+          "norm_type": "none",
+          "out_key": "states",
+          "pos_key": "robot0_eef_pos",
+          "quat_key": "robot0_eef_quat",
+          "state_dim": null,
+          "type": "LiberoProprioFromInputs"
+        },
+        {
+          "interleave": true,
+          "keys": [
+            "states"
+          ],
+          "target_dims": {
+            "states": 16
+          },
+          "type": "SinCosKeys"
+        },
+        {
+          "image_key": "pixel_values",
+          "require_actions": false,
+          "state_dim": 16,
+          "type": "PrepareDiT4DiTInputs"
+        }
+      ],
+      "type": "LiberoParquetEvalDataset"
+    },
+    "denormalize_action": {
+      "action_dim": 7,
+      "action_norm_mask": [
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        false
+      ],
+      "clip_normalized_action": true,
+      "invert_gripper_action": true,
+      "norm_type": "min_max",
+      "normalize_gripper_action": true,
+      "type": "DenormalizeLiberoAction"
+    },
+    "enable_mixed_precision_training": false,
+    "eval_chunk_size": 8,
+    "mixed_precision_dtype": "bf16",
+    "model_family": "dit4dit",
+    "norm_stats_key": "franka",
+    "num_steps_wait": 10,
+    "num_trials_per_task": 50,
+    "resize_size": 224,
+    "seed": 7,
+    "task_suite_name": [
+      "libero_spatial",
+      "libero_object",
+      "libero_goal",
+      "libero_10"
+    ],
+    "truncate_action_chunk_at_horizon": true,
+    "type": "LiberoEvalRunner"
+  },
+  "inference_model": {
+    "freeze_vlm_backbone": false,
+    "name_mapping": null,
+    "pretrained_name_or_path": null,
+    "repeated_diffusion_steps": 4,
+    "strict_mapping": false,
+    "type": "DiT4DiTVLA",
+    "vla_head": {
+      "action_dim": 8,
+      "action_horizon": 8,
+      "action_model_type": "DiT-B",
+      "add_pos_embed": true,
+      "diffusion_model_cfg": {
+        "cross_attention_dim": 2048,
+        "dropout": 0.2,
+        "final_dropout": true,
+        "interleave_self_attention": true,
+        "norm_type": "ada_norm",
+        "num_layers": 16,
+        "output_dim": 2560,
+        "positional_embeddings": null
+      },
+      "future_action_window_size": 7,
+      "hidden_size": 2560,
+      "max_seq_len": 1024,
+      "noise_beta_alpha": 1.5,
+      "noise_beta_beta": 1.0,
+      "noise_s": 0.999,
+      "num_inference_timesteps": 4,
+      "num_timestep_buckets": 1000,
+      "ori_action_dim": 7,
+      "state_dim": 16,
+      "type": "DiT4DiTActionHead"
+    },
+    "vlm_backbone": {
+      "base_model": "./checkpoints/Cosmos-Predict2.5-2B",
+      "conditional_frame_timestep": 0.0001,
+      "detach_hidden_states": true,
+      "extract_layer": 17,
+      "fixed_seed": null,
+      "flow_matching_high_sigma_min": null,
+      "flow_matching_high_sigma_ratio": null,
+      "flow_matching_time_distribution": "uniform",
+      "frozen_submodules": [
+        "text_encoder",
+        "vae"
+      ],
+      "fsdp_min_num_params": 0,
+      "future_loss_type": "flow_matching",
+      "local_files_only": true,
+      "max_sequence_length": 512,
+      "num_frames_out": 5,
+      "num_inference_steps": 1,
+      "revision": "diffusers/base/post-trained",
+      "split_future_frames": true,
+      "torch_dtype": "bf16",
+      "trainable": true,
+      "type": "Cosmos25Backbone"
+    }
+  },
+  "model": {
+    "freeze_vlm_backbone": false,
+    "name_mapping": {
+      "vla_head": "action_model",
+      "vlm_backbone.text_encoder": "backbone_interface.extractor.text_encoder",
+      "vlm_backbone.transformer": "backbone_interface.extractor.transformer",
+      "vlm_backbone.vae": "backbone_interface.extractor.vae"
+    },
+    "pretrained_name_or_path": null,
+    "repeated_diffusion_steps": 4,
+    "strict_mapping": false,
+    "type": "DiT4DiTVLA",
+    "vla_head": {
+      "action_dim": 8,
+      "action_horizon": 8,
+      "action_model_type": "DiT-B",
+      "add_pos_embed": true,
+      "diffusion_model_cfg": {
+        "cross_attention_dim": 2048,
+        "dropout": 0.2,
+        "final_dropout": true,
+        "interleave_self_attention": true,
+        "norm_type": "ada_norm",
+        "num_layers": 16,
+        "output_dim": 2560,
+        "positional_embeddings": null
+      },
+      "future_action_window_size": 7,
+      "hidden_size": 2560,
+      "max_seq_len": 1024,
+      "noise_beta_alpha": 1.5,
+      "noise_beta_beta": 1.0,
+      "noise_s": 0.999,
+      "num_inference_timesteps": 4,
+      "num_timestep_buckets": 1000,
+      "ori_action_dim": 7,
+      "state_dim": 16,
+      "type": "DiT4DiTActionHead"
+    },
+    "vlm_backbone": {
+      "base_model": "./checkpoints/Cosmos-Predict2.5-2B",
+      "conditional_frame_timestep": 0.0001,
+      "detach_hidden_states": true,
+      "extract_layer": 17,
+      "fixed_seed": null,
+      "flow_matching_high_sigma_min": null,
+      "flow_matching_high_sigma_ratio": null,
+      "flow_matching_time_distribution": "uniform",
+      "frozen_submodules": [
+        "text_encoder",
+        "vae"
+      ],
+      "fsdp_min_num_params": 0,
+      "future_loss_type": "flow_matching",
+      "local_files_only": true,
+      "max_sequence_length": 512,
+      "num_frames_out": 5,
+      "num_inference_steps": 1,
+      "revision": "diffusers/base/post-trained",
+      "split_future_frames": true,
+      "torch_dtype": "bf16",
+      "trainable": true,
+      "type": "Cosmos25Backbone"
+    }
+  },
+  "official_eval_task_suites": [
+    "libero_spatial",
+    "libero_object",
+    "libero_goal",
+    "libero_10"
+  ],
+  "runner": {
+    "change_key_name": false,
+    "collator": {
+      "keys": [
+        "states",
+        "timestamp",
+        "images",
+        "img_masks",
+        "actions",
+        "action_masks",
+        "frame_masks",
+        "lang_tokens",
+        "lang_masks"
+      ],
+      "meta_keys": [
+        "info",
+        "stats"
+      ],
+      "type": "DictCollator"
+    },
+    "enable_gradient_checkpointing": false,
+    "enable_mixed_precision_training": true,
+    "grad_accumulation_steps": 2,
+    "lr_scheduler": {
+      "min_lr": 5e-07,
+      "type": "cosine_with_min_lr",
+      "warmup_steps": 10000
+    },
+    "max_grad_norm": 1.0,
+    "max_keep_ckpts": 2,
+    "max_steps": 160000,
+    "metric": {
+      "active_trackers": [
+        "jsonl",
+        "wandb"
+      ],
+      "grad_accumulation_steps": 1,
+      "run_dir": "work_dirs",
+      "type": "VLAMetric",
+      "window_size": 1
+    },
+    "mixed_precision_dtype": "bf16",
+    "optimizer": {
+      "betas": [
+        0.9,
+        0.95
+      ],
+      "eps": 1e-08,
+      "lr": 3e-05,
+      "paramwise_learning_rate": {
+        "vla_head": 0.0001,
+        "vlm_backbone.transformer": 0.0001
+      },
+      "type": "AdamW",
+      "weight_decay": 1e-08
+    },
+    "pre_fsdp_param_dtype": "fp32",
+    "reduce_in_full_precision": false,
+    "sampler": null,
+    "save_iter_interval": 40000,
+    "sharding_strategy": "shard-grad-op",
+    "tokenizer": {
+      "model_max_length": 512,
+      "model_path": "./checkpoints/Cosmos-Predict2.5-2B/tokenizer",
+      "type": "PretrainedTokenizer"
+    },
+    "type": "FSDPTrainRunner"
+  },
+  "seed": 42,
+  "train_dataloader": {
+    "dataset": {
+      "dataset_statistics": {
+        "franka": {
+          "action": {
+            "mask": [
+              true,
+              true,
+              true,
+              true,
+              true,
+              true,
+              false
+            ],
+            "max": [
+              0.9375,
+              0.9375,
+              0.9375,
+              0.3557142913341522,
+              0.375,
+              0.375,
+              1.0
+            ],
+            "mean": [
+              0.07237596483901143,
+              0.08987006871029735,
+              -0.10144743137061596,
+              -0.00045383188989944756,
+              0.006273590726777911,
+              -0.003878799732774496,
+              0.524486355483532
+            ],
+            "min": [
+              -0.9375,
+              -0.9375,
+              -0.9375,
+              -0.2582142949104309,
+              -0.375,
+              -0.3675000071525574,
+              0.0
+            ],
+            "q01": [
+              -0.8785714507102966,
+              -0.8758928775787354,
+              -0.9375,
+              -0.1510714292526245,
+              -0.20678570866584778,
+              -0.2742857038974762,
+              0.0
+            ],
+            "q99": [
+              0.9375,
+              0.9107142686843872,
+              0.9375,
+              0.20357142388820648,
+              0.26357144117355347,
+              0.375,
+              1.0
+            ],
+            "std": [
+              0.3498823308902479,
+              0.37794140366375184,
+              0.460084266976933,
+              0.0403885784928603,
+              0.06616144248501059,
+              0.07763074391911857,
+              0.4994683356809767
+            ]
+          },
+          "num_trajectories": 1693,
+          "num_transitions": 273465,
+          "state": {
+            "max": [
+              0.21031762659549713,
+              0.39128610491752625,
+              1.3660105466842651,
+              3.6714255809783936,
+              3.560650587081909,
+              1.386339545249939,
+              0.04233968257904053,
+              0.0013633022317662835
+            ],
+            "mean": [
+              -0.04889854742214084,
+              0.03689368185587227,
+              0.7890402488410473,
+              2.9771945476531982,
+              -0.1417286954820156,
+              -0.11769362539052963,
+              0.026436020154505968,
+              -0.02665513101965189
+            ],
+            "min": [
+              -0.4828203022480011,
+              -0.3255046010017395,
+              0.008128180168569088,
+              0.35277295112609863,
+              -3.641430377960205,
+              -1.842738389968872,
+              -0.0013586411951109767,
+              -0.042040832340717316
+            ],
+            "q01": [
+              -0.42401049643754957,
+              -0.2838300323486328,
+              0.009925739830359817,
+              1.3085840785503386,
+              -2.886677579879761,
+              -1.1599004411697387,
+              0.001503719249740243,
+              -0.040336399003863335
+            ],
+            "q99": [
+              0.1530261474847791,
+              0.3629165390133857,
+              1.2910678112506866,
+              3.303542451858519,
+              2.7496529006957933,
+              0.6893712210655194,
+              0.040610933862626555,
+              -0.0015016929572448147
+            ],
+            "std": [
+              0.10639013941746686,
+              0.15115733130675715,
+              0.38406895599530033,
+              0.3530238395244304,
+              0.8227341427331599,
+              0.32357567121520087,
+              0.014583991652936385,
+              0.014467005007200339
+            ]
+          }
+        }
+      },
+      "datasets": [
+        {
+          "action_key": "action",
+          "action_window_size": 8,
+          "data_root_path": "./datasets/libero_object_no_noops_lerobotv2.1",
+          "frame_window_size": 9,
+          "statistic_name": "franka",
+          "transforms": [
+            {
+              "frame_stride": 2,
+              "name_mappings": {
+                "actions": [
+                  "actions"
+                ],
+                "observation.state": [
+                  "states"
+                ]
+              },
+              "parquet_keys": [
+                "observation.state",
+                "timestamp",
+                "actions",
+                "info",
+                "stats",
+                "action_masks"
+              ],
+              "type": "ProcessParquetInputs",
+              "video_keys": [
+                "observation.images.image",
+                "observation.images.wrist_image"
+              ]
+            },
+            {
+              "input_keys": [
+                "prompt",
+                "task_description",
+                "lang",
+                "instruction",
+                "instructions",
+                "text"
+              ],
+              "output_key": "prompt",
+              "remove_source_keys": true,
+              "type": "CanonicalizePrompt"
+            },
+            {
+              "input_key": "prompt",
+              "remove_input_key": true,
+              "tokenizer": {
+                "model_max_length": 512,
+                "model_path": "./checkpoints/Cosmos-Predict2.5-2B/tokenizer",
+                "type": "PretrainedTokenizer"
+              },
+              "type": "ProcessCosmos25Prompt"
+            },
+            {
+              "backend": "torch",
+              "height": 224,
+              "output_layout": "nchw",
+              "scale_divisor": 255.0,
+              "type": "ResizeImages",
+              "width": 224
+            },
+            {
+              "frame_stride": 1,
+              "keep_time_dim": true,
+              "key": "images",
+              "num_views": 2,
+              "type": "ConcatImagesHorizontally",
+              "views_first": true
+            },
+            {
+              "backend": "torch",
+              "interleave": true,
+              "keys": [
+                "states"
+              ],
+              "target_dims": {
+                "states": 16
+              },
+              "type": "SinCosKeys"
+            },
+            {
+              "action_dim": null,
+              "action_key": "action",
+              "action_norm_mask": [
+                true,
+                true,
+                true,
+                true,
+                true,
+                true,
+                false
+              ],
+              "action_norm_type": "min_max",
+              "clip_norm": false,
+              "norm_type": "min_max",
+              "normalization_epsilon": 0.0,
+              "output_dtype": "float16",
+              "preserve_input_dtype": true,
+              "state_dim": null,
+              "state_key": "proprio",
+              "state_norm_type": "none",
+              "type": "NormalizeStatesAndActions"
+            },
+            {
+              "action_dim": 8,
+              "type": "PadActionsAndActionMasks",
+              "valid_action_dim": 7
+            },
+            {
+              "action_dim": 8,
+              "action_horizon": 8,
+              "mark_all_action_steps_valid": true,
+              "state_dim": 16,
+              "type": "PrepareDiT4DiTInputs",
+              "valid_action_dim": 7
+            }
+          ],
+          "type": "ParquetDataset",
+          "use_delta": false,
+          "window_start_idx": 0
+        },
+        {
+          "action_key": "action",
+          "action_window_size": 8,
+          "data_root_path": "./datasets/libero_goal_no_noops_lerobotv2.1",
+          "frame_window_size": 9,
+          "statistic_name": "franka",
+          "transforms": [
+            {
+              "frame_stride": 2,
+              "name_mappings": {
+                "actions": [
+                  "actions"
+                ],
+                "observation.state": [
+                  "states"
+                ]
+              },
+              "parquet_keys": [
+                "observation.state",
+                "timestamp",
+                "actions",
+                "info",
+                "stats",
+                "action_masks"
+              ],
+              "type": "ProcessParquetInputs",
+              "video_keys": [
+                "observation.images.image",
+                "observation.images.wrist_image"
+              ]
+            },
+            {
+              "input_keys": [
+                "prompt",
+                "task_description",
+                "lang",
+                "instruction",
+                "instructions",
+                "text"
+              ],
+              "output_key": "prompt",
+              "remove_source_keys": true,
+              "type": "CanonicalizePrompt"
+            },
+            {
+              "input_key": "prompt",
+              "remove_input_key": true,
+              "tokenizer": {
+                "model_max_length": 512,
+                "model_path": "./checkpoints/Cosmos-Predict2.5-2B/tokenizer",
+                "type": "PretrainedTokenizer"
+              },
+              "type": "ProcessCosmos25Prompt"
+            },
+            {
+              "backend": "torch",
+              "height": 224,
+              "output_layout": "nchw",
+              "scale_divisor": 255.0,
+              "type": "ResizeImages",
+              "width": 224
+            },
+            {
+              "frame_stride": 1,
+              "keep_time_dim": true,
+              "key": "images",
+              "num_views": 2,
+              "type": "ConcatImagesHorizontally",
+              "views_first": true
+            },
+            {
+              "backend": "torch",
+              "interleave": true,
+              "keys": [
+                "states"
+              ],
+              "target_dims": {
+                "states": 16
+              },
+              "type": "SinCosKeys"
+            },
+            {
+              "action_dim": null,
+              "action_key": "action",
+              "action_norm_mask": [
+                true,
+                true,
+                true,
+                true,
+                true,
+                true,
+                false
+              ],
+              "action_norm_type": "min_max",
+              "clip_norm": false,
+              "norm_type": "min_max",
+              "normalization_epsilon": 0.0,
+              "output_dtype": "float16",
+              "preserve_input_dtype": true,
+              "state_dim": null,
+              "state_key": "proprio",
+              "state_norm_type": "none",
+              "type": "NormalizeStatesAndActions"
+            },
+            {
+              "action_dim": 8,
+              "type": "PadActionsAndActionMasks",
+              "valid_action_dim": 7
+            },
+            {
+              "action_dim": 8,
+              "action_horizon": 8,
+              "mark_all_action_steps_valid": true,
+              "state_dim": 16,
+              "type": "PrepareDiT4DiTInputs",
+              "valid_action_dim": 7
+            }
+          ],
+          "type": "ParquetDataset",
+          "use_delta": false,
+          "window_start_idx": 0
+        },
+        {
+          "action_key": "action",
+          "action_window_size": 8,
+          "data_root_path": "./datasets/libero_spatial_no_noops_lerobotv2.1",
+          "frame_window_size": 9,
+          "statistic_name": "franka",
+          "transforms": [
+            {
+              "frame_stride": 2,
+              "name_mappings": {
+                "actions": [
+                  "actions"
+                ],
+                "observation.state": [
+                  "states"
+                ]
+              },
+              "parquet_keys": [
+                "observation.state",
+                "timestamp",
+                "actions",
+                "info",
+                "stats",
+                "action_masks"
+              ],
+              "type": "ProcessParquetInputs",
+              "video_keys": [
+                "observation.images.image",
+                "observation.images.wrist_image"
+              ]
+            },
+            {
+              "input_keys": [
+                "prompt",
+                "task_description",
+                "lang",
+                "instruction",
+                "instructions",
+                "text"
+              ],
+              "output_key": "prompt",
+              "remove_source_keys": true,
+              "type": "CanonicalizePrompt"
+            },
+            {
+              "input_key": "prompt",
+              "remove_input_key": true,
+              "tokenizer": {
+                "model_max_length": 512,
+                "model_path": "./checkpoints/Cosmos-Predict2.5-2B/tokenizer",
+                "type": "PretrainedTokenizer"
+              },
+              "type": "ProcessCosmos25Prompt"
+            },
+            {
+              "backend": "torch",
+              "height": 224,
+              "output_layout": "nchw",
+              "scale_divisor": 255.0,
+              "type": "ResizeImages",
+              "width": 224
+            },
+            {
+              "frame_stride": 1,
+              "keep_time_dim": true,
+              "key": "images",
+              "num_views": 2,
+              "type": "ConcatImagesHorizontally",
+              "views_first": true
+            },
+            {
+              "backend": "torch",
+              "interleave": true,
+              "keys": [
+                "states"
+              ],
+              "target_dims": {
+                "states": 16
+              },
+              "type": "SinCosKeys"
+            },
+            {
+              "action_dim": null,
+              "action_key": "action",
+              "action_norm_mask": [
+                true,
+                true,
+                true,
+                true,
+                true,
+                true,
+                false
+              ],
+              "action_norm_type": "min_max",
+              "clip_norm": false,
+              "norm_type": "min_max",
+              "normalization_epsilon": 0.0,
+              "output_dtype": "float16",
+              "preserve_input_dtype": true,
+              "state_dim": null,
+              "state_key": "proprio",
+              "state_norm_type": "none",
+              "type": "NormalizeStatesAndActions"
+            },
+            {
+              "action_dim": 8,
+              "type": "PadActionsAndActionMasks",
+              "valid_action_dim": 7
+            },
+            {
+              "action_dim": 8,
+              "action_horizon": 8,
+              "mark_all_action_steps_valid": true,
+              "state_dim": 16,
+              "type": "PrepareDiT4DiTInputs",
+              "valid_action_dim": 7
+            }
+          ],
+          "type": "ParquetDataset",
+          "use_delta": false,
+          "window_start_idx": 0
+        },
+        {
+          "action_key": "action",
+          "action_window_size": 8,
+          "data_root_path": "./datasets/libero_10_no_noops_lerobotv2.1",
+          "frame_window_size": 9,
+          "statistic_name": "franka",
+          "transforms": [
+            {
+              "frame_stride": 2,
+              "name_mappings": {
+                "actions": [
+                  "actions"
+                ],
+                "observation.state": [
+                  "states"
+                ]
+              },
+              "parquet_keys": [
+                "observation.state",
+                "timestamp",
+                "actions",
+                "info",
+                "stats",
+                "action_masks"
+              ],
+              "type": "ProcessParquetInputs",
+              "video_keys": [
+                "observation.images.image",
+                "observation.images.wrist_image"
+              ]
+            },
+            {
+              "input_keys": [
+                "prompt",
+                "task_description",
+                "lang",
+                "instruction",
+                "instructions",
+                "text"
+              ],
+              "output_key": "prompt",
+              "remove_source_keys": true,
+              "type": "CanonicalizePrompt"
+            },
+            {
+              "input_key": "prompt",
+              "remove_input_key": true,
+              "tokenizer": {
+                "model_max_length": 512,
+                "model_path": "./checkpoints/Cosmos-Predict2.5-2B/tokenizer",
+                "type": "PretrainedTokenizer"
+              },
+              "type": "ProcessCosmos25Prompt"
+            },
+            {
+              "backend": "torch",
+              "height": 224,
+              "output_layout": "nchw",
+              "scale_divisor": 255.0,
+              "type": "ResizeImages",
+              "width": 224
+            },
+            {
+              "frame_stride": 1,
+              "keep_time_dim": true,
+              "key": "images",
+              "num_views": 2,
+              "type": "ConcatImagesHorizontally",
+              "views_first": true
+            },
+            {
+              "backend": "torch",
+              "interleave": true,
+              "keys": [
+                "states"
+              ],
+              "target_dims": {
+                "states": 16
+              },
+              "type": "SinCosKeys"
+            },
+            {
+              "action_dim": null,
+              "action_key": "action",
+              "action_norm_mask": [
+                true,
+                true,
+                true,
+                true,
+                true,
+                true,
+                false
+              ],
+              "action_norm_type": "min_max",
+              "clip_norm": false,
+              "norm_type": "min_max",
+              "normalization_epsilon": 0.0,
+              "output_dtype": "float16",
+              "preserve_input_dtype": true,
+              "state_dim": null,
+              "state_key": "proprio",
+              "state_norm_type": "none",
+              "type": "NormalizeStatesAndActions"
+            },
+            {
+              "action_dim": 8,
+              "type": "PadActionsAndActionMasks",
+              "valid_action_dim": 7
+            },
+            {
+              "action_dim": 8,
+              "action_horizon": 8,
+              "mark_all_action_steps_valid": true,
+              "state_dim": 16,
+              "type": "PrepareDiT4DiTInputs",
+              "valid_action_dim": 7
+            }
+          ],
+          "type": "ParquetDataset",
+          "use_delta": false,
+          "window_start_idx": 0
+        }
+      ],
+      "name_mappings": {
+        "action": [
+          "action"
+        ],
+        "observation.state": [
+          "proprio"
+        ]
+      },
+      "reshuffle_each_epoch": true,
+      "sampling_weights": [
+        1.0,
+        1.0,
+        1.0,
+        1.0
+      ],
+      "seed": 42,
+      "shuffle": false,
+      "statistic_keys": [
+        "observation.state",
+        "timestamp",
+        "action"
+      ],
+      "statistic_name": "franka",
+      "type": "DistributedBalancedRepeatingDataset"
+    },
+    "per_device_batch_size": 8,
+    "per_device_num_workers": 4
+  }
+}

dit4dit_libero_all_full_finetune_bc071270ff2_bs256/config.yaml

@@ -0,0 +1,1082 @@
+_action_dim: 8
+_action_horizon: 8
+_action_norm_mask:
+- true
+- true
+- true
+- true
+- true
+- true
+- false
+_ckpt_root: ./checkpoints
+_cosmos_base_model: ./checkpoints/Cosmos-Predict2.5-2B
+_cosmos_revision: diffusers/base/post-trained
+_cosmos_tokenizer:
+  model_max_length: 512
+  model_path: ./checkpoints/Cosmos-Predict2.5-2B/tokenizer
+  type: PretrainedTokenizer
+_dit4dit_parquet_dataset:
+  action_key: action
+  action_window_size: 8
+  frame_window_size: 9
+  statistic_name: franka
+  transforms:
+  - frame_stride: 2
+    name_mappings:
+      actions:
+      - actions
+      observation.state:
+      - states
+    parquet_keys:
+    - observation.state
+    - timestamp
+    - actions
+    - info
+    - stats
+    - action_masks
+    type: ProcessParquetInputs
+    video_keys:
+    - observation.images.image
+    - observation.images.wrist_image
+  - input_keys:
+    - prompt
+    - task_description
+    - lang
+    - instruction
+    - instructions
+    - text
+    output_key: prompt
+    remove_source_keys: true
+    type: CanonicalizePrompt
+  - input_key: prompt
+    remove_input_key: true
+    tokenizer:
+      model_max_length: 512
+      model_path: ./checkpoints/Cosmos-Predict2.5-2B/tokenizer
+      type: PretrainedTokenizer
+    type: ProcessCosmos25Prompt
+  - backend: torch
+    height: 224
+    output_layout: nchw
+    scale_divisor: 255.0
+    type: ResizeImages
+    width: 224
+  - frame_stride: 1
+    keep_time_dim: true
+    key: images
+    num_views: 2
+    type: ConcatImagesHorizontally
+    views_first: true
+  - backend: torch
+    interleave: true
+    keys:
+    - states
+    target_dims:
+      states: 16
+    type: SinCosKeys
+  - action_dim: null
+    action_key: action
+    action_norm_mask:
+    - true
+    - true
+    - true
+    - true
+    - true
+    - true
+    - false
+    action_norm_type: min_max
+    clip_norm: false
+    norm_type: min_max
+    normalization_epsilon: 0.0
+    output_dtype: float16
+    preserve_input_dtype: true
+    state_dim: null
+    state_key: proprio
+    state_norm_type: none
+    type: NormalizeStatesAndActions
+  - action_dim: 8
+    type: PadActionsAndActionMasks
+    valid_action_dim: 7
+  - action_dim: 8
+    action_horizon: 8
+    mark_all_action_steps_valid: true
+    state_dim: 16
+    type: PrepareDiT4DiTInputs
+    valid_action_dim: 7
+  type: ParquetDataset
+  use_delta: false
+  window_start_idx: 0
+_dit4dit_train_transforms:
+- frame_stride: 2
+  name_mappings:
+    actions:
+    - actions
+    observation.state:
+    - states
+  parquet_keys:
+  - observation.state
+  - timestamp
+  - actions
+  - info
+  - stats
+  - action_masks
+  type: ProcessParquetInputs
+  video_keys:
+  - observation.images.image
+  - observation.images.wrist_image
+- input_keys:
+  - prompt
+  - task_description
+  - lang
+  - instruction
+  - instructions
+  - text
+  output_key: prompt
+  remove_source_keys: true
+  type: CanonicalizePrompt
+- input_key: prompt
+  remove_input_key: true
+  tokenizer:
+    model_max_length: 512
+    model_path: ./checkpoints/Cosmos-Predict2.5-2B/tokenizer
+    type: PretrainedTokenizer
+  type: ProcessCosmos25Prompt
+- backend: torch
+  height: 224
+  output_layout: nchw
+  scale_divisor: 255.0
+  type: ResizeImages
+  width: 224
+- frame_stride: 1
+  keep_time_dim: true
+  key: images
+  num_views: 2
+  type: ConcatImagesHorizontally
+  views_first: true
+- backend: torch
+  interleave: true
+  keys:
+  - states
+  target_dims:
+    states: 16
+  type: SinCosKeys
+- action_dim: null
+  action_key: action
+  action_norm_mask:
+  - true
+  - true
+  - true
+  - true
+  - true
+  - true
+  - false
+  action_norm_type: min_max
+  clip_norm: false
+  norm_type: min_max
+  normalization_epsilon: 0.0
+  output_dtype: float16
+  preserve_input_dtype: true
+  state_dim: null
+  state_key: proprio
+  state_norm_type: none
+  type: NormalizeStatesAndActions
+- action_dim: 8
+  type: PadActionsAndActionMasks
+  valid_action_dim: 7
+- action_dim: 8
+  action_horizon: 8
+  mark_all_action_steps_valid: true
+  state_dim: 16
+  type: PrepareDiT4DiTInputs
+  valid_action_dim: 7
+_frame_window_size: 9
+_image_frame_stride: 2
+_image_size: 224
+_libero_data_roots:
+- ./datasets/libero_object_no_noops_lerobotv2.1
+- ./datasets/libero_goal_no_noops_lerobotv2.1
+- ./datasets/libero_spatial_no_noops_lerobotv2.1
+- ./datasets/libero_10_no_noops_lerobotv2.1
+_official_dataset_statistics:
+  franka:
+    action:
+      mask:
+      - true
+      - true
+      - true
+      - true
+      - true
+      - true
+      - false
+      max:
+      - 0.9375
+      - 0.9375
+      - 0.9375
+      - 0.3557142913341522
+      - 0.375
+      - 0.375
+      - 1.0
+      mean:
+      - 0.07237596483901143
+      - 0.08987006871029735
+      - -0.10144743137061596
+      - -0.00045383188989944756
+      - 0.006273590726777911
+      - -0.003878799732774496
+      - 0.524486355483532
+      min:
+      - -0.9375
+      - -0.9375
+      - -0.9375
+      - -0.2582142949104309
+      - -0.375
+      - -0.3675000071525574
+      - 0.0
+      q01:
+      - -0.8785714507102966
+      - -0.8758928775787354
+      - -0.9375
+      - -0.1510714292526245
+      - -0.20678570866584778
+      - -0.2742857038974762
+      - 0.0
+      q99:
+      - 0.9375
+      - 0.9107142686843872
+      - 0.9375
+      - 0.20357142388820648
+      - 0.26357144117355347
+      - 0.375
+      - 1.0
+      std:
+      - 0.3498823308902479
+      - 0.37794140366375184
+      - 0.460084266976933
+      - 0.0403885784928603
+      - 0.06616144248501059
+      - 0.07763074391911857
+      - 0.4994683356809767
+    num_trajectories: 1693
+    num_transitions: 273465
+    state:
+      max:
+      - 0.21031762659549713
+      - 0.39128610491752625
+      - 1.3660105466842651
+      - 3.6714255809783936
+      - 3.560650587081909
+      - 1.386339545249939
+      - 0.04233968257904053
+      - 0.0013633022317662835
+      mean:
+      - -0.04889854742214084
+      - 0.03689368185587227
+      - 0.7890402488410473
+      - 2.9771945476531982
+      - -0.1417286954820156
+      - -0.11769362539052963
+      - 0.026436020154505968
+      - -0.02665513101965189
+      min:
+      - -0.4828203022480011
+      - -0.3255046010017395
+      - 0.008128180168569088
+      - 0.35277295112609863
+      - -3.641430377960205
+      - -1.842738389968872
+      - -0.0013586411951109767
+      - -0.042040832340717316
+      q01:
+      - -0.42401049643754957
+      - -0.2838300323486328
+      - 0.009925739830359817
+      - 1.3085840785503386
+      - -2.886677579879761
+      - -1.1599004411697387
+      - 0.001503719249740243
+      - -0.040336399003863335
+      q99:
+      - 0.1530261474847791
+      - 0.3629165390133857
+      - 1.2910678112506866
+      - 3.303542451858519
+      - 2.7496529006957933
+      - 0.6893712210655194
+      - 0.040610933862626555
+      - -0.0015016929572448147
+      std:
+      - 0.10639013941746686
+      - 0.15115733130675715
+      - 0.38406895599530033
+      - 0.3530238395244304
+      - 0.8227341427331599
+      - 0.32357567121520087
+      - 0.014583991652936385
+      - 0.014467005007200339
+_official_dit4dit_ckpt: ./checkpoints/dit4dit-model/dit4dit_libero/final_model/pytorch_model.pt
+_ori_action_dim: 7
+_prompt_input_keys:
+- prompt
+- task_description
+- lang
+- instruction
+- instructions
+- text
+_state_dim: 16
+eval:
+  dataset:
+    img_buffer_len: 1
+    require_lang_tokens: true
+    transforms:
+    - input_keys:
+      - prompt
+      - task_description
+      - lang
+      - instruction
+      - instructions
+      - text
+      output_key: prompt
+      remove_source_keys: true
+      type: CanonicalizePrompt
+    - input_key: prompt
+      remove_input_key: true
+      tokenizer:
+        model_max_length: 512
+        model_path: ./checkpoints/Cosmos-Predict2.5-2B/tokenizer
+        type: PretrainedTokenizer
+      type: ProcessCosmos25Prompt
+    - image_size: 224
+      img_keys:
+      - agentview_image
+      - robot0_eye_in_hand_image
+      type: ProcessDiT4DiTLiberoEvalInputs
+    - gripper_key: robot0_gripper_qpos
+      norm_type: none
+      out_key: states
+      pos_key: robot0_eef_pos
+      quat_key: robot0_eef_quat
+      state_dim: null
+      type: LiberoProprioFromInputs
+    - interleave: true
+      keys:
+      - states
+      target_dims:
+        states: 16
+      type: SinCosKeys
+    - image_key: pixel_values
+      require_actions: false
+      state_dim: 16
+      type: PrepareDiT4DiTInputs
+    type: LiberoParquetEvalDataset
+  denormalize_action:
+    action_dim: 7
+    action_norm_mask:
+    - true
+    - true
+    - true
+    - true
+    - true
+    - true
+    - false
+    clip_normalized_action: true
+    invert_gripper_action: true
+    norm_type: min_max
+    normalize_gripper_action: true
+    type: DenormalizeLiberoAction
+  enable_mixed_precision_training: false
+  eval_chunk_size: 8
+  mixed_precision_dtype: bf16
+  model_family: dit4dit
+  norm_stats_key: franka
+  num_steps_wait: 10
+  num_trials_per_task: 50
+  resize_size: 224
+  seed: 7
+  task_suite_name:
+  - libero_spatial
+  - libero_object
+  - libero_goal
+  - libero_10
+  truncate_action_chunk_at_horizon: true
+  type: LiberoEvalRunner
+inference_model:
+  freeze_vlm_backbone: false
+  name_mapping: null
+  pretrained_name_or_path: null
+  repeated_diffusion_steps: 4
+  strict_mapping: false
+  type: DiT4DiTVLA
+  vla_head:
+    action_dim: 8
+    action_horizon: 8
+    action_model_type: DiT-B
+    add_pos_embed: true
+    diffusion_model_cfg:
+      cross_attention_dim: 2048
+      dropout: 0.2
+      final_dropout: true
+      interleave_self_attention: true
+      norm_type: ada_norm
+      num_layers: 16
+      output_dim: 2560
+      positional_embeddings: null
+    future_action_window_size: 7
+    hidden_size: 2560
+    max_seq_len: 1024
+    noise_beta_alpha: 1.5
+    noise_beta_beta: 1.0
+    noise_s: 0.999
+    num_inference_timesteps: 4
+    num_timestep_buckets: 1000
+    ori_action_dim: 7
+    state_dim: 16
+    type: DiT4DiTActionHead
+  vlm_backbone:
+    base_model: ./checkpoints/Cosmos-Predict2.5-2B
+    conditional_frame_timestep: 0.0001
+    detach_hidden_states: true
+    extract_layer: 17
+    fixed_seed: null
+    flow_matching_high_sigma_min: null
+    flow_matching_high_sigma_ratio: null
+    flow_matching_time_distribution: uniform
+    frozen_submodules:
+    - text_encoder
+    - vae
+    fsdp_min_num_params: 0
+    future_loss_type: flow_matching
+    local_files_only: true
+    max_sequence_length: 512
+    num_frames_out: 5
+    num_inference_steps: 1
+    revision: diffusers/base/post-trained
+    split_future_frames: true
+    torch_dtype: bf16
+    trainable: true
+    type: Cosmos25Backbone
+model:
+  freeze_vlm_backbone: false
+  name_mapping:
+    vla_head: action_model
+    vlm_backbone.text_encoder: backbone_interface.extractor.text_encoder
+    vlm_backbone.transformer: backbone_interface.extractor.transformer
+    vlm_backbone.vae: backbone_interface.extractor.vae
+  pretrained_name_or_path: null
+  repeated_diffusion_steps: 4
+  strict_mapping: false
+  type: DiT4DiTVLA
+  vla_head:
+    action_dim: 8
+    action_horizon: 8
+    action_model_type: DiT-B
+    add_pos_embed: true
+    diffusion_model_cfg:
+      cross_attention_dim: 2048
+      dropout: 0.2
+      final_dropout: true
+      interleave_self_attention: true
+      norm_type: ada_norm
+      num_layers: 16
+      output_dim: 2560
+      positional_embeddings: null
+    future_action_window_size: 7
+    hidden_size: 2560
+    max_seq_len: 1024
+    noise_beta_alpha: 1.5
+    noise_beta_beta: 1.0
+    noise_s: 0.999
+    num_inference_timesteps: 4
+    num_timestep_buckets: 1000
+    ori_action_dim: 7
+    state_dim: 16
+    type: DiT4DiTActionHead
+  vlm_backbone:
+    base_model: ./checkpoints/Cosmos-Predict2.5-2B
+    conditional_frame_timestep: 0.0001
+    detach_hidden_states: true
+    extract_layer: 17
+    fixed_seed: null
+    flow_matching_high_sigma_min: null
+    flow_matching_high_sigma_ratio: null
+    flow_matching_time_distribution: uniform
+    frozen_submodules:
+    - text_encoder
+    - vae
+    fsdp_min_num_params: 0
+    future_loss_type: flow_matching
+    local_files_only: true
+    max_sequence_length: 512
+    num_frames_out: 5
+    num_inference_steps: 1
+    revision: diffusers/base/post-trained
+    split_future_frames: true
+    torch_dtype: bf16
+    trainable: true
+    type: Cosmos25Backbone
+official_eval_task_suites:
+- libero_spatial
+- libero_object
+- libero_goal
+- libero_10
+runner:
+  change_key_name: false
+  collator:
+    keys:
+    - states
+    - timestamp
+    - images
+    - img_masks
+    - actions
+    - action_masks
+    - frame_masks
+    - lang_tokens
+    - lang_masks
+    meta_keys:
+    - info
+    - stats
+    type: DictCollator
+  enable_gradient_checkpointing: false
+  enable_mixed_precision_training: true
+  grad_accumulation_steps: 2
+  lr_scheduler:
+    min_lr: 5.0e-07
+    type: cosine_with_min_lr
+    warmup_steps: 10000
+  max_grad_norm: 1.0
+  max_keep_ckpts: 2
+  max_steps: 160000
+  metric:
+    active_trackers:
+    - jsonl
+    - wandb
+    grad_accumulation_steps: 1
+    run_dir: work_dirs
+    type: VLAMetric
+    window_size: 1
+  mixed_precision_dtype: bf16
+  optimizer:
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    lr: 3.0e-05
+    paramwise_learning_rate:
+      vla_head: 0.0001
+      vlm_backbone.transformer: 0.0001
+    type: AdamW
+    weight_decay: 1.0e-08
+  pre_fsdp_param_dtype: fp32
+  reduce_in_full_precision: false
+  sampler: null
+  save_iter_interval: 40000
+  sharding_strategy: shard-grad-op
+  tokenizer:
+    model_max_length: 512
+    model_path: ./checkpoints/Cosmos-Predict2.5-2B/tokenizer
+    type: PretrainedTokenizer
+  type: FSDPTrainRunner
+seed: 42
+train_dataloader:
+  dataset:
+    dataset_statistics:
+      franka:
+        action:
+          mask:
+          - true
+          - true
+          - true
+          - true
+          - true
+          - true
+          - false
+          max:
+          - 0.9375
+          - 0.9375
+          - 0.9375
+          - 0.3557142913341522
+          - 0.375
+          - 0.375
+          - 1.0
+          mean:
+          - 0.07237596483901143
+          - 0.08987006871029735
+          - -0.10144743137061596
+          - -0.00045383188989944756
+          - 0.006273590726777911
+          - -0.003878799732774496
+          - 0.524486355483532
+          min:
+          - -0.9375
+          - -0.9375
+          - -0.9375
+          - -0.2582142949104309
+          - -0.375
+          - -0.3675000071525574
+          - 0.0
+          q01:
+          - -0.8785714507102966
+          - -0.8758928775787354
+          - -0.9375
+          - -0.1510714292526245
+          - -0.20678570866584778
+          - -0.2742857038974762
+          - 0.0
+          q99:
+          - 0.9375
+          - 0.9107142686843872
+          - 0.9375
+          - 0.20357142388820648
+          - 0.26357144117355347
+          - 0.375
+          - 1.0
+          std:
+          - 0.3498823308902479
+          - 0.37794140366375184
+          - 0.460084266976933
+          - 0.0403885784928603
+          - 0.06616144248501059
+          - 0.07763074391911857
+          - 0.4994683356809767
+        num_trajectories: 1693
+        num_transitions: 273465
+        state:
+          max:
+          - 0.21031762659549713
+          - 0.39128610491752625
+          - 1.3660105466842651
+          - 3.6714255809783936
+          - 3.560650587081909
+          - 1.386339545249939
+          - 0.04233968257904053
+          - 0.0013633022317662835
+          mean:
+          - -0.04889854742214084
+          - 0.03689368185587227
+          - 0.7890402488410473
+          - 2.9771945476531982
+          - -0.1417286954820156
+          - -0.11769362539052963
+          - 0.026436020154505968
+          - -0.02665513101965189
+          min:
+          - -0.4828203022480011
+          - -0.3255046010017395
+          - 0.008128180168569088
+          - 0.35277295112609863
+          - -3.641430377960205
+          - -1.842738389968872
+          - -0.0013586411951109767
+          - -0.042040832340717316
+          q01:
+          - -0.42401049643754957
+          - -0.2838300323486328
+          - 0.009925739830359817
+          - 1.3085840785503386
+          - -2.886677579879761
+          - -1.1599004411697387
+          - 0.001503719249740243
+          - -0.040336399003863335
+          q99:
+          - 0.1530261474847791
+          - 0.3629165390133857
+          - 1.2910678112506866
+          - 3.303542451858519
+          - 2.7496529006957933
+          - 0.6893712210655194
+          - 0.040610933862626555
+          - -0.0015016929572448147
+          std:
+          - 0.10639013941746686
+          - 0.15115733130675715
+          - 0.38406895599530033
+          - 0.3530238395244304
+          - 0.8227341427331599
+          - 0.32357567121520087
+          - 0.014583991652936385
+          - 0.014467005007200339
+    datasets:
+    - action_key: action
+      action_window_size: 8
+      data_root_path: ./datasets/libero_object_no_noops_lerobotv2.1
+      frame_window_size: 9
+      statistic_name: franka
+      transforms:
+      - frame_stride: 2
+        name_mappings:
+          actions:
+          - actions
+          observation.state:
+          - states
+        parquet_keys:
+        - observation.state
+        - timestamp
+        - actions
+        - info
+        - stats
+        - action_masks
+        type: ProcessParquetInputs
+        video_keys:
+        - observation.images.image
+        - observation.images.wrist_image
+      - input_keys:
+        - prompt
+        - task_description
+        - lang
+        - instruction
+        - instructions
+        - text
+        output_key: prompt
+        remove_source_keys: true
+        type: CanonicalizePrompt
+      - input_key: prompt
+        remove_input_key: true
+        tokenizer:
+          model_max_length: 512
+          model_path: ./checkpoints/Cosmos-Predict2.5-2B/tokenizer
+          type: PretrainedTokenizer
+        type: ProcessCosmos25Prompt
+      - backend: torch
+        height: 224
+        output_layout: nchw
+        scale_divisor: 255.0
+        type: ResizeImages
+        width: 224
+      - frame_stride: 1
+        keep_time_dim: true
+        key: images
+        num_views: 2
+        type: ConcatImagesHorizontally
+        views_first: true
+      - backend: torch
+        interleave: true
+        keys:
+        - states
+        target_dims:
+          states: 16
+        type: SinCosKeys
+      - action_dim: null
+        action_key: action
+        action_norm_mask:
+        - true
+        - true
+        - true
+        - true
+        - true
+        - true
+        - false
+        action_norm_type: min_max
+        clip_norm: false
+        norm_type: min_max
+        normalization_epsilon: 0.0
+        output_dtype: float16
+        preserve_input_dtype: true
+        state_dim: null
+        state_key: proprio
+        state_norm_type: none
+        type: NormalizeStatesAndActions
+      - action_dim: 8
+        type: PadActionsAndActionMasks
+        valid_action_dim: 7
+      - action_dim: 8
+        action_horizon: 8
+        mark_all_action_steps_valid: true
+        state_dim: 16
+        type: PrepareDiT4DiTInputs
+        valid_action_dim: 7
+      type: ParquetDataset
+      use_delta: false
+      window_start_idx: 0
+    - action_key: action
+      action_window_size: 8
+      data_root_path: ./datasets/libero_goal_no_noops_lerobotv2.1
+      frame_window_size: 9
+      statistic_name: franka
+      transforms:
+      - frame_stride: 2
+        name_mappings:
+          actions:
+          - actions
+          observation.state:
+          - states
+        parquet_keys:
+        - observation.state
+        - timestamp
+        - actions
+        - info
+        - stats
+        - action_masks
+        type: ProcessParquetInputs
+        video_keys:
+        - observation.images.image
+        - observation.images.wrist_image
+      - input_keys:
+        - prompt
+        - task_description
+        - lang
+        - instruction
+        - instructions
+        - text
+        output_key: prompt
+        remove_source_keys: true
+        type: CanonicalizePrompt
+      - input_key: prompt
+        remove_input_key: true
+        tokenizer:
+          model_max_length: 512
+          model_path: ./checkpoints/Cosmos-Predict2.5-2B/tokenizer
+          type: PretrainedTokenizer
+        type: ProcessCosmos25Prompt
+      - backend: torch
+        height: 224
+        output_layout: nchw
+        scale_divisor: 255.0
+        type: ResizeImages
+        width: 224
+      - frame_stride: 1
+        keep_time_dim: true
+        key: images
+        num_views: 2
+        type: ConcatImagesHorizontally
+        views_first: true
+      - backend: torch
+        interleave: true
+        keys:
+        - states
+        target_dims:
+          states: 16
+        type: SinCosKeys
+      - action_dim: null
+        action_key: action
+        action_norm_mask:
+        - true
+        - true
+        - true
+        - true
+        - true
+        - true
+        - false
+        action_norm_type: min_max
+        clip_norm: false
+        norm_type: min_max
+        normalization_epsilon: 0.0
+        output_dtype: float16
+        preserve_input_dtype: true
+        state_dim: null
+        state_key: proprio
+        state_norm_type: none
+        type: NormalizeStatesAndActions
+      - action_dim: 8
+        type: PadActionsAndActionMasks
+        valid_action_dim: 7
+      - action_dim: 8
+        action_horizon: 8
+        mark_all_action_steps_valid: true
+        state_dim: 16
+        type: PrepareDiT4DiTInputs
+        valid_action_dim: 7
+      type: ParquetDataset
+      use_delta: false
+      window_start_idx: 0
+    - action_key: action
+      action_window_size: 8
+      data_root_path: ./datasets/libero_spatial_no_noops_lerobotv2.1
+      frame_window_size: 9
+      statistic_name: franka
+      transforms:
+      - frame_stride: 2
+        name_mappings:
+          actions:
+          - actions
+          observation.state:
+          - states
+        parquet_keys:
+        - observation.state
+        - timestamp
+        - actions
+        - info
+        - stats
+        - action_masks
+        type: ProcessParquetInputs
+        video_keys:
+        - observation.images.image
+        - observation.images.wrist_image
+      - input_keys:
+        - prompt
+        - task_description
+        - lang
+        - instruction
+        - instructions
+        - text
+        output_key: prompt
+        remove_source_keys: true
+        type: CanonicalizePrompt
+      - input_key: prompt
+        remove_input_key: true
+        tokenizer:
+          model_max_length: 512
+          model_path: ./checkpoints/Cosmos-Predict2.5-2B/tokenizer
+          type: PretrainedTokenizer
+        type: ProcessCosmos25Prompt
+      - backend: torch
+        height: 224
+        output_layout: nchw
+        scale_divisor: 255.0
+        type: ResizeImages
+        width: 224
+      - frame_stride: 1
+        keep_time_dim: true
+        key: images
+        num_views: 2
+        type: ConcatImagesHorizontally
+        views_first: true
+      - backend: torch
+        interleave: true
+        keys:
+        - states
+        target_dims:
+          states: 16
+        type: SinCosKeys
+      - action_dim: null
+        action_key: action
+        action_norm_mask:
+        - true
+        - true
+        - true
+        - true
+        - true
+        - true
+        - false
+        action_norm_type: min_max
+        clip_norm: false
+        norm_type: min_max
+        normalization_epsilon: 0.0
+        output_dtype: float16
+        preserve_input_dtype: true
+        state_dim: null
+        state_key: proprio
+        state_norm_type: none
+        type: NormalizeStatesAndActions
+      - action_dim: 8
+        type: PadActionsAndActionMasks
+        valid_action_dim: 7
+      - action_dim: 8
+        action_horizon: 8
+        mark_all_action_steps_valid: true
+        state_dim: 16
+        type: PrepareDiT4DiTInputs
+        valid_action_dim: 7
+      type: ParquetDataset
+      use_delta: false
+      window_start_idx: 0
+    - action_key: action
+      action_window_size: 8
+      data_root_path: ./datasets/libero_10_no_noops_lerobotv2.1
+      frame_window_size: 9
+      statistic_name: franka
+      transforms:
+      - frame_stride: 2
+        name_mappings:
+          actions:
+          - actions
+          observation.state:
+          - states
+        parquet_keys:
+        - observation.state
+        - timestamp
+        - actions
+        - info
+        - stats
+        - action_masks
+        type: ProcessParquetInputs
+        video_keys:
+        - observation.images.image
+        - observation.images.wrist_image
+      - input_keys:
+        - prompt
+        - task_description
+        - lang
+        - instruction
+        - instructions
+        - text
+        output_key: prompt
+        remove_source_keys: true
+        type: CanonicalizePrompt
+      - input_key: prompt
+        remove_input_key: true
+        tokenizer:
+          model_max_length: 512
+          model_path: ./checkpoints/Cosmos-Predict2.5-2B/tokenizer
+          type: PretrainedTokenizer
+        type: ProcessCosmos25Prompt
+      - backend: torch
+        height: 224
+        output_layout: nchw
+        scale_divisor: 255.0
+        type: ResizeImages
+        width: 224
+      - frame_stride: 1
+        keep_time_dim: true
+        key: images
+        num_views: 2
+        type: ConcatImagesHorizontally
+        views_first: true
+      - backend: torch
+        interleave: true
+        keys:
+        - states
+        target_dims:
+          states: 16
+        type: SinCosKeys
+      - action_dim: null
+        action_key: action
+        action_norm_mask:
+        - true
+        - true
+        - true
+        - true
+        - true
+        - true
+        - false
+        action_norm_type: min_max
+        clip_norm: false
+        norm_type: min_max
+        normalization_epsilon: 0.0
+        output_dtype: float16
+        preserve_input_dtype: true
+        state_dim: null
+        state_key: proprio
+        state_norm_type: none
+        type: NormalizeStatesAndActions
+      - action_dim: 8
+        type: PadActionsAndActionMasks
+        valid_action_dim: 7
+      - action_dim: 8
+        action_horizon: 8
+        mark_all_action_steps_valid: true
+        state_dim: 16
+        type: PrepareDiT4DiTInputs
+        valid_action_dim: 7
+      type: ParquetDataset
+      use_delta: false
+      window_start_idx: 0
+    name_mappings:
+      action:
+      - action
+      observation.state:
+      - proprio
+    reshuffle_each_epoch: true
+    sampling_weights:
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    seed: 42
+    shuffle: false
+    statistic_keys:
+    - observation.state
+    - timestamp
+    - action
+    statistic_name: franka
+    type: DistributedBalancedRepeatingDataset
+  per_device_batch_size: 8
+  per_device_num_workers: 4

dit4dit_libero_all_full_finetune_bc071270ff2_bs256/dataset_statistics.json

@@ -0,0 +1,133 @@
+{
+  "franka": {
+    "action": {
+      "mean": [
+        0.07237596483901143,
+        0.08987006871029735,
+        -0.10144743137061596,
+        -0.00045383188989944756,
+        0.006273590726777911,
+        -0.003878799732774496,
+        0.524486355483532
+      ],
+      "std": [
+        0.3498823308902479,
+        0.37794140366375184,
+        0.460084266976933,
+        0.0403885784928603,
+        0.06616144248501059,
+        0.07763074391911857,
+        0.4994683356809767
+      ],
+      "max": [
+        0.9375,
+        0.9375,
+        0.9375,
+        0.3557142913341522,
+        0.375,
+        0.375,
+        1.0
+      ],
+      "min": [
+        -0.9375,
+        -0.9375,
+        -0.9375,
+        -0.2582142949104309,
+        -0.375,
+        -0.3675000071525574,
+        0.0
+      ],
+      "q01": [
+        -0.8785714507102966,
+        -0.8758928775787354,
+        -0.9375,
+        -0.1510714292526245,
+        -0.20678570866584778,
+        -0.2742857038974762,
+        0.0
+      ],
+      "q99": [
+        0.9375,
+        0.9107142686843872,
+        0.9375,
+        0.20357142388820648,
+        0.26357144117355347,
+        0.375,
+        1.0
+      ],
+      "mask": [
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        false
+      ]
+    },
+    "state": {
+      "mean": [
+        -0.04889854742214084,
+        0.03689368185587227,
+        0.7890402488410473,
+        2.9771945476531982,
+        -0.1417286954820156,
+        -0.11769362539052963,
+        0.026436020154505968,
+        -0.02665513101965189
+      ],
+      "std": [
+        0.10639013941746686,
+        0.15115733130675715,
+        0.38406895599530033,
+        0.3530238395244304,
+        0.8227341427331599,
+        0.32357567121520087,
+        0.014583991652936385,
+        0.014467005007200339
+      ],
+      "max": [
+        0.21031762659549713,
+        0.39128610491752625,
+        1.3660105466842651,
+        3.6714255809783936,
+        3.560650587081909,
+        1.386339545249939,
+        0.04233968257904053,
+        0.0013633022317662835
+      ],
+      "min": [
+        -0.4828203022480011,
+        -0.3255046010017395,
+        0.008128180168569088,
+        0.35277295112609863,
+        -3.641430377960205,
+        -1.842738389968872,
+        -0.0013586411951109767,
+        -0.042040832340717316
+      ],
+      "q01": [
+        -0.42401049643754957,
+        -0.2838300323486328,
+        0.009925739830359817,
+        1.3085840785503386,
+        -2.886677579879761,
+        -1.1599004411697387,
+        0.001503719249740243,
+        -0.040336399003863335
+      ],
+      "q99": [
+        0.1530261474847791,
+        0.3629165390133857,
+        1.2910678112506866,
+        3.303542451858519,
+        2.7496529006957933,
+        0.6893712210655194,
+        0.040610933862626555,
+        -0.0015016929572448147
+      ]
+    },
+    "num_transitions": 273465,
+    "num_trajectories": 1693
+  }
+}

dit4dit_libero_all_full_finetune_bc071270ff2_bs256/dit4dit_libero_all_full_finetune_2026_08_12_20_51_03.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0895afd366cddc06686303d4b2d7fb8cfe96621b6b77c13596b7e7467b50f58a
+size 61119743

dit4dit_libero_all_full_finetune_bc071270ff2_bs256/run-metrics.jsonl

@@ -0,0 +1 @@
+{"hparams": "{'_ckpt_root': './checkpoints', '_cosmos_base_model': './checkpoints/Cosmos-Predict2.5-2B', '_cosmos_revision': 'diffusers/base/post-trained', '_cosmos_tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, '_official_dit4dit_ckpt': './checkpoints/dit4dit-model/dit4dit_libero/final_model/pytorch_model.pt', '_official_dataset_statistics': {'franka': {'action': {'mean': [0.07237596483901143, 0.08987006871029735, -0.10144743137061596, -0.00045383188989944756, 0.006273590726777911, -0.003878799732774496, 0.524486355483532], 'std': [0.3498823308902479, 0.37794140366375184, 0.460084266976933, 0.0403885784928603, 0.06616144248501059, 0.07763074391911857, 0.4994683356809767], 'max': [0.9375, 0.9375, 0.9375, 0.3557142913341522, 0.375, 0.375, 1.0], 'min': [-0.9375, -0.9375, -0.9375, -0.2582142949104309, -0.375, -0.3675000071525574, 0.0], 'q01': [-0.8785714507102966, -0.8758928775787354, -0.9375, -0.1510714292526245, -0.20678570866584778, -0.2742857038974762, 0.0], 'q99': [0.9375, 0.9107142686843872, 0.9375, 0.20357142388820648, 0.26357144117355347, 0.375, 1.0], 'mask': [True, True, True, True, True, True, False]}, 'state': {'mean': [-0.04889854742214084, 0.03689368185587227, 0.7890402488410473, 2.9771945476531982, -0.1417286954820156, -0.11769362539052963, 0.026436020154505968, -0.02665513101965189], 'std': [0.10639013941746686, 0.15115733130675715, 0.38406895599530033, 0.3530238395244304, 0.8227341427331599, 0.32357567121520087, 0.014583991652936385, 0.014467005007200339], 'max': [0.21031762659549713, 0.39128610491752625, 1.3660105466842651, 3.6714255809783936, 3.560650587081909, 1.386339545249939, 0.04233968257904053, 0.0013633022317662835], 'min': [-0.4828203022480011, -0.3255046010017395, 0.008128180168569088, 0.35277295112609863, -3.641430377960205, -1.842738389968872, -0.0013586411951109767, -0.042040832340717316], 'q01': [-0.42401049643754957, -0.2838300323486328, 0.009925739830359817, 1.3085840785503386, -2.886677579879761, -1.1599004411697387, 0.001503719249740243, -0.040336399003863335], 'q99': [0.1530261474847791, 0.3629165390133857, 1.2910678112506866, 3.303542451858519, 2.7496529006957933, 0.6893712210655194, 0.040610933862626555, -0.0015016929572448147]}, 'num_transitions': 273465, 'num_trajectories': 1693}}, '_action_dim': 8, '_ori_action_dim': 7, '_state_dim': 16, '_action_horizon': 8, '_frame_window_size': 9, '_image_frame_stride': 2, '_image_size': 224, '_action_norm_mask': [True, True, True, True, True, True, False], '_libero_data_roots': ['./datasets/libero_object_no_noops_lerobotv2.1', './datasets/libero_goal_no_noops_lerobotv2.1', './datasets/libero_spatial_no_noops_lerobotv2.1', './datasets/libero_10_no_noops_lerobotv2.1'], '_prompt_input_keys': ['prompt', 'task_description', 'lang', 'instruction', 'instructions', 'text'], 'seed': 42, 'official_eval_task_suites': ['libero_spatial', 'libero_object', 'libero_goal', 'libero_10'], 'model': {'type': 'DiT4DiTVLA', 'pretrained_name_or_path': None, 'name_mapping': {'vlm_backbone.text_encoder': 'backbone_interface.extractor.text_encoder', 'vlm_backbone.transformer': 'backbone_interface.extractor.transformer', 'vlm_backbone.vae': 'backbone_interface.extractor.vae', 'vla_head': 'action_model'}, 'strict_mapping': False, 'repeated_diffusion_steps': 4, 'vlm_backbone': {'type': 'Cosmos25Backbone', 'base_model': './checkpoints/Cosmos-Predict2.5-2B', 'revision': 'diffusers/base/post-trained', 'torch_dtype': 'bf16', 'local_files_only': True, 'extract_layer': 17, 'max_sequence_length': 512, 'trainable': True, 'frozen_submodules': ['text_encoder', 'vae'], 'split_future_frames': True, 'num_frames_out': 5, 'fixed_seed': None, 'num_inference_steps': 1, 'conditional_frame_timestep': 0.0001, 'future_loss_type': 'flow_matching', 'detach_hidden_states': True, 'flow_matching_time_distribution': 'uniform', 'flow_matching_high_sigma_ratio': None, 'flow_matching_high_sigma_min': None, 'fsdp_min_num_params': 0}, 'vla_head': {'type': 'DiT4DiTActionHead', 'action_model_type': 'DiT-B', 'hidden_size': 2560, 'add_pos_embed': True, 'max_seq_len': 1024, 'action_dim': 8, 'ori_action_dim': 7, 'state_dim': 16, 'future_action_window_size': 7, 'action_horizon': 8, 'noise_beta_alpha': 1.5, 'noise_beta_beta': 1.0, 'noise_s': 0.999, 'num_timestep_buckets': 1000, 'num_inference_timesteps': 4, 'diffusion_model_cfg': {'cross_attention_dim': 2048, 'dropout': 0.2, 'final_dropout': True, 'interleave_self_attention': True, 'norm_type': 'ada_norm', 'num_layers': 16, 'output_dim': 2560, 'positional_embeddings': None}}, 'freeze_vlm_backbone': False}, 'inference_model': {'type': 'DiT4DiTVLA', 'pretrained_name_or_path': None, 'name_mapping': None, 'strict_mapping': False, 'repeated_diffusion_steps': 4, 'vlm_backbone': {'type': 'Cosmos25Backbone', 'base_model': './checkpoints/Cosmos-Predict2.5-2B', 'revision': 'diffusers/base/post-trained', 'torch_dtype': 'bf16', 'local_files_only': True, 'extract_layer': 17, 'max_sequence_length': 512, 'trainable': True, 'frozen_submodules': ['text_encoder', 'vae'], 'split_future_frames': True, 'num_frames_out': 5, 'fixed_seed': None, 'num_inference_steps': 1, 'conditional_frame_timestep': 0.0001, 'future_loss_type': 'flow_matching', 'detach_hidden_states': True, 'flow_matching_time_distribution': 'uniform', 'flow_matching_high_sigma_ratio': None, 'flow_matching_high_sigma_min': None, 'fsdp_min_num_params': 0}, 'vla_head': {'type': 'DiT4DiTActionHead', 'action_model_type': 'DiT-B', 'hidden_size': 2560, 'add_pos_embed': True, 'max_seq_len': 1024, 'action_dim': 8, 'ori_action_dim': 7, 'state_dim': 16, 'future_action_window_size': 7, 'action_horizon': 8, 'noise_beta_alpha': 1.5, 'noise_beta_beta': 1.0, 'noise_s': 0.999, 'num_timestep_buckets': 1000, 'num_inference_timesteps': 4, 'diffusion_model_cfg': {'cross_attention_dim': 2048, 'dropout': 0.2, 'final_dropout': True, 'interleave_self_attention': True, 'norm_type': 'ada_norm', 'num_layers': 16, 'output_dim': 2560, 'positional_embeddings': None}}, 'freeze_vlm_backbone': False}, '_dit4dit_train_transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'frame_stride': 2}, {'type': 'CanonicalizePrompt', 'output_key': 'prompt', 'input_keys': ['prompt', 'task_description', 'lang', 'instruction', 'instructions', 'text'], 'remove_source_keys': True}, {'type': 'ProcessCosmos25Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, 'input_key': 'prompt', 'remove_input_key': True}, {'type': 'ResizeImages', 'height': 224, 'width': 224, 'backend': 'torch', 'scale_divisor': 255.0, 'output_layout': 'nchw'}, {'type': 'ConcatImagesHorizontally', 'key': 'images', 'num_views': 2, 'frame_stride': 1, 'views_first': True, 'keep_time_dim': True}, {'type': 'SinCosKeys', 'keys': ['states'], 'target_dims': {'states': 16}, 'interleave': True, 'backend': 'torch'}, {'type': 'NormalizeStatesAndActions', 'state_key': 'proprio', 'action_key': 'action', 'action_dim': None, 'state_dim': None, 'state_norm_type': 'none', 'action_norm_type': 'min_max', 'norm_type': 'min_max', 'action_norm_mask': [True, True, True, True, True, True, False], 'clip_norm': False, 'normalization_epsilon': 0.0, 'preserve_input_dtype': True, 'output_dtype': 'float16'}, {'type': 'PadActionsAndActionMasks', 'action_dim': 8, 'valid_action_dim': 7}, {'type': 'PrepareDiT4DiTInputs', 'state_dim': 16, 'action_horizon': 8, 'action_dim': 8, 'valid_action_dim': 7, 'mark_all_action_steps_valid': True}], '_dit4dit_parquet_dataset': {'type': 'ParquetDataset', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'frame_stride': 2}, {'type': 'CanonicalizePrompt', 'output_key': 'prompt', 'input_keys': ['prompt', 'task_description', 'lang', 'instruction', 'instructions', 'text'], 'remove_source_keys': True}, {'type': 'ProcessCosmos25Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, 'input_key': 'prompt', 'remove_input_key': True}, {'type': 'ResizeImages', 'height': 224, 'width': 224, 'backend': 'torch', 'scale_divisor': 255.0, 'output_layout': 'nchw'}, {'type': 'ConcatImagesHorizontally', 'key': 'images', 'num_views': 2, 'frame_stride': 1, 'views_first': True, 'keep_time_dim': True}, {'type': 'SinCosKeys', 'keys': ['states'], 'target_dims': {'states': 16}, 'interleave': True, 'backend': 'torch'}, {'type': 'NormalizeStatesAndActions', 'state_key': 'proprio', 'action_key': 'action', 'action_dim': None, 'state_dim': None, 'state_norm_type': 'none', 'action_norm_type': 'min_max', 'norm_type': 'min_max', 'action_norm_mask': [True, True, True, True, True, True, False], 'clip_norm': False, 'normalization_epsilon': 0.0, 'preserve_input_dtype': True, 'output_dtype': 'float16'}, {'type': 'PadActionsAndActionMasks', 'action_dim': 8, 'valid_action_dim': 7}, {'type': 'PrepareDiT4DiTInputs', 'state_dim': 16, 'action_horizon': 8, 'action_dim': 8, 'valid_action_dim': 7, 'mark_all_action_steps_valid': True}], 'action_window_size': 8, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'franka', 'window_start_idx': 0, 'frame_window_size': 9}, 'train_dataloader': {'per_device_batch_size': 8, 'per_device_num_workers': 4, 'dataset': {'type': 'DistributedBalancedRepeatingDataset', 'name_mappings': {'observation.state': ['proprio'], 'action': ['action']}, 'statistic_keys': ['observation.state', 'timestamp', 'action'], 'statistic_name': 'franka', 'datasets': [{'type': 'ParquetDataset', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'frame_stride': 2}, {'type': 'CanonicalizePrompt', 'output_key': 'prompt', 'input_keys': ['prompt', 'task_description', 'lang', 'instruction', 'instructions', 'text'], 'remove_source_keys': True}, {'type': 'ProcessCosmos25Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, 'input_key': 'prompt', 'remove_input_key': True}, {'type': 'ResizeImages', 'height': 224, 'width': 224, 'backend': 'torch', 'scale_divisor': 255.0, 'output_layout': 'nchw'}, {'type': 'ConcatImagesHorizontally', 'key': 'images', 'num_views': 2, 'frame_stride': 1, 'views_first': True, 'keep_time_dim': True}, {'type': 'SinCosKeys', 'keys': ['states'], 'target_dims': {'states': 16}, 'interleave': True, 'backend': 'torch'}, {'type': 'NormalizeStatesAndActions', 'state_key': 'proprio', 'action_key': 'action', 'action_dim': None, 'state_dim': None, 'state_norm_type': 'none', 'action_norm_type': 'min_max', 'norm_type': 'min_max', 'action_norm_mask': [True, True, True, True, True, True, False], 'clip_norm': False, 'normalization_epsilon': 0.0, 'preserve_input_dtype': True, 'output_dtype': 'float16'}, {'type': 'PadActionsAndActionMasks', 'action_dim': 8, 'valid_action_dim': 7}, {'type': 'PrepareDiT4DiTInputs', 'state_dim': 16, 'action_horizon': 8, 'action_dim': 8, 'valid_action_dim': 7, 'mark_all_action_steps_valid': True}], 'action_window_size': 8, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'franka', 'window_start_idx': 0, 'frame_window_size': 9, 'data_root_path': './datasets/libero_object_no_noops_lerobotv2.1'}, {'type': 'ParquetDataset', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'frame_stride': 2}, {'type': 'CanonicalizePrompt', 'output_key': 'prompt', 'input_keys': ['prompt', 'task_description', 'lang', 'instruction', 'instructions', 'text'], 'remove_source_keys': True}, {'type': 'ProcessCosmos25Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, 'input_key': 'prompt', 'remove_input_key': True}, {'type': 'ResizeImages', 'height': 224, 'width': 224, 'backend': 'torch', 'scale_divisor': 255.0, 'output_layout': 'nchw'}, {'type': 'ConcatImagesHorizontally', 'key': 'images', 'num_views': 2, 'frame_stride': 1, 'views_first': True, 'keep_time_dim': True}, {'type': 'SinCosKeys', 'keys': ['states'], 'target_dims': {'states': 16}, 'interleave': True, 'backend': 'torch'}, {'type': 'NormalizeStatesAndActions', 'state_key': 'proprio', 'action_key': 'action', 'action_dim': None, 'state_dim': None, 'state_norm_type': 'none', 'action_norm_type': 'min_max', 'norm_type': 'min_max', 'action_norm_mask': [True, True, True, True, True, True, False], 'clip_norm': False, 'normalization_epsilon': 0.0, 'preserve_input_dtype': True, 'output_dtype': 'float16'}, {'type': 'PadActionsAndActionMasks', 'action_dim': 8, 'valid_action_dim': 7}, {'type': 'PrepareDiT4DiTInputs', 'state_dim': 16, 'action_horizon': 8, 'action_dim': 8, 'valid_action_dim': 7, 'mark_all_action_steps_valid': True}], 'action_window_size': 8, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'franka', 'window_start_idx': 0, 'frame_window_size': 9, 'data_root_path': './datasets/libero_goal_no_noops_lerobotv2.1'}, {'type': 'ParquetDataset', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'frame_stride': 2}, {'type': 'CanonicalizePrompt', 'output_key': 'prompt', 'input_keys': ['prompt', 'task_description', 'lang', 'instruction', 'instructions', 'text'], 'remove_source_keys': True}, {'type': 'ProcessCosmos25Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, 'input_key': 'prompt', 'remove_input_key': True}, {'type': 'ResizeImages', 'height': 224, 'width': 224, 'backend': 'torch', 'scale_divisor': 255.0, 'output_layout': 'nchw'}, {'type': 'ConcatImagesHorizontally', 'key': 'images', 'num_views': 2, 'frame_stride': 1, 'views_first': True, 'keep_time_dim': True}, {'type': 'SinCosKeys', 'keys': ['states'], 'target_dims': {'states': 16}, 'interleave': True, 'backend': 'torch'}, {'type': 'NormalizeStatesAndActions', 'state_key': 'proprio', 'action_key': 'action', 'action_dim': None, 'state_dim': None, 'state_norm_type': 'none', 'action_norm_type': 'min_max', 'norm_type': 'min_max', 'action_norm_mask': [True, True, True, True, True, True, False], 'clip_norm': False, 'normalization_epsilon': 0.0, 'preserve_input_dtype': True, 'output_dtype': 'float16'}, {'type': 'PadActionsAndActionMasks', 'action_dim': 8, 'valid_action_dim': 7}, {'type': 'PrepareDiT4DiTInputs', 'state_dim': 16, 'action_horizon': 8, 'action_dim': 8, 'valid_action_dim': 7, 'mark_all_action_steps_valid': True}], 'action_window_size': 8, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'franka', 'window_start_idx': 0, 'frame_window_size': 9, 'data_root_path': './datasets/libero_spatial_no_noops_lerobotv2.1'}, {'type': 'ParquetDataset', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'frame_stride': 2}, {'type': 'CanonicalizePrompt', 'output_key': 'prompt', 'input_keys': ['prompt', 'task_description', 'lang', 'instruction', 'instructions', 'text'], 'remove_source_keys': True}, {'type': 'ProcessCosmos25Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, 'input_key': 'prompt', 'remove_input_key': True}, {'type': 'ResizeImages', 'height': 224, 'width': 224, 'backend': 'torch', 'scale_divisor': 255.0, 'output_layout': 'nchw'}, {'type': 'ConcatImagesHorizontally', 'key': 'images', 'num_views': 2, 'frame_stride': 1, 'views_first': True, 'keep_time_dim': True}, {'type': 'SinCosKeys', 'keys': ['states'], 'target_dims': {'states': 16}, 'interleave': True, 'backend': 'torch'}, {'type': 'NormalizeStatesAndActions', 'state_key': 'proprio', 'action_key': 'action', 'action_dim': None, 'state_dim': None, 'state_norm_type': 'none', 'action_norm_type': 'min_max', 'norm_type': 'min_max', 'action_norm_mask': [True, True, True, True, True, True, False], 'clip_norm': False, 'normalization_epsilon': 0.0, 'preserve_input_dtype': True, 'output_dtype': 'float16'}, {'type': 'PadActionsAndActionMasks', 'action_dim': 8, 'valid_action_dim': 7}, {'type': 'PrepareDiT4DiTInputs', 'state_dim': 16, 'action_horizon': 8, 'action_dim': 8, 'valid_action_dim': 7, 'mark_all_action_steps_valid': True}], 'action_window_size': 8, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'franka', 'window_start_idx': 0, 'frame_window_size': 9, 'data_root_path': './datasets/libero_10_no_noops_lerobotv2.1'}], 'sampling_weights': [1.0, 1.0, 1.0, 1.0], 'shuffle': False, 'reshuffle_each_epoch': True, 'seed': 42, 'dataset_statistics': {'franka': {'action': {'mean': [0.07237596483901143, 0.08987006871029735, -0.10144743137061596, -0.00045383188989944756, 0.006273590726777911, -0.003878799732774496, 0.524486355483532], 'std': [0.3498823308902479, 0.37794140366375184, 0.460084266976933, 0.0403885784928603, 0.06616144248501059, 0.07763074391911857, 0.4994683356809767], 'max': [0.9375, 0.9375, 0.9375, 0.3557142913341522, 0.375, 0.375, 1.0], 'min': [-0.9375, -0.9375, -0.9375, -0.2582142949104309, -0.375, -0.3675000071525574, 0.0], 'q01': [-0.8785714507102966, -0.8758928775787354, -0.9375, -0.1510714292526245, -0.20678570866584778, -0.2742857038974762, 0.0], 'q99': [0.9375, 0.9107142686843872, 0.9375, 0.20357142388820648, 0.26357144117355347, 0.375, 1.0], 'mask': [True, True, True, True, True, True, False]}, 'state': {'mean': [-0.04889854742214084, 0.03689368185587227, 0.7890402488410473, 2.9771945476531982, -0.1417286954820156, -0.11769362539052963, 0.026436020154505968, -0.02665513101965189], 'std': [0.10639013941746686, 0.15115733130675715, 0.38406895599530033, 0.3530238395244304, 0.8227341427331599, 0.32357567121520087, 0.014583991652936385, 0.014467005007200339], 'max': [0.21031762659549713, 0.39128610491752625, 1.3660105466842651, 3.6714255809783936, 3.560650587081909, 1.386339545249939, 0.04233968257904053, 0.0013633022317662835], 'min': [-0.4828203022480011, -0.3255046010017395, 0.008128180168569088, 0.35277295112609863, -3.641430377960205, -1.842738389968872, -0.0013586411951109767, -0.042040832340717316], 'q01': [-0.42401049643754957, -0.2838300323486328, 0.009925739830359817, 1.3085840785503386, -2.886677579879761, -1.1599004411697387, 0.001503719249740243, -0.040336399003863335], 'q99': [0.1530261474847791, 0.3629165390133857, 1.2910678112506866, 3.303542451858519, 2.7496529006957933, 0.6893712210655194, 0.040610933862626555, -0.0015016929572448147]}, 'num_transitions': 273465, 'num_trajectories': 1693}}}}, 'runner': {'type': 'FSDPTrainRunner', 'max_steps': 160000, 'grad_accumulation_steps': 2, 'optimizer': {'lr': 3e-05, 'type': 'AdamW', 'weight_decay': 1e-08, 'eps': 1e-08, 'betas': (0.9, 0.95), 'paramwise_learning_rate': {'vlm_backbone.transformer': 0.0001, 'vla_head': 0.0001}}, 'max_grad_norm': 1.0, 'save_iter_interval': 40000, 'max_keep_ckpts': 2, 'collator': {'type': 'DictCollator', 'keys': ['states', 'timestamp', 'images', 'img_masks', 'actions', 'action_masks', 'frame_masks', 'lang_tokens', 'lang_masks'], 'meta_keys': ['info', 'stats']}, 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, 'sampler': None, 'metric': {'type': 'VLAMetric', 'active_trackers': ('jsonl', 'wandb'), 'run_dir': './work_dirs/dit4dit_libero_all_full_finetune_bc071270ff2_bs256', 'grad_accumulation_steps': 1, 'window_size': 1}, 'lr_scheduler': {'type': 'cosine_with_min_lr', 'warmup_steps': 10000, 'min_lr': 5e-07}, 'sharding_strategy': 'shard-grad-op', 'pre_fsdp_param_dtype': 'fp32', 'enable_gradient_checkpointing': False, 'enable_mixed_precision_training': True, 'mixed_precision_dtype': 'bf16', 'reduce_in_full_precision': False, 'change_key_name': False, 'cfg': Config (path: configs/dit4dit/dit4dit_libero_all_full_finetune.py): {'_ckpt_root': './checkpoints', '_cosmos_base_model': './checkpoints/Cosmos-Predict2.5-2B', '_cosmos_revision': 'diffusers/base/post-trained', '_cosmos_tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, '_official_dit4dit_ckpt': './checkpoints/dit4dit-model/dit4dit_libero/final_model/pytorch_model.pt', '_official_dataset_statistics': {'franka': {'action': {'mean': [0.07237596483901143, 0.08987006871029735, -0.10144743137061596, -0.00045383188989944756, 0.006273590726777911, -0.003878799732774496, 0.524486355483532], 'std': [0.3498823308902479, 0.37794140366375184, 0.460084266976933, 0.0403885784928603, 0.06616144248501059, 0.07763074391911857, 0.4994683356809767], 'max': [0.9375, 0.9375, 0.9375, 0.3557142913341522, 0.375, 0.375, 1.0], 'min': [-0.9375, -0.9375, -0.9375, -0.2582142949104309, -0.375, -0.3675000071525574, 0.0], 'q01': [-0.8785714507102966, -0.8758928775787354, -0.9375, -0.1510714292526245, -0.20678570866584778, -0.2742857038974762, 0.0], 'q99': [0.9375, 0.9107142686843872, 0.9375, 0.20357142388820648, 0.26357144117355347, 0.375, 1.0], 'mask': [True, True, True, True, True, True, False]}, 'state': {'mean': [-0.04889854742214084, 0.03689368185587227, 0.7890402488410473, 2.9771945476531982, -0.1417286954820156, -0.11769362539052963, 0.026436020154505968, -0.02665513101965189], 'std': [0.10639013941746686, 0.15115733130675715, 0.38406895599530033, 0.3530238395244304, 0.8227341427331599, 0.32357567121520087, 0.014583991652936385, 0.014467005007200339], 'max': [0.21031762659549713, 0.39128610491752625, 1.3660105466842651, 3.6714255809783936, 3.560650587081909, 1.386339545249939, 0.04233968257904053, 0.0013633022317662835], 'min': [-0.4828203022480011, -0.3255046010017395, 0.008128180168569088, 0.35277295112609863, -3.641430377960205, -1.842738389968872, -0.0013586411951109767, -0.042040832340717316], 'q01': [-0.42401049643754957, -0.2838300323486328, 0.009925739830359817, 1.3085840785503386, -2.886677579879761, -1.1599004411697387, 0.001503719249740243, -0.040336399003863335], 'q99': [0.1530261474847791, 0.3629165390133857, 1.2910678112506866, 3.303542451858519, 2.7496529006957933, 0.6893712210655194, 0.040610933862626555, -0.0015016929572448147]}, 'num_transitions': 273465, 'num_trajectories': 1693}}, '_action_dim': 8, '_ori_action_dim': 7, '_state_dim': 16, '_action_horizon': 8, '_frame_window_size': 9, '_image_frame_stride': 2, '_image_size': 224, '_action_norm_mask': [True, True, True, True, True, True, False], '_libero_data_roots': ['./datasets/libero_object_no_noops_lerobotv2.1', './datasets/libero_goal_no_noops_lerobotv2.1', './datasets/libero_spatial_no_noops_lerobotv2.1', './datasets/libero_10_no_noops_lerobotv2.1'], '_prompt_input_keys': ['prompt', 'task_description', 'lang', 'instruction', 'instructions', 'text'], 'seed': 42, 'official_eval_task_suites': ['libero_spatial', 'libero_object', 'libero_goal', 'libero_10'], 'model': {'type': 'DiT4DiTVLA', 'pretrained_name_or_path': None, 'name_mapping': {'vlm_backbone.text_encoder': 'backbone_interface.extractor.text_encoder', 'vlm_backbone.transformer': 'backbone_interface.extractor.transformer', 'vlm_backbone.vae': 'backbone_interface.extractor.vae', 'vla_head': 'action_model'}, 'strict_mapping': False, 'repeated_diffusion_steps': 4, 'vlm_backbone': {'type': 'Cosmos25Backbone', 'base_model': './checkpoints/Cosmos-Predict2.5-2B', 'revision': 'diffusers/base/post-trained', 'torch_dtype': 'bf16', 'local_files_only': True, 'extract_layer': 17, 'max_sequence_length': 512, 'trainable': True, 'frozen_submodules': ['text_encoder', 'vae'], 'split_future_frames': True, 'num_frames_out': 5, 'fixed_seed': None, 'num_inference_steps': 1, 'conditional_frame_timestep': 0.0001, 'future_loss_type': 'flow_matching', 'detach_hidden_states': True, 'flow_matching_time_distribution': 'uniform', 'flow_matching_high_sigma_ratio': None, 'flow_matching_high_sigma_min': None, 'fsdp_min_num_params': 0}, 'vla_head': {'type': 'DiT4DiTActionHead', 'action_model_type': 'DiT-B', 'hidden_size': 2560, 'add_pos_embed': True, 'max_seq_len': 1024, 'action_dim': 8, 'ori_action_dim': 7, 'state_dim': 16, 'future_action_window_size': 7, 'action_horizon': 8, 'noise_beta_alpha': 1.5, 'noise_beta_beta': 1.0, 'noise_s': 0.999, 'num_timestep_buckets': 1000, 'num_inference_timesteps': 4, 'diffusion_model_cfg': {'cross_attention_dim': 2048, 'dropout': 0.2, 'final_dropout': True, 'interleave_self_attention': True, 'norm_type': 'ada_norm', 'num_layers': 16, 'output_dim': 2560, 'positional_embeddings': None}}, 'freeze_vlm_backbone': False}, 'inference_model': {'type': 'DiT4DiTVLA', 'pretrained_name_or_path': None, 'name_mapping': None, 'strict_mapping': False, 'repeated_diffusion_steps': 4, 'vlm_backbone': {'type': 'Cosmos25Backbone', 'base_model': './checkpoints/Cosmos-Predict2.5-2B', 'revision': 'diffusers/base/post-trained', 'torch_dtype': 'bf16', 'local_files_only': True, 'extract_layer': 17, 'max_sequence_length': 512, 'trainable': True, 'frozen_submodules': ['text_encoder', 'vae'], 'split_future_frames': True, 'num_frames_out': 5, 'fixed_seed': None, 'num_inference_steps': 1, 'conditional_frame_timestep': 0.0001, 'future_loss_type': 'flow_matching', 'detach_hidden_states': True, 'flow_matching_time_distribution': 'uniform', 'flow_matching_high_sigma_ratio': None, 'flow_matching_high_sigma_min': None, 'fsdp_min_num_params': 0}, 'vla_head': {'type': 'DiT4DiTActionHead', 'action_model_type': 'DiT-B', 'hidden_size': 2560, 'add_pos_embed': True, 'max_seq_len': 1024, 'action_dim': 8, 'ori_action_dim': 7, 'state_dim': 16, 'future_action_window_size': 7, 'action_horizon': 8, 'noise_beta_alpha': 1.5, 'noise_beta_beta': 1.0, 'noise_s': 0.999, 'num_timestep_buckets': 1000, 'num_inference_timesteps': 4, 'diffusion_model_cfg': {'cross_attention_dim': 2048, 'dropout': 0.2, 'final_dropout': True, 'interleave_self_attention': True, 'norm_type': 'ada_norm', 'num_layers': 16, 'output_dim': 2560, 'positional_embeddings': None}}, 'freeze_vlm_backbone': False}, '_dit4dit_train_transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'frame_stride': 2}, {'type': 'CanonicalizePrompt', 'output_key': 'prompt', 'input_keys': ['prompt', 'task_description', 'lang', 'instruction', 'instructions', 'text'], 'remove_source_keys': True}, {'type': 'ProcessCosmos25Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, 'input_key': 'prompt', 'remove_input_key': True}, {'type': 'ResizeImages', 'height': 224, 'width': 224, 'backend': 'torch', 'scale_divisor': 255.0, 'output_layout': 'nchw'}, {'type': 'ConcatImagesHorizontally', 'key': 'images', 'num_views': 2, 'frame_stride': 1, 'views_first': True, 'keep_time_dim': True}, {'type': 'SinCosKeys', 'keys': ['states'], 'target_dims': {'states': 16}, 'interleave': True, 'backend': 'torch'}, {'type': 'NormalizeStatesAndActions', 'state_key': 'proprio', 'action_key': 'action', 'action_dim': None, 'state_dim': None, 'state_norm_type': 'none', 'action_norm_type': 'min_max', 'norm_type': 'min_max', 'action_norm_mask': [True, True, True, True, True, True, False], 'clip_norm': False, 'normalization_epsilon': 0.0, 'preserve_input_dtype': True, 'output_dtype': 'float16'}, {'type': 'PadActionsAndActionMasks', 'action_dim': 8, 'valid_action_dim': 7}, {'type': 'PrepareDiT4DiTInputs', 'state_dim': 16, 'action_horizon': 8, 'action_dim': 8, 'valid_action_dim': 7, 'mark_all_action_steps_valid': True}], '_dit4dit_parquet_dataset': {'type': 'ParquetDataset', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'frame_stride': 2}, {'type': 'CanonicalizePrompt', 'output_key': 'prompt', 'input_keys': ['prompt', 'task_description', 'lang', 'instruction', 'instructions', 'text'], 'remove_source_keys': True}, {'type': 'ProcessCosmos25Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, 'input_key': 'prompt', 'remove_input_key': True}, {'type': 'ResizeImages', 'height': 224, 'width': 224, 'backend': 'torch', 'scale_divisor': 255.0, 'output_layout': 'nchw'}, {'type': 'ConcatImagesHorizontally', 'key': 'images', 'num_views': 2, 'frame_stride': 1, 'views_first': True, 'keep_time_dim': True}, {'type': 'SinCosKeys', 'keys': ['states'], 'target_dims': {'states': 16}, 'interleave': True, 'backend': 'torch'}, {'type': 'NormalizeStatesAndActions', 'state_key': 'proprio', 'action_key': 'action', 'action_dim': None, 'state_dim': None, 'state_norm_type': 'none', 'action_norm_type': 'min_max', 'norm_type': 'min_max', 'action_norm_mask': [True, True, True, True, True, True, False], 'clip_norm': False, 'normalization_epsilon': 0.0, 'preserve_input_dtype': True, 'output_dtype': 'float16'}, {'type': 'PadActionsAndActionMasks', 'action_dim': 8, 'valid_action_dim': 7}, {'type': 'PrepareDiT4DiTInputs', 'state_dim': 16, 'action_horizon': 8, 'action_dim': 8, 'valid_action_dim': 7, 'mark_all_action_steps_valid': True}], 'action_window_size': 8, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'franka', 'window_start_idx': 0, 'frame_window_size': 9}, 'train_dataloader': {'per_device_batch_size': 8, 'per_device_num_workers': 4, 'dataset': {'type': 'DistributedBalancedRepeatingDataset', 'name_mappings': {'observation.state': ['proprio'], 'action': ['action']}, 'statistic_keys': ['observation.state', 'timestamp', 'action'], 'statistic_name': 'franka', 'datasets': [{'type': 'ParquetDataset', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'frame_stride': 2}, {'type': 'CanonicalizePrompt', 'output_key': 'prompt', 'input_keys': ['prompt', 'task_description', 'lang', 'instruction', 'instructions', 'text'], 'remove_source_keys': True}, {'type': 'ProcessCosmos25Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, 'input_key': 'prompt', 'remove_input_key': True}, {'type': 'ResizeImages', 'height': 224, 'width': 224, 'backend': 'torch', 'scale_divisor': 255.0, 'output_layout': 'nchw'}, {'type': 'ConcatImagesHorizontally', 'key': 'images', 'num_views': 2, 'frame_stride': 1, 'views_first': True, 'keep_time_dim': True}, {'type': 'SinCosKeys', 'keys': ['states'], 'target_dims': {'states': 16}, 'interleave': True, 'backend': 'torch'}, {'type': 'NormalizeStatesAndActions', 'state_key': 'proprio', 'action_key': 'action', 'action_dim': None, 'state_dim': None, 'state_norm_type': 'none', 'action_norm_type': 'min_max', 'norm_type': 'min_max', 'action_norm_mask': [True, True, True, True, True, True, False], 'clip_norm': False, 'normalization_epsilon': 0.0, 'preserve_input_dtype': True, 'output_dtype': 'float16'}, {'type': 'PadActionsAndActionMasks', 'action_dim': 8, 'valid_action_dim': 7}, {'type': 'PrepareDiT4DiTInputs', 'state_dim': 16, 'action_horizon': 8, 'action_dim': 8, 'valid_action_dim': 7, 'mark_all_action_steps_valid': True}], 'action_window_size': 8, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'franka', 'window_start_idx': 0, 'frame_window_size': 9, 'data_root_path': './datasets/libero_object_no_noops_lerobotv2.1'}, {'type': 'ParquetDataset', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'frame_stride': 2}, {'type': 'CanonicalizePrompt', 'output_key': 'prompt', 'input_keys': ['prompt', 'task_description', 'lang', 'instruction', 'instructions', 'text'], 'remove_source_keys': True}, {'type': 'ProcessCosmos25Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, 'input_key': 'prompt', 'remove_input_key': True}, {'type': 'ResizeImages', 'height': 224, 'width': 224, 'backend': 'torch', 'scale_divisor': 255.0, 'output_layout': 'nchw'}, {'type': 'ConcatImagesHorizontally', 'key': 'images', 'num_views': 2, 'frame_stride': 1, 'views_first': True, 'keep_time_dim': True}, {'type': 'SinCosKeys', 'keys': ['states'], 'target_dims': {'states': 16}, 'interleave': True, 'backend': 'torch'}, {'type': 'NormalizeStatesAndActions', 'state_key': 'proprio', 'action_key': 'action', 'action_dim': None, 'state_dim': None, 'state_norm_type': 'none', 'action_norm_type': 'min_max', 'norm_type': 'min_max', 'action_norm_mask': [True, True, True, True, True, True, False], 'clip_norm': False, 'normalization_epsilon': 0.0, 'preserve_input_dtype': True, 'output_dtype': 'float16'}, {'type': 'PadActionsAndActionMasks', 'action_dim': 8, 'valid_action_dim': 7}, {'type': 'PrepareDiT4DiTInputs', 'state_dim': 16, 'action_horizon': 8, 'action_dim': 8, 'valid_action_dim': 7, 'mark_all_action_steps_valid': True}], 'action_window_size': 8, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'franka', 'window_start_idx': 0, 'frame_window_size': 9, 'data_root_path': './datasets/libero_goal_no_noops_lerobotv2.1'}, {'type': 'ParquetDataset', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'frame_stride': 2}, {'type': 'CanonicalizePrompt', 'output_key': 'prompt', 'input_keys': ['prompt', 'task_description', 'lang', 'instruction', 'instructions', 'text'], 'remove_source_keys': True}, {'type': 'ProcessCosmos25Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, 'input_key': 'prompt', 'remove_input_key': True}, {'type': 'ResizeImages', 'height': 224, 'width': 224, 'backend': 'torch', 'scale_divisor': 255.0, 'output_layout': 'nchw'}, {'type': 'ConcatImagesHorizontally', 'key': 'images', 'num_views': 2, 'frame_stride': 1, 'views_first': True, 'keep_time_dim': True}, {'type': 'SinCosKeys', 'keys': ['states'], 'target_dims': {'states': 16}, 'interleave': True, 'backend': 'torch'}, {'type': 'NormalizeStatesAndActions', 'state_key': 'proprio', 'action_key': 'action', 'action_dim': None, 'state_dim': None, 'state_norm_type': 'none', 'action_norm_type': 'min_max', 'norm_type': 'min_max', 'action_norm_mask': [True, True, True, True, True, True, False], 'clip_norm': False, 'normalization_epsilon': 0.0, 'preserve_input_dtype': True, 'output_dtype': 'float16'}, {'type': 'PadActionsAndActionMasks', 'action_dim': 8, 'valid_action_dim': 7}, {'type': 'PrepareDiT4DiTInputs', 'state_dim': 16, 'action_horizon': 8, 'action_dim': 8, 'valid_action_dim': 7, 'mark_all_action_steps_valid': True}], 'action_window_size': 8, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'franka', 'window_start_idx': 0, 'frame_window_size': 9, 'data_root_path': './datasets/libero_spatial_no_noops_lerobotv2.1'}, {'type': 'ParquetDataset', 'transforms': [{'type': 'ProcessParquetInputs', 'parquet_keys': ['observation.state', 'timestamp', 'actions', 'info', 'stats', 'action_masks'], 'video_keys': ['observation.images.image', 'observation.images.wrist_image'], 'name_mappings': {'observation.state': ['states'], 'actions': ['actions']}, 'frame_stride': 2}, {'type': 'CanonicalizePrompt', 'output_key': 'prompt', 'input_keys': ['prompt', 'task_description', 'lang', 'instruction', 'instructions', 'text'], 'remove_source_keys': True}, {'type': 'ProcessCosmos25Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, 'input_key': 'prompt', 'remove_input_key': True}, {'type': 'ResizeImages', 'height': 224, 'width': 224, 'backend': 'torch', 'scale_divisor': 255.0, 'output_layout': 'nchw'}, {'type': 'ConcatImagesHorizontally', 'key': 'images', 'num_views': 2, 'frame_stride': 1, 'views_first': True, 'keep_time_dim': True}, {'type': 'SinCosKeys', 'keys': ['states'], 'target_dims': {'states': 16}, 'interleave': True, 'backend': 'torch'}, {'type': 'NormalizeStatesAndActions', 'state_key': 'proprio', 'action_key': 'action', 'action_dim': None, 'state_dim': None, 'state_norm_type': 'none', 'action_norm_type': 'min_max', 'norm_type': 'min_max', 'action_norm_mask': [True, True, True, True, True, True, False], 'clip_norm': False, 'normalization_epsilon': 0.0, 'preserve_input_dtype': True, 'output_dtype': 'float16'}, {'type': 'PadActionsAndActionMasks', 'action_dim': 8, 'valid_action_dim': 7}, {'type': 'PrepareDiT4DiTInputs', 'state_dim': 16, 'action_horizon': 8, 'action_dim': 8, 'valid_action_dim': 7, 'mark_all_action_steps_valid': True}], 'action_window_size': 8, 'action_key': 'action', 'use_delta': False, 'statistic_name': 'franka', 'window_start_idx': 0, 'frame_window_size': 9, 'data_root_path': './datasets/libero_10_no_noops_lerobotv2.1'}], 'sampling_weights': [1.0, 1.0, 1.0, 1.0], 'shuffle': False, 'reshuffle_each_epoch': True, 'seed': 42, 'dataset_statistics': {'franka': {'action': {'mean': [0.07237596483901143, 0.08987006871029735, -0.10144743137061596, -0.00045383188989944756, 0.006273590726777911, -0.003878799732774496, 0.524486355483532], 'std': [0.3498823308902479, 0.37794140366375184, 0.460084266976933, 0.0403885784928603, 0.06616144248501059, 0.07763074391911857, 0.4994683356809767], 'max': [0.9375, 0.9375, 0.9375, 0.3557142913341522, 0.375, 0.375, 1.0], 'min': [-0.9375, -0.9375, -0.9375, -0.2582142949104309, -0.375, -0.3675000071525574, 0.0], 'q01': [-0.8785714507102966, -0.8758928775787354, -0.9375, -0.1510714292526245, -0.20678570866584778, -0.2742857038974762, 0.0], 'q99': [0.9375, 0.9107142686843872, 0.9375, 0.20357142388820648, 0.26357144117355347, 0.375, 1.0], 'mask': [True, True, True, True, True, True, False]}, 'state': {'mean': [-0.04889854742214084, 0.03689368185587227, 0.7890402488410473, 2.9771945476531982, -0.1417286954820156, -0.11769362539052963, 0.026436020154505968, -0.02665513101965189], 'std': [0.10639013941746686, 0.15115733130675715, 0.38406895599530033, 0.3530238395244304, 0.8227341427331599, 0.32357567121520087, 0.014583991652936385, 0.014467005007200339], 'max': [0.21031762659549713, 0.39128610491752625, 1.3660105466842651, 3.6714255809783936, 3.560650587081909, 1.386339545249939, 0.04233968257904053, 0.0013633022317662835], 'min': [-0.4828203022480011, -0.3255046010017395, 0.008128180168569088, 0.35277295112609863, -3.641430377960205, -1.842738389968872, -0.0013586411951109767, -0.042040832340717316], 'q01': [-0.42401049643754957, -0.2838300323486328, 0.009925739830359817, 1.3085840785503386, -2.886677579879761, -1.1599004411697387, 0.001503719249740243, -0.040336399003863335], 'q99': [0.1530261474847791, 0.3629165390133857, 1.2910678112506866, 3.303542451858519, 2.7496529006957933, 0.6893712210655194, 0.040610933862626555, -0.0015016929572448147]}, 'num_transitions': 273465, 'num_trajectories': 1693}}}}, 'runner': {'type': 'FSDPTrainRunner', 'max_steps': 160000, 'grad_accumulation_steps': 2, 'optimizer': {'lr': 3e-05, 'type': 'AdamW', 'weight_decay': 1e-08, 'eps': 1e-08, 'betas': (0.9, 0.95), 'paramwise_learning_rate': {'vlm_backbone.transformer': 0.0001, 'vla_head': 0.0001}}, 'max_grad_norm': 1.0, 'save_iter_interval': 40000, 'max_keep_ckpts': 2, 'collator': {'type': 'DictCollator', 'keys': ['states', 'timestamp', 'images', 'img_masks', 'actions', 'action_masks', 'frame_masks', 'lang_tokens', 'lang_masks'], 'meta_keys': ['info', 'stats']}, 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, 'sampler': None, 'metric': {'type': 'VLAMetric', 'active_trackers': ('jsonl', 'wandb'), 'run_dir': './work_dirs/dit4dit_libero_all_full_finetune_bc071270ff2_bs256', 'grad_accumulation_steps': 1, 'window_size': 1}, 'lr_scheduler': {'type': 'cosine_with_min_lr', 'warmup_steps': 10000, 'min_lr': 5e-07}, 'sharding_strategy': 'shard-grad-op', 'pre_fsdp_param_dtype': 'fp32', 'enable_gradient_checkpointing': False, 'enable_mixed_precision_training': True, 'mixed_precision_dtype': 'bf16', 'reduce_in_full_precision': False, 'change_key_name': False, 'cfg': Config (path: configs/dit4dit/dit4dit_libero_all_full_finetune.py): {...}, 'args': Namespace(config='configs/dit4dit/dit4dit_libero_all_full_finetune.py', work_dir='./work_dirs/dit4dit_libero_all_full_finetune_bc071270ff2_bs256', cfg_options={'train_dataloader.per_device_batch_size': 8, 'runner.grad_accumulation_steps': 2}, eval_after_train=True, resume_from=None)}, 'eval': {'type': 'LiberoEvalRunner', 'task_suite_name': ['libero_spatial', 'libero_object', 'libero_goal', 'libero_10'], 'model_family': 'dit4dit', 'norm_stats_key': 'franka', 'eval_chunk_size': 8, 'truncate_action_chunk_at_horizon': True, 'resize_size': 224, 'num_trials_per_task': 50, 'num_steps_wait': 10, 'seed': 7, 'enable_mixed_precision_training': False, 'mixed_precision_dtype': 'bf16', 'dataset': {'type': 'LiberoParquetEvalDataset', 'img_buffer_len': 1, 'require_lang_tokens': True, 'transforms': [{'type': 'CanonicalizePrompt', 'output_key': 'prompt', 'input_keys': ['prompt', 'task_description', 'lang', 'instruction', 'instructions', 'text'], 'remove_source_keys': True}, {'type': 'ProcessCosmos25Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, 'input_key': 'prompt', 'remove_input_key': True}, {'type': 'ProcessDiT4DiTLiberoEvalInputs', 'img_keys': ['agentview_image', 'robot0_eye_in_hand_image'], 'image_size': 224}, {'type': 'LiberoProprioFromInputs', 'norm_type': 'none', 'pos_key': 'robot0_eef_pos', 'quat_key': 'robot0_eef_quat', 'gripper_key': 'robot0_gripper_qpos', 'state_dim': None, 'out_key': 'states'}, {'type': 'SinCosKeys', 'keys': ['states'], 'target_dims': {'states': 16}, 'interleave': True}, {'type': 'PrepareDiT4DiTInputs', 'image_key': 'pixel_values', 'state_dim': 16, 'require_actions': False}]}, 'denormalize_action': {'type': 'DenormalizeLiberoAction', 'norm_type': 'min_max', 'action_dim': 7, 'action_norm_mask': [True, True, True, True, True, True, False], 'clip_normalized_action': True, 'normalize_gripper_action': True, 'invert_gripper_action': True}}}, 'args': Namespace(config='configs/dit4dit/dit4dit_libero_all_full_finetune.py', work_dir='./work_dirs/dit4dit_libero_all_full_finetune_bc071270ff2_bs256', cfg_options={'train_dataloader.per_device_batch_size': 8, 'runner.grad_accumulation_steps': 2}, eval_after_train=True, resume_from=None)}, 'eval': {'type': 'LiberoEvalRunner', 'task_suite_name': ['libero_spatial', 'libero_object', 'libero_goal', 'libero_10'], 'model_family': 'dit4dit', 'norm_stats_key': 'franka', 'eval_chunk_size': 8, 'truncate_action_chunk_at_horizon': True, 'resize_size': 224, 'num_trials_per_task': 50, 'num_steps_wait': 10, 'seed': 7, 'enable_mixed_precision_training': False, 'mixed_precision_dtype': 'bf16', 'dataset': {'type': 'LiberoParquetEvalDataset', 'img_buffer_len': 1, 'require_lang_tokens': True, 'transforms': [{'type': 'CanonicalizePrompt', 'output_key': 'prompt', 'input_keys': ['prompt', 'task_description', 'lang', 'instruction', 'instructions', 'text'], 'remove_source_keys': True}, {'type': 'ProcessCosmos25Prompt', 'tokenizer': {'type': 'PretrainedTokenizer', 'model_path': './checkpoints/Cosmos-Predict2.5-2B/tokenizer', 'model_max_length': 512}, 'input_key': 'prompt', 'remove_input_key': True}, {'type': 'ProcessDiT4DiTLiberoEvalInputs', 'img_keys': ['agentview_image', 'robot0_eye_in_hand_image'], 'image_size': 224}, {'type': 'LiberoProprioFromInputs', 'norm_type': 'none', 'pos_key': 'robot0_eef_pos', 'quat_key': 'robot0_eef_quat', 'gripper_key': 'robot0_gripper_qpos', 'state_dim': None, 'out_key': 'states'}, {'type': 'SinCosKeys', 'keys': ['states'], 'target_dims': {'states': 16}, 'interleave': True}, {'type': 'PrepareDiT4DiTInputs', 'image_key': 'pixel_values', 'state_dim': 16, 'require_actions': False}]}, 'denormalize_action': {'type': 'DenormalizeLiberoAction', 'norm_type': 'min_max', 'action_dim': 7, 'action_norm_mask': [True, True, True, True, True, True, False], 'clip_normalized_action': True, 'normalize_gripper_action': True, 'invert_gripper_action': True}}}", "run_id": "dit4dit_libero_all_full_finetune_2026_08_12_20_51_03"}

dit4dit_libero_all_full_finetune_bc071270ff2_bs256/tokenizer/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

dit4dit_libero_all_full_finetune_bc071270ff2_bs256/tokenizer/chat_template.jinja

@@ -0,0 +1,7 @@
+{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
+You are a helpful assistant.<|im_end|>
+{% endif %}<|im_start|>{{ message['role'] }}
+{% if message['content'] is string %}{{ message['content'] }}<|im_end|>
+{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
+{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
+{% endif %}

dit4dit_libero_all_full_finetune_bc071270ff2_bs256/tokenizer/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

dit4dit_libero_all_full_finetune_bc071270ff2_bs256/tokenizer/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

dit4dit_libero_all_full_finetune_bc071270ff2_bs256/tokenizer/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}