{ "allow_patterns_overrides": [ "*/*.safetensors" ], "architectures": [ "Cosmos3ForConditionalGeneration" ], "backbone_type": "cosmos3_multiview", "cosmos3_nano_sim_auto": { "checkpoint_hash": "1c13c34424f1a040f9a7b4a73c469059a4347b50de730e7a7b7bbe8657577b49", "checkpoint_id": "sim_auto_joint_stage3_sfdmd_mads_all@iter_000001800", "checkpoint_iteration": 1800, "schema_version": 1 }, "image_token_id": 151655, "model": { "_recursive_": false, "_target": "omni_mot_causal_model", "config": { "_type": "omni_mot_causal_model_config", "action_gen": false, "action_global_asinh": false, "action_io_projector_type": "domain_aware", "action_metric_gripper": false, "action_schema": "native", "activation_checkpointing": { "_type": "activation_checkpointing_config", "determinism_check": "default", "min_cgroup_memory_free_fraction": 0.1, "mode": "full", "offload_to_cpu": false, "preserve_rng_state": true, "save_only_marked_ops": false, "save_ops_regex": [ "fmha_forward", "_flash_attn_varlen_forward", "_flash_attn_forward", "fmha_fwd" ] }, "ar_distilled_sample_type": null, "ar_static_und_cache_max_len": 512, "attention_io_layout": "sequence_sharded", "attention_sink_size": 0, "causal_training_strategy": "teacher_forcing", "clamp_empty_varlen_kv": true, "compile": { "_type": "compile_config", "ar_post_saturation_mode": "default", "compile_dynamic": true, "compiled_region": "language", "coordinate_descent_tuning": false, "cuda_graph_scope": "block", "enabled": false, "mark_unbacked": false, "max_autotune_pointwise": false, "use_cuda_graphs": false }, "correct_cp_gradients": true, "diffusion_expert_config": { "_type": "diffusion_expert_config", "base_fps": 30, "enable_action_modality_embedding": true, "enable_fps_modulation": true, "enable_media_modality_embedding": false, "enable_sound_modality_embedding": true, "enable_vision_modality_embeddings": false, "lidar_patch_spatial_hw": 1, "load_weights_from_pretrained": false, "max_vae_latent_side_after_patchify": 52, "num_view_embeddings": 12, "patch_spatial": 2, "radar_patch_spatial_hw": null, "sound_base_temporal_compression_factor": null, "timestep_range": 1.0, "unified_3d_mrope_reset_spatial_ids": true, "unified_3d_mrope_temporal_modality_margin": 15000, "vision_temporal_position_mode": "latent_index" }, "ema": { "_type": "ema_config", "enabled": false, "iteration_shift": 0, "rate": 0.1 }, "enable_input_bias": true, "enable_moba": false, "exclude_reasoner_weights_from_checkpoint": false, "fixed_step_sampler_config": { "_type": "fixed_step_sampler_config", "sample_type": "sde", "t_list": [ 1.0, 0.9375, 0.8333333333333334, 0.625 ] }, "flex_attention": { "_type": "causal_flex_attention_config", "backend": "auto", "geometry_position_encoding": "baseline", "mask": { "_type": "causal_flex_attention_mask_config", "attention_scope": "all_views", "control_attends_sensor": false, "cross_view_temporal_window": { "_type": "cross_view_temporal_window_config", "bounds": null, "includes_first_frame": false }, "lidar_attends_captions": true, "radar_attends_captions": true, "same_view_temporal_window": { "_type": "same_view_temporal_window_config", "control_to_control_bounds": null, "control_to_sensor_bounds": null, "sensor_to_control_bounds": null, "sensor_to_sensor_bounds": null } }, "maskless_cross_view_band_keys": "other_views", "maskless_cross_view_band_radius": 0, "maskless_cross_view_include_frame_zero": false, "prope_intrinsics": false, "rigrope_canonical_frame": false, "rigrope_channels": "all", "rigrope_head_fraction": 1.0, "rigrope_include_time": false, "rigrope_max_freq_exponent": 3.0, "rigrope_min_freq_exponent": 0.0, "rigrope_missing_geometry": "error", "rigrope_moment_normalization": "fixed", "rigrope_moment_scale_floor_m": 0.05, "rigrope_moment_scale_m": 25.0, "rigrope_pose_world_frame": "scene", "rigrope_time_scale_s": 1.0 }, "grad_clip": true, "input_caption_key": "ai_caption", "input_image_key": "images", "input_video_key": "video", "joint_attn_implementation": "multiview", "kv_cache_dtype": null, "kv_cache_inference_size": null, "kv_cache_kernel_impl": "triton", "latent_downsample_factor": 16, "lbl": { "_type": "lbl_config", "coeff_gen": null, "coeff_und": null, "method": "local" }, "lidar_fps": 10.0, "lidar_state_ch": 128, "lidar_tokenizer": { "_target": "lidar_tokenizer_v1_interface", "apply_validity_mask": true, "bucket_name": "bucket", "device": "cuda", "dtype": "float32", "load_checkpoint": true, "network_config": { "base_channels": 128, "bottleneck_3d": true, "bottleneck_3d_causal_time": true, "bottleneck_3d_max_t": 32, "bottleneck_3d_rope": true, "depths": [ 3, 3, 3, 3 ], "dilation": [ 1, 1, 1, 1 ], "formulation": "VAE", "in_channels": 3, "mapping_depth": 2, "mask_as_input": false, "mlp_ratio": 3.0, "num_heads": [ 4, 4, 8, 8 ], "out_channels": 3, "patch_size": [ 2, 2 ], "positional_embedding": "learnable_embedding", "resolution": [ 128, 1808 ], "temporal_downsample": [ false, false, false ], "temporal_upsample": [ false, false, false ], "window_size": [ 5, 45 ], "z_dim": 128 }, "object_store_credential_path_pretrained": null, "range_projection": { "_target": "lidar_range_projection_config", "azimuth_end_degrees": -180.0, "azimuth_endpoint": false, "azimuth_start_degrees": 180.0, "coordinate_system": "x_forward_y_left_z_up", "intensity_encoding": "unit", "invalid_range_m": 0.0, "max_range_m": 100.0, "min_range_m": 5.0, "model_width": 1808, "model_width_transform": "circular_pad", "native_height": 128, "native_width": 3600, "return_selection": "nearest", "semantic_height": 128, "semantic_width": 1800, "sensor": "pandar128", "validity_threshold": 0.5 }, "sample_posterior": false, "spatial_compression": [ 16, 16 ], "spatial_compression_factor": 16, "streaming_chunk_frames": 20, "streaming_context_frames": 21, "temporal_compression_factor": 1, "vae_path": "pretrained/tokenizers/lidar/diffusion_pytorch_model.safetensors" }, "load_vision_tokenizer": true, "lora_alpha": 32, "lora_enabled": false, "lora_rank": 16, "lora_target_modules": "q_proj_moe_gen,k_proj_moe_gen,v_proj_moe_gen,o_proj_moe_gen", "max_action_dim": 32, "max_num_tokens_after_packing": 1400000, "moba_bidirectional_steps": 1, "moba_causal_steps": 1, "multiview_action_conditioning": null, "multiview_attention": { "_type": "flex_attention_config", "backend": "maskless", "geometry_position_encoding": "baseline", "mask": { "_type": "flex_attention_mask_config", "attention_scope": "decomposed", "control_attends_sensor": true, "cross_view_temporal_window": { "_type": "cross_view_temporal_window_config", "bounds": [ -0.4, 0.0 ], "includes_first_frame": false }, "lidar_attends_captions": true, "radar_attends_captions": true, "same_view_temporal_window": { "_type": "same_view_temporal_window_config", "control_to_control_bounds": null, "control_to_sensor_bounds": null, "sensor_to_control_bounds": null, "sensor_to_sensor_bounds": null } }, "maskless_cross_view_band_keys": "other_views", "maskless_cross_view_band_radius": 0, "maskless_cross_view_include_frame_zero": false, "prope_intrinsics": false, "rigrope_canonical_frame": false, "rigrope_channels": "all", "rigrope_head_fraction": 1.0, "rigrope_include_time": false, "rigrope_max_freq_exponent": 3.0, "rigrope_min_freq_exponent": 0.0, "rigrope_missing_geometry": "error", "rigrope_moment_normalization": "fixed", "rigrope_moment_scale_floor_m": 0.05, "rigrope_moment_scale_m": 25.0, "rigrope_pose_world_frame": "scene", "rigrope_time_scale_s": 1.0 }, "natten_parameter_list": null, "net": null, "num_embodiment_domains": 32, "nvfp4_linears": null, "parallelism": { "_type": "parallelism_config", "cfg_parallel_shard_degree": 1, "context_parallel_shard_degree": 1, "data_parallel_replicate_degree": 1, "data_parallel_shard_degree": 1, "enable_inference_mode": false, "fsdp_cpu_offload": false, "fsdp_master_dtype": "float32", "fsdp_reduce_dtype": "bfloat16", "vae_load_balance_group_size": 1 }, "precision": "bfloat16", "radar_fps": null, "radar_state_ch": null, "radar_tokenizer": null, "rectified_flow_inference_config": { "_type": "rectified_flow_inference_config", "num_train_timesteps": 1000, "scheduler_type": "unipc", "shift": 1, "use_dynamic_shifting": false }, "rectified_flow_training_config": { "_type": "rectified_flow_training_config", "action_loss_weight": 10.0, "exclude_fully_conditioned_items": false, "image_loss_scale": null, "independent_action_schedule": false, "independent_sound_schedule": false, "lidar_loss_scale": 200.0, "loss_reduction_mode": "local_item_mean", "loss_scale": 100.0, "normalize_loss_by_active": false, "radar_loss_scale": null, "shift": { "256": 3, "480": 5, "720": 10 }, "shift_action": null, "shift_image": null, "shift_lidar": null, "shift_sound": null, "sound_loss_scale": null, "train_time_action_distribution": "logitnormal", "train_time_image_distribution": "logitnormal", "train_time_sound_distribution": "logitnormal", "train_time_video_distribution": "uniform", "train_time_weight": "uniform", "use_discrete_rf": false, "use_dynamic_shift": false }, "release_cp_raw_batch_after_preprocessing": false, "resolution": "720", "rolling_kv_cache_chunks": 2, "rolling_kv_sink_chunks": 1, "rolling_media_rope_scale": 1.0, "rolling_media_sink_age_cap_seconds": null, "rolling_media_text_extent": null, "rolling_prompt_context_after": 0, "rolling_prompt_context_before": 0, "rolling_prompt_policy": "segmented", "rolling_static_caption_max_tokens": 1024, "rolling_text_position_mode": "fixed", "rolling_text_sink_tokens": 4, "sigma_diffusion_forcing": 0.02, "sound_dim": null, "sound_gen": false, "sound_latent_fps": 25, "sound_tokenizer": { "_target": "avae_interface", "audio_channels": 2, "avae_config_path": "", "avae_path": "pretrained/tokenizers/audio/avae/avae_48k_noncausal_25hz_64ch.ckpt", "bucket_name": "bucket", "hop_size": 1920, "io_channels": 64, "latent_mean": null, "latent_std": null, "normalization_type": "none", "normalize_latents": true, "object_store_credential_path_pretrained": null, "sample_rate": 48000, "tanh_clamp": 0.995, "tanh_input_scale": 1.5, "tanh_output_scale": 3.5 }, "sr_latent_condition_noise": null, "state_ch": 48, "state_t": 300, "teacher_forcing_detach_clean_kv": false, "teacher_forcing_frames_per_chunk": 3, "teacher_forcing_kv_implementation": "multiview_maskless_kv", "teacher_forcing_replay_policy": { "_type": "teacher_forcing_replay_policy_config", "clean_pass_causality": "chunk", "control_visibility": "causal", "controls_read_strict_past_clean_rgb": true, "decomposed_temporal_window_seconds": 0.4, "deduplicate_cross_view": true, "multiview_attention_scope": "decomposed" }, "teacher_forcing_target_only_no_text_pass2": false, "teacher_forcing_transfer_control_dropout_rate": 0.0, "timestep_embedding_chunk_size": 128000, "tokenizer": { "_target": "wan2pt2_vae_interface", "bucket_name": "bucket", "causal": true, "chunk_duration": 93, "encode_bucket_multiple": null, "encode_chunk_frames": { "256": 68, "480": 24, "720": 8 }, "encode_exact_durations": [ 17, 61, 73 ], "keep_decoder_cache": false, "object_store_credential_path_pretrained": null, "spatial_compression_factor": 16, "temporal_compression_factor": 4, "temporal_window": null, "use_streaming_encode": false, "vae_path": "pretrained/tokenizers/video/wan2pt2/Wan2.2_VAE.pth" }, "video_temporal_causal": true, "vision_gen": true, "vlm_config": { "_type": "vlm_config", "layer_module": null, "model_instance": { "_target": "qwen3_vl_text_for_causal_lm", "config": { "_target": "create_vlm_config", "base_config": { "_target": "qwen3_vl_mot_config_from_json_file", "json_file": "cosmos3://vfm/models/vlm/qwen3_vl/configs/Qwen3-VL-8B-Instruct.json" }, "qk_norm_for_text": true } }, "model_name": "nvidia/Cosmos3-Nano-Reasoner", "pretrained_weights": { "_type": "pretrained_weights_config", "backbone_path": "", "checkpoint_format": null, "credentials_path": "", "enable_gcs_patch_in_boto3": false, "enabled": false }, "qk_norm": false, "safetensors_path": "", "tie_word_embeddings": false, "tokenizer": { "_target": "create_qwen2_tokenizer_with_download", "config_variant": "hf", "pretrained_model_name": "Qwen/Qwen3-VL-8B-Instruct" }, "use_system_prompt": false } } }, "model_type": "cosmos3_omni", "per_view_caption_task": "av_joint_camera_lidar_transfer", "per_view_captions": true, "text_config": { "attention_bias": false, "attention_dropout": 0.0, "bos_token_id": 151643, "dtype": "bfloat16", "eos_token_id": 151645, "head_dim": 128, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 12288, "max_position_embeddings": 262144, "model_type": "qwen3_vl_text", "num_attention_heads": 32, "num_hidden_layers": 36, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_scaling": { "mrope_interleaved": true, "mrope_section": [ 24, 20, 20 ], "rope_type": "default" }, "rope_theta": 5000000, "use_cache": true, "vocab_size": 151936 }, "tie_word_embeddings": false, "transformers_version": "4.57.0.dev0", "video_token_id": 151656, "vision_config": { "deepstack_visual_indexes": [ 8, 16, 24 ], "depth": 27, "hidden_act": "gelu_pytorch_tanh", "hidden_size": 1152, "in_channels": 3, "initializer_range": 0.02, "intermediate_size": 4304, "model_type": "qwen3_vl", "num_heads": 16, "num_position_embeddings": 2304, "out_hidden_size": 4096, "patch_size": 16, "spatial_merge_size": 2, "temporal_patch_size": 2 }, "vision_end_token_id": 151653, "vision_start_token_id": 151652 }