| { |
| "architectures": [ |
| "Cosmos3OmniModel" |
| ], |
| "auto_map": { |
| "AutoConfig": "cosmos3.model.Cosmos3OmniConfig", |
| "AutoModel": "cosmos3.model.Cosmos3OmniModel" |
| }, |
| "dtype": "bfloat16", |
| "model": { |
| "_recursive_": false, |
| "_target_": "cosmos_framework.model.generator.omni_mot_model.OmniMoTModel", |
| "config": { |
| "action_expert": true, |
| "action_gen": true, |
| "action_io_projector_type": "domain_aware", |
| "action_loss_reduction": "mean_over_valid_body_parts_v1", |
| "activation_checkpointing": { |
| "determinism_check": "default", |
| "mode": "full", |
| "preserve_rng_state": true, |
| "save_ops_regex": [ |
| "fmha" |
| ] |
| }, |
| "attention_io_layout": "sequence_sharded", |
| "causal_training_strategy": "none", |
| "compile": { |
| "_type": "cosmos_framework.configs.base.defaults.compile.CompileConfig", |
| "ar_post_saturation_mode": "default", |
| "compile_dynamic": true, |
| "compiled_region": "language", |
| "coordinate_descent_tuning": false, |
| "enabled": false, |
| "max_autotune_pointwise": false, |
| "use_cuda_graphs": false |
| }, |
| "diffusion_expert_config": { |
| "base_fps": 24, |
| "enable_action_modality_embedding": true, |
| "enable_fps_modulation": true, |
| "enable_media_modality_embedding": false, |
| "enable_sound_modality_embedding": true, |
| "enable_vision_modality_embeddings": false, |
| "load_weights_from_pretrained": true, |
| "max_vae_latent_side_after_patchify": 20, |
| "patch_spatial": 2, |
| "sound_base_temporal_compression_factor": null, |
| "timestep_range": 1.0, |
| "unified_3d_mrope_reset_spatial_ids": true, |
| "unified_3d_mrope_temporal_modality_margin": 15000, |
| "vision_temporal_position_mode": "latent_index" |
| }, |
| "ema": { |
| "enabled": false, |
| "iteration_shift": 0, |
| "rate": 0.1 |
| }, |
| "enable_input_bias": true, |
| "exclude_reasoner_weights_from_checkpoint": false, |
| "fixed_step_sampler_config": null, |
| "flex_attention": { |
| "_type": "cosmos_framework.configs.base.defaults.flex_attention.FlexAttentionConfig", |
| "backend": "auto", |
| "enabled": false, |
| "mask": { |
| "attention_scope": "all_views", |
| "decomposed_temporal_window_seconds": null |
| } |
| }, |
| "inference_noise_source": "torch_cuda", |
| "inference_reuse_request_cache": false, |
| "input_caption_key": "ai_caption", |
| "input_image_key": "images", |
| "input_video_key": "video", |
| "joint_attn_implementation": "two_way", |
| "latent_downsample_factor": 16, |
| "lbl": { |
| "coeff_gen": null, |
| "coeff_und": null, |
| "method": "local" |
| }, |
| "lidar_fps": null, |
| "lidar_state_ch": null, |
| "lidar_tokenizer": null, |
| "load_vision_tokenizer": true, |
| "lora_alpha": 32, |
| "lora_enabled": false, |
| "lora_rank": 16, |
| "lora_target_modules": "q_proj_moe_gen,k_proj_moe_gen,v_proj_moe_gen,o_proj_moe_gen", |
| "max_action_dim": 128, |
| "max_num_tokens_after_packing": 45056, |
| "native_action_dim": null, |
| "natten_parameter_list": null, |
| "net": null, |
| "normalize_video_in_model_dtype": true, |
| "num_action_contracts": 45, |
| "num_embodiment_domains": 32, |
| "pad_segment_only_when_padded": true, |
| "parallelism": { |
| "_type": "cosmos_framework.configs.base.defaults.parallelism.ParallelismConfig", |
| "cfg_parallel_shard_degree": 1, |
| "context_parallel_shard_degree": 1, |
| "data_parallel_replicate_degree": 1, |
| "data_parallel_shard_degree": -1, |
| "enable_inference_mode": false, |
| "fsdp_master_dtype": "float32", |
| "fsdp_reduce_dtype": null, |
| "vae_load_balance_group_size": 1 |
| }, |
| "precision": "bfloat16", |
| "quantization": { |
| "_type": "cosmos_framework.configs.base.defaults.quantization.QuantizationConfig", |
| "exclude_regex": [], |
| "fp8_granularity": "per_row", |
| "include_regex": [], |
| "method": null, |
| "mixed_precision_first_steps": 0, |
| "mixed_precision_last_steps": 0, |
| "mixed_precision_reasoner_policy": "high_precision", |
| "mixed_precision_w8a16_cache": "none", |
| "modelopt_fp8_checkpoint_path": null, |
| "modelopt_fp8_target_fqns": [] |
| }, |
| "rectified_flow_inference_config": { |
| "num_train_timesteps": 1000, |
| "scheduler_type": "unipc", |
| "shift": 1, |
| "use_dynamic_shifting": false |
| }, |
| "rectified_flow_training_config": { |
| "action_loss_weight": 10.0, |
| "image_loss_scale": null, |
| "independent_action_schedule": false, |
| "independent_sound_schedule": false, |
| "lidar_loss_scale": null, |
| "loss_scale": 10.0, |
| "normalize_loss_by_active": true, |
| "sample_level_loss_averaging": false, |
| "shift": { |
| "256": 3, |
| "480": 5, |
| "720": 10 |
| }, |
| "shift_action": null, |
| "shift_image": null, |
| "shift_sound": null, |
| "sound_loss_scale": 2.0, |
| "train_time_action_distribution": "logitnormal", |
| "train_time_image_distribution": "logitnormal", |
| "train_time_sound_distribution": "logitnormal", |
| "train_time_video_distribution": "waver", |
| "train_time_weight": "uniform", |
| "use_discrete_rf": false, |
| "use_dynamic_shift": false |
| }, |
| "resolution": "256", |
| "semantic_projectors": true, |
| "sound_dim": null, |
| "sound_gen": false, |
| "sound_latent_fps": 25, |
| "sound_tokenizer": null, |
| "state_ch": 48, |
| "state_t": 300, |
| "time_embedding_per_token": true, |
| "tokenizer": { |
| "_target_": "cosmos_framework.model.generator.tokenizers.wan2pt2_vae_4x16x16.Wan2pt2VAEInterface", |
| "bucket_name": "", |
| "causal": true, |
| "chunk_duration": 93, |
| "encode_bucket_multiple": null, |
| "encode_chunk_frames": { |
| "256": 68, |
| "480": 24, |
| "720": 12 |
| }, |
| "encode_exact_durations": [ |
| 5, |
| 9, |
| 13, |
| 17, |
| 21, |
| 25, |
| 29, |
| 33 |
| ], |
| "keep_decoder_cache": false, |
| "object_store_credential_path_pretrained": "", |
| "spatial_compression_factor": 16, |
| "temporal_compression_factor": 4, |
| "temporal_window": null, |
| "use_streaming_encode": false, |
| "vae_path": "${oc.env:WAN_VAE_PATH}" |
| }, |
| "video_temporal_causal": false, |
| "vision_gen": true, |
| "vlm_config": { |
| "layer_module": null, |
| "model_instance": { |
| "_target_": "cosmos_framework.model.generator.mot.unified_mot.Nemotron3DenseVLTextForCausalLM", |
| "config": { |
| "_target_": "cosmos_framework.configs.base.defaults.reasoner.create_vlm_config", |
| "base_config": { |
| "_target_": "cosmos_framework.model.generator.mot.unified_mot.Nemotron3DenseVLMoTConfig.from_json_file", |
| "json_file": "cosmos_framework/model/generator/reasoner/nemotron_3_dense_vl/configs/Nemotron-2B-Dense-VL.json" |
| }, |
| "freeze_und": false, |
| "include_visual": null, |
| "layer_module": "MoTDecoderLayer", |
| "qk_norm_for_text": false, |
| "tie_word_embeddings": true, |
| "use_und_k_norm_for_gen": true |
| } |
| }, |
| "model_name": "nvidia/Cosmos3-Edge-Reasoner", |
| "pretrained_weights": { |
| "backbone_path": "", |
| "checkpoint_format": null, |
| "credentials_path": "", |
| "enable_gcs_patch_in_boto3": true, |
| "enabled": false |
| }, |
| "qk_norm": false, |
| "safetensors_path": "", |
| "tie_word_embeddings": false, |
| "tokenizer": { |
| "_target_": "cosmos_framework.data.generator.processors.build_processor_lazy", |
| "repository": null, |
| "revision": null, |
| "subdir": "", |
| "tokenizer_type": "${oc.env:COSMOS3_EDGE_DIR}" |
| }, |
| "use_system_prompt": false |
| } |
| } |
| }, |
| "model_type": "cosmos3_omni", |
| "transformers_version": "4.57.6" |
| } |
|
|
