File size: 8,142 Bytes
9459a45 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 | {
"architectures": [
"Cosmos3OmniModel"
],
"auto_map": {
"AutoConfig": "cosmos3.model.Cosmos3OmniConfig",
"AutoModel": "cosmos3.model.Cosmos3OmniModel"
},
"dtype": "bfloat16",
"model": {
"_recursive_": false,
"_target_": "greenwam.callbacks.evaluation.build_validation_model",
"config": {
"action_expert": true,
"action_gen": true,
"action_io_projector_type": "domain_aware",
"action_loss_reduction": "mean_over_valid_coordinates_v1",
"activation_checkpointing": {
"determinism_check": "default",
"mode": "selective",
"preserve_rng_state": true,
"save_ops_regex": [
"fmha"
]
},
"attention_io_layout": "sequence_sharded",
"causal_training_strategy": "none",
"compile": {
"_type": "cosmos_framework.configs.base.defaults.compile.CompileConfig",
"ar_post_saturation_mode": "default",
"compile_dynamic": true,
"compiled_region": "language",
"coordinate_descent_tuning": false,
"enabled": false,
"max_autotune_pointwise": false,
"use_cuda_graphs": false
},
"diffusion_expert_config": {
"base_fps": 24,
"enable_action_modality_embedding": true,
"enable_fps_modulation": true,
"enable_media_modality_embedding": false,
"enable_sound_modality_embedding": true,
"enable_vision_modality_embeddings": false,
"load_weights_from_pretrained": false,
"max_vae_latent_side_after_patchify": 20,
"patch_spatial": 2,
"sound_base_temporal_compression_factor": null,
"timestep_range": 1.0,
"unified_3d_mrope_reset_spatial_ids": true,
"unified_3d_mrope_temporal_modality_margin": 15000,
"vision_temporal_position_mode": "latent_index"
},
"ema": {
"enabled": true,
"iteration_shift": 0,
"rate": 0.1
},
"enable_input_bias": true,
"exclude_reasoner_weights_from_checkpoint": false,
"fixed_step_sampler_config": null,
"flex_attention": {
"backend": "auto",
"enabled": false,
"mask": {
"attention_scope": "all_views",
"decomposed_temporal_window_seconds": null
}
},
"inference_noise_source": "numpy",
"inference_reuse_request_cache": true,
"input_caption_key": "ai_caption",
"input_image_key": "images",
"input_video_key": "video",
"joint_attn_implementation": "two_way",
"latent_downsample_factor": 16,
"lbl": {
"coeff_gen": null,
"coeff_und": null,
"method": "local"
},
"lidar_fps": null,
"lidar_state_ch": null,
"lidar_tokenizer": null,
"load_vision_tokenizer": true,
"lora_alpha": 32,
"lora_enabled": false,
"lora_rank": 16,
"lora_target_modules": "q_proj_moe_gen,k_proj_moe_gen,v_proj_moe_gen,o_proj_moe_gen",
"max_action_dim": 128,
"max_num_tokens_after_packing": 45056,
"native_action_dim": 64,
"natten_parameter_list": null,
"net": null,
"normalize_video_in_model_dtype": false,
"num_action_contracts": 90,
"num_embodiment_domains": 32,
"pad_segment_only_when_padded": false,
"parallelism": {
"_type": "cosmos_framework.configs.base.defaults.parallelism.ParallelismConfig",
"cfg_parallel_shard_degree": 1,
"context_parallel_shard_degree": 1,
"data_parallel_replicate_degree": 1,
"data_parallel_shard_degree": -1,
"enable_inference_mode": false,
"fsdp_master_dtype": "float32",
"fsdp_reduce_dtype": null,
"vae_load_balance_group_size": 1
},
"precision": "bfloat16",
"quantization": {
"_type": "cosmos_framework.configs.base.defaults.quantization.QuantizationConfig",
"exclude_regex": [],
"fp8_granularity": "per_row",
"include_regex": [],
"method": null,
"mixed_precision_first_steps": 0,
"mixed_precision_last_steps": 0,
"mixed_precision_reasoner_policy": "high_precision",
"mixed_precision_w8a16_cache": "none",
"modelopt_fp8_checkpoint_path": null,
"modelopt_fp8_target_fqns": []
},
"rectified_flow_inference_config": {
"num_train_timesteps": 1000,
"scheduler_type": "unipc",
"shift": 1,
"use_dynamic_shifting": false
},
"rectified_flow_training_config": {
"action_loss_weight": 10.0,
"image_loss_scale": null,
"independent_action_schedule": false,
"independent_sound_schedule": false,
"lidar_loss_scale": null,
"loss_scale": 10.0,
"normalize_loss_by_active": true,
"sample_level_loss_averaging": false,
"shift": {
"256": 3,
"480": 5,
"720": 10
},
"shift_action": null,
"shift_image": null,
"shift_sound": null,
"sound_loss_scale": 2.0,
"train_time_action_distribution": "logitnormal",
"train_time_image_distribution": "logitnormal",
"train_time_sound_distribution": "logitnormal",
"train_time_video_distribution": "waver",
"train_time_weight": "uniform",
"use_discrete_rf": false,
"use_dynamic_shift": false
},
"resolution": "480",
"semantic_projectors": true,
"sound_dim": null,
"sound_gen": false,
"sound_latent_fps": 25,
"sound_tokenizer": null,
"state_ch": 48,
"state_t": 300,
"time_embedding_per_token": false,
"tokenizer": {
"_target_": "cosmos_framework.model.generator.tokenizers.wan2pt2_vae_4x16x16.Wan2pt2VAEInterface",
"bucket_name": "",
"causal": true,
"chunk_duration": 93,
"encode_bucket_multiple": null,
"encode_chunk_frames": {
"256": 68,
"480": 24,
"720": 8,
"768": 8
},
"encode_exact_durations": [
1,
5,
9,
13,
17,
21,
25,
29,
33
],
"keep_decoder_cache": false,
"object_store_credential_path_pretrained": "",
"spatial_compression_factor": 16,
"temporal_compression_factor": 4,
"temporal_window": null,
"use_streaming_encode": false,
"vae_path": "${oc.env:WAN_VAE_PATH}"
},
"video_temporal_causal": false,
"vision_gen": true,
"vlm_config": {
"layer_module": null,
"model_instance": {
"_target_": "cosmos_framework.model.generator.mot.unified_mot.Nemotron3DenseVLTextForCausalLM",
"config": {
"_target_": "cosmos_framework.configs.base.defaults.reasoner.create_vlm_config",
"base_config": {
"_target_": "cosmos_framework.model.generator.mot.unified_mot.Nemotron3DenseVLMoTConfig.from_json_file",
"json_file": "cosmos_framework/model/generator/reasoner/nemotron_3_dense_vl/configs/Nemotron-2B-Dense-VL.json"
},
"freeze_und": false,
"include_visual": null,
"layer_module": "MoTDecoderLayer",
"qk_norm_for_text": false,
"tie_word_embeddings": true,
"use_und_k_norm_for_gen": true
}
},
"model_name": "nvidia/Cosmos3-Edge-Reasoner",
"pretrained_weights": {
"backbone_path": "",
"checkpoint_format": null,
"credentials_path": "",
"enable_gcs_patch_in_boto3": true,
"enabled": false
},
"qk_norm": false,
"safetensors_path": "",
"tie_word_embeddings": false,
"tokenizer": {
"_target_": "cosmos_framework.data.generator.processors.build_processor_lazy",
"repository": null,
"revision": null,
"subdir": "",
"tokenizer_type": "${oc.env:COSMOS3_EDGE_DIR}"
},
"use_system_prompt": false
}
}
},
"model_type": "cosmos3_omni",
"transformers_version": "4.57.6"
}
|