{ "architectures": [ "Step5ForConditionalGeneration" ], "model_type": "step5_vl", "im_end_token": "", "im_patch_token": "", "im_start_token": "", "image_token_len": 169, "patch_token_len": 81, "image_token_id": 128001, "understand_projector_stride": 2, "use_im_start_end": "true", "vision_select_layer": -1, "projector_bias": false, "vision_config": { "model_type": "perception_encoder", "image_size": 728, "patch_size": 14, "width": 1536, "layers": 47, "heads": 16, "pool_type": "none", "output_dim": null, "use_cls_token": false, "ls_init_value": 0.1, "use_ln_post": false, "hidden_act": "quick_gelu" }, "text_config": { "architectures": [ "Step5ForCausalLM" ], "model_type": "step5", "att_impl_type": "GQA", "torch_dtype": "bfloat16", "norm_dtype": "float32", "bos_token_id": 0, "eos_token_id": [ 1, 2 ], "hidden_size": 4096, "intermediate_size": 13824, "num_hidden_layers": 92, "vocab_size": 128896, "rms_norm_eps": 1e-05, "fp32_residual_connection": true, "num_attention_heads": 64, "num_attention_groups": 4, "num_sliding_attention_heads": 64, "head_dim": 192, "head_dim_full_attention": 192, "head_dim_sliding_attention": 192, "num_attention_groups_full_attention": 4, "num_attention_groups_sliding_attention": 4, "rope_theta": [ 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0 ], "layer_types": [ "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention" ], "partial_rotary_factors": [ 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0 ], "sliding_window": 512, "attention_other_setting": { "attention_type": "sliding_attention", "num_attention_heads": 64, "num_attention_groups": 4, "head_dim": 192, "true_head_dim": 192 }, "use_head_wise_attn_gate": true, "num_nextn_predict_layers": 3, "use_moe": true, "moe_intermediate_size": 1536, "share_expert_dim": 1536, "moe_num_experts": 352, "moe_top_k": 8, "num_experts_per_tok": 8, "moe_layer_list": [ 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90 ], "moe_layers_enum": "3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90", "use_moe_router_bias": true, "need_fp32_gate": true, "swiglu_limits": [ 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0 ], "swiglu_limits_shared": [ 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0 ], "max_seq_len": 1048576, "max_position_embeddings": 1048576, "norm_expert_weight": true, "moe_router_scaling_factor": 3.0, "rope_scaling": { "rope_type": "llama3", "factor": 1.0, "original_max_position_embeddings": 1048576, "low_freq_factor": 1.0, "high_freq_factor": 32.0 }, "yarn_only_types": [ "full_attention" ], "sparse_config": { "enabled": true, "proxy_dim": 256, "sparse_indexer_rope_dim": 32, "sparse_indexer_use_rope": true, "sparse_indexer_num_heads": 16, "sparse_indexer_num_k_heads": 1, "sparse_indexer_q_norm_type": "rmsnorm", "sparse_indexer_k_norm_type": "layernorm", "sparse_indexer_csa_z_norm_type": "none", "sparse_indexer_softmax_variant": "ssmax", "sparse_indexer_ssmax_s_granularity": "q_head", "num_provider_groups": 4, "topk": 512, "region_block_size": 8, "compression_method": "csa_block_compress", "attention_impl": "sparse_gqa", "apply_to_layer_types": [ "full_attention" ] } }, "quantization_config": { "quant_method": "modelopt", "quant_algo": "NVFP4", "group_size": 16, "kv_cache_quant_algo": null, "ignore": [ "lm_head", "model.embed_tokens", "model.norm", "*layers.0.self_attn*", "*layers.0.mlp*", "*layers.1.self_attn*", "*layers.1.mlp*", "*layers.2.self_attn*", "*layers.2.mlp*", "*layers.3.self_attn*", "*layers.3.share_expert*", "*layers.3.moe.share_expert*", "*layers.3.moe.gate*", "*layers.4.self_attn*", "*layers.4.share_expert*", "*layers.4.moe.share_expert*", "*layers.4.moe.gate*", "*layers.5.self_attn*", "*layers.5.share_expert*", "*layers.5.moe.share_expert*", "*layers.5.moe.gate*", "*layers.6.self_attn*", "*layers.6.share_expert*", "*layers.6.moe.share_expert*", "*layers.6.moe.gate*", "*layers.7.self_attn*", "*layers.7.share_expert*", "*layers.7.moe.share_expert*", "*layers.7.moe.gate*", "*layers.8.self_attn*", "*layers.8.share_expert*", "*layers.8.moe.share_expert*", "*layers.8.moe.gate*", "*layers.9.self_attn*", "*layers.9.share_expert*", "*layers.9.moe.share_expert*", "*layers.9.moe.gate*", "*layers.10.self_attn*", "*layers.10.share_expert*", "*layers.10.moe.share_expert*", "*layers.10.moe.gate*", "*layers.11.self_attn*", "*layers.11.share_expert*", "*layers.11.moe.share_expert*", "*layers.11.moe.gate*", "*layers.12.self_attn*", "*layers.12.share_expert*", "*layers.12.moe.share_expert*", "*layers.12.moe.gate*", "*layers.13.self_attn*", "*layers.13.share_expert*", "*layers.13.moe.share_expert*", "*layers.13.moe.gate*", "*layers.14.self_attn*", "*layers.14.share_expert*", "*layers.14.moe.share_expert*", "*layers.14.moe.gate*", "*layers.15.self_attn*", "*layers.15.share_expert*", "*layers.15.moe.share_expert*", "*layers.15.moe.gate*", "*layers.16.self_attn*", "*layers.16.share_expert*", "*layers.16.moe.share_expert*", "*layers.16.moe.gate*", "*layers.17.self_attn*", "*layers.17.share_expert*", "*layers.17.moe.share_expert*", "*layers.17.moe.gate*", "*layers.18.self_attn*", "*layers.18.share_expert*", "*layers.18.moe.share_expert*", "*layers.18.moe.gate*", "*layers.19.self_attn*", "*layers.19.share_expert*", "*layers.19.moe.share_expert*", "*layers.19.moe.gate*", "*layers.20.self_attn*", "*layers.20.share_expert*", "*layers.20.moe.share_expert*", "*layers.20.moe.gate*", "*layers.21.self_attn*", "*layers.21.share_expert*", "*layers.21.moe.share_expert*", "*layers.21.moe.gate*", "*layers.22.self_attn*", "*layers.22.share_expert*", "*layers.22.moe.share_expert*", "*layers.22.moe.gate*", "*layers.23.self_attn*", "*layers.23.share_expert*", "*layers.23.moe.share_expert*", "*layers.23.moe.gate*", "*layers.24.self_attn*", "*layers.24.share_expert*", "*layers.24.moe.share_expert*", "*layers.24.moe.gate*", "*layers.25.self_attn*", "*layers.25.share_expert*", "*layers.25.moe.share_expert*", "*layers.25.moe.gate*", "*layers.26.self_attn*", "*layers.26.share_expert*", "*layers.26.moe.share_expert*", "*layers.26.moe.gate*", "*layers.27.self_attn*", "*layers.27.share_expert*", "*layers.27.moe.share_expert*", "*layers.27.moe.gate*", "*layers.28.self_attn*", "*layers.28.share_expert*", "*layers.28.moe.share_expert*", "*layers.28.moe.gate*", "*layers.29.self_attn*", "*layers.29.share_expert*", "*layers.29.moe.share_expert*", "*layers.29.moe.gate*", "*layers.30.self_attn*", "*layers.30.share_expert*", "*layers.30.moe.share_expert*", "*layers.30.moe.gate*", "*layers.31.self_attn*", "*layers.31.share_expert*", "*layers.31.moe.share_expert*", "*layers.31.moe.gate*", "*layers.32.self_attn*", "*layers.32.share_expert*", "*layers.32.moe.share_expert*", "*layers.32.moe.gate*", "*layers.33.self_attn*", "*layers.33.share_expert*", "*layers.33.moe.share_expert*", "*layers.33.moe.gate*", "*layers.34.self_attn*", "*layers.34.share_expert*", "*layers.34.moe.share_expert*", "*layers.34.moe.gate*", "*layers.35.self_attn*", "*layers.35.share_expert*", "*layers.35.moe.share_expert*", "*layers.35.moe.gate*", "*layers.36.self_attn*", "*layers.36.share_expert*", "*layers.36.moe.share_expert*", "*layers.36.moe.gate*", "*layers.37.self_attn*", "*layers.37.share_expert*", "*layers.37.moe.share_expert*", "*layers.37.moe.gate*", "*layers.38.self_attn*", "*layers.38.share_expert*", "*layers.38.moe.share_expert*", "*layers.38.moe.gate*", "*layers.39.self_attn*", "*layers.39.share_expert*", "*layers.39.moe.share_expert*", "*layers.39.moe.gate*", "*layers.40.self_attn*", "*layers.40.share_expert*", "*layers.40.moe.share_expert*", "*layers.40.moe.gate*", "*layers.41.self_attn*", "*layers.41.share_expert*", "*layers.41.moe.share_expert*", "*layers.41.moe.gate*", "*layers.42.self_attn*", "*layers.42.share_expert*", "*layers.42.moe.share_expert*", "*layers.42.moe.gate*", "*layers.43.self_attn*", "*layers.43.share_expert*", "*layers.43.moe.share_expert*", "*layers.43.moe.gate*", "*layers.44.self_attn*", "*layers.44.share_expert*", "*layers.44.moe.share_expert*", "*layers.44.moe.gate*", "*layers.45.self_attn*", "*layers.45.share_expert*", "*layers.45.moe.share_expert*", "*layers.45.moe.gate*", "*layers.46.self_attn*", "*layers.46.share_expert*", "*layers.46.moe.share_expert*", "*layers.46.moe.gate*", "*layers.47.self_attn*", "*layers.47.share_expert*", "*layers.47.moe.share_expert*", "*layers.47.moe.gate*", "*layers.48.self_attn*", "*layers.48.share_expert*", "*layers.48.moe.share_expert*", "*layers.48.moe.gate*", "*layers.49.self_attn*", "*layers.49.share_expert*", "*layers.49.moe.share_expert*", "*layers.49.moe.gate*", "*layers.50.self_attn*", "*layers.50.share_expert*", "*layers.50.moe.share_expert*", "*layers.50.moe.gate*", "*layers.51.self_attn*", "*layers.51.share_expert*", "*layers.51.moe.share_expert*", "*layers.51.moe.gate*", "*layers.52.self_attn*", "*layers.52.share_expert*", "*layers.52.moe.share_expert*", "*layers.52.moe.gate*", "*layers.53.self_attn*", "*layers.53.share_expert*", "*layers.53.moe.share_expert*", "*layers.53.moe.gate*", "*layers.54.self_attn*", "*layers.54.share_expert*", "*layers.54.moe.share_expert*", "*layers.54.moe.gate*", "*layers.55.self_attn*", "*layers.55.share_expert*", "*layers.55.moe.share_expert*", "*layers.55.moe.gate*", "*layers.56.self_attn*", "*layers.56.share_expert*", "*layers.56.moe.share_expert*", "*layers.56.moe.gate*", "*layers.57.self_attn*", "*layers.57.share_expert*", "*layers.57.moe.share_expert*", "*layers.57.moe.gate*", "*layers.58.self_attn*", "*layers.58.share_expert*", "*layers.58.moe.share_expert*", "*layers.58.moe.gate*", "*layers.59.self_attn*", "*layers.59.share_expert*", "*layers.59.moe.share_expert*", "*layers.59.moe.gate*", "*layers.60.self_attn*", "*layers.60.share_expert*", "*layers.60.moe.share_expert*", "*layers.60.moe.gate*", "*layers.61.self_attn*", "*layers.61.share_expert*", "*layers.61.moe.share_expert*", "*layers.61.moe.gate*", "*layers.62.self_attn*", "*layers.62.share_expert*", "*layers.62.moe.share_expert*", "*layers.62.moe.gate*", "*layers.63.self_attn*", "*layers.63.share_expert*", "*layers.63.moe.share_expert*", "*layers.63.moe.gate*", "*layers.64.self_attn*", "*layers.64.share_expert*", "*layers.64.moe.share_expert*", "*layers.64.moe.gate*", "*layers.65.self_attn*", "*layers.65.share_expert*", "*layers.65.moe.share_expert*", "*layers.65.moe.gate*", "*layers.66.self_attn*", "*layers.66.share_expert*", "*layers.66.moe.share_expert*", "*layers.66.moe.gate*", "*layers.67.self_attn*", "*layers.67.share_expert*", "*layers.67.moe.share_expert*", "*layers.67.moe.gate*", "*layers.68.self_attn*", "*layers.68.share_expert*", "*layers.68.moe.share_expert*", "*layers.68.moe.gate*", "*layers.69.self_attn*", "*layers.69.share_expert*", "*layers.69.moe.share_expert*", "*layers.69.moe.gate*", "*layers.70.self_attn*", "*layers.70.share_expert*", "*layers.70.moe.share_expert*", "*layers.70.moe.gate*", "*layers.71.self_attn*", "*layers.71.share_expert*", "*layers.71.moe.share_expert*", "*layers.71.moe.gate*", "*layers.72.self_attn*", "*layers.72.share_expert*", "*layers.72.moe.share_expert*", "*layers.72.moe.gate*", "*layers.73.self_attn*", "*layers.73.share_expert*", "*layers.73.moe.share_expert*", "*layers.73.moe.gate*", "*layers.74.self_attn*", "*layers.74.share_expert*", "*layers.74.moe.share_expert*", "*layers.74.moe.gate*", "*layers.75.self_attn*", "*layers.75.share_expert*", "*layers.75.moe.share_expert*", "*layers.75.moe.gate*", "*layers.76.self_attn*", "*layers.76.share_expert*", "*layers.76.moe.share_expert*", "*layers.76.moe.gate*", "*layers.77.self_attn*", "*layers.77.share_expert*", "*layers.77.moe.share_expert*", "*layers.77.moe.gate*", "*layers.78.self_attn*", "*layers.78.share_expert*", "*layers.78.moe.share_expert*", "*layers.78.moe.gate*", "*layers.79.self_attn*", "*layers.79.share_expert*", "*layers.79.moe.share_expert*", "*layers.79.moe.gate*", "*layers.80.self_attn*", "*layers.80.share_expert*", "*layers.80.moe.share_expert*", "*layers.80.moe.gate*", "*layers.81.self_attn*", "*layers.81.share_expert*", "*layers.81.moe.share_expert*", "*layers.81.moe.gate*", "*layers.82.self_attn*", "*layers.82.share_expert*", "*layers.82.moe.share_expert*", "*layers.82.moe.gate*", "*layers.83.self_attn*", "*layers.83.share_expert*", "*layers.83.moe.share_expert*", "*layers.83.moe.gate*", "*layers.84.self_attn*", "*layers.84.share_expert*", "*layers.84.moe.share_expert*", "*layers.84.moe.gate*", "*layers.85.self_attn*", "*layers.85.share_expert*", "*layers.85.moe.share_expert*", "*layers.85.moe.gate*", "*layers.86.self_attn*", "*layers.86.share_expert*", "*layers.86.moe.share_expert*", "*layers.86.moe.gate*", "*layers.87.self_attn*", "*layers.87.share_expert*", "*layers.87.moe.share_expert*", "*layers.87.moe.gate*", "*layers.88.self_attn*", "*layers.88.share_expert*", "*layers.88.moe.share_expert*", "*layers.88.moe.gate*", "*layers.88.moe.experts*", "*layers.89.self_attn*", "*layers.89.share_expert*", "*layers.89.moe.share_expert*", "*layers.89.moe.gate*", "*layers.89.moe.experts*", "*layers.90.self_attn*", "*layers.90.share_expert*", "*layers.90.moe.share_expert*", "*layers.90.moe.gate*", "*layers.90.moe.experts*", "*layers.91.self_attn*", "*layers.91.mlp*", "*layers.92.*", "*layers.93.*", "*layers.94.*", "vision_model*", "vit_large_projector*" ], "producer": { "name": "modelopt", "version": "0.47.0" } } }