{ "transformers_version": "5.23.0", "architectures": [ "HYV4FlashForConditionalGeneration" ], "dtype": "bfloat16", "model_type": "hy_v4_flash", "vision_start_token_id": 120118, "spatial_merge_size": 2, "vision_config": { "add_patchemb_bias": true, "attention_dropout": 0.0, "cat_extra_token": 0, "hidden_act": "gelu", "hidden_dropout": 0.0, "hidden_size": 1152, "img_max_token_num": 4096, "intermediate_size": 4304, "interpolate_mode": "bilinear", "max_image_size": 2048, "max_vit_seq_len": 16384, "num_attention_heads": 16, "num_channels": 3, "num_hidden_layers": 27, "out_hidden_size": 3072, "patch_size": 16, "rms_norm_eps": 1e-05, "temporal_patch_size": 1, "spatial_patch_size": 1, "spatial_merge_size": 2, "perceive_post_norm": false, "video_spatial_compression": false, "video_temporal_compression": false, "projector_type": "rmsnorm_mlp_adapter" }, "text_config": { "architectures": [ "HYV4FlashForCausalLM" ], "vocab_size": 120832, "hidden_size": 3072, "intermediate_size": 13824, "moe_intermediate_size": 1536, "routed_expert_hidden_size": 1536, "num_hidden_layers": 42, "num_attention_heads": 32, "num_key_value_heads": 32, "n_shared_experts": 1, "n_routed_experts": 512, "routed_scaling_factor": 3.998, "kv_lora_rank": 512, "q_lora_rank": 1024, "qk_rope_head_dim": 64, "v_head_dim": 256, "qk_nope_head_dim": 192, "swa_num_attention_heads": 64, "swa_qk_rope_head_dim": 64, "swa_v_head_dim": 256, "swa_qk_nope_head_dim": 192, "n_group": 1, "topk_group": 1, "num_experts_per_tok": 16, "norm_topk_prob": true, "moe_aux_loss_coeff": 0.0001, "hidden_act": "silu", "max_position_embeddings": 1048576, "initializer_range": 0.006, "rms_norm_eps": 1e-05, "use_cache": true, "pad_token_id": 120002, "bos_token_id": 120000, "eos_token_id": 120020, "tie_word_embeddings": false, "rope_parameters": { "deepseek_sparse_attention": { "rope_theta": 10000000, "rope_type": "default", "mrope_section": [ 12, 10, 10 ], "mrope_interleaved": true }, "sliding_attention": { "rope_theta": 10000, "rope_type": "default", "mrope_section": [ 12, 10, 10 ], "mrope_interleaved": true } }, "first_k_dense_replace": 1, "mlp_layer_types": [ "dense", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse" ], "layer_types": [ "deepseek_sparse_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention" ], "attention_bias": false, "attention_dropout": 0.0, "swa_pattern": [ 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0 ], "window_size": [ 511, 0 ], "sliding_window": 512, "full_nope": false, "index_topk": 2048, "index_head_dim": 128, "index_n_heads": 16, "use_dsa": true, "use_compressed_indexer": true, "compressed_indexer_compress_ratio": 4, "compressed_indexer_num_attention_heads": 16, "compressed_indexer_head_dim": 128, "compressed_indexer_group_topk": 512, "compressed_indexer_candidate_topk": 2048, "compressed_indexer_stage2_query_block_size": 8192, "compressed_indexer_use_triton_pooling": true, "compressed_indexer_triton_pooling_min_groups": 1, "compressed_indexer_reuse_fine_indexer": false, "compressed_indexer_skip_fine_select": true, "compressed_indexer_key_pooling": "gated", "indexer_types": [ "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full" ], "enable_ihc": true, "enable_lm_head_fp32": true, "bitwise_backward_align": true, "hc_mult": 4, "hc_magnitude": 2.0, "hc_eps": 1e-06, "sandwich_norm": true, "return_max_logit": true, "return_bos_score": true, "engram_fp8_embedding": true, "fast_op_fp32_linear_gemm": true, "dsa_skip_attn_post_norm": false, "gated_mla": true, "gating_type": "elementwise", "gate_lora_rank": null, "gate_groups": 1, "gate_lora_op": "none", "o_lora_rank": null, "o_groups": 1, "o_lora_op": "none", "learnable_sink": true, "learnable_sink_init": 0.0, "swiglu_limit": 10.0, "use_grouped_mm": false, "num_nextn_predict_layers": 0, "use_mla": true, "qk_norm": true, "moe_router_use_sigmoid": true, "moe_router_enable_expert_bias": true, "output_router_logits": false, "engram_layer_ids": [ 2, 12, 22, 32 ], "engram_vocab_size": [ 10312000, 10312000 ], "engram_max_ngram_size": 3, "engram_n_embed_per_ngram": 768, "engram_n_head_per_ngram": 8, "engram_embed_random_seed": 1234, "engram_seed": 0, "engram_pad_id": 120002, "engram_isolate_fffd_namespace": false, "engram_layer_head_vocab_sizes": { "2": [ [ 10312007, 10312021, 10312039, 10312069, 10312091, 10312121, 10312123, 10312147 ], [ 10312177, 10312187, 10312207, 10312213, 10312229, 10312241, 10312249, 10312259 ] ], "12": [ [ 10312277, 10312283, 10312301, 10312343, 10312373, 10312397, 10312409, 10312453 ], [ 10312457, 10312483, 10312507, 10312513, 10312531, 10312543, 10312567, 10312571 ] ], "22": [ [ 10312613, 10312619, 10312633, 10312657, 10312681, 10312699, 10312723, 10312747 ], [ 10312759, 10312789, 10312811, 10312823, 10312867, 10312873, 10312889, 10312903 ] ], "32": [ [ 10312909, 10312919, 10312927, 10312957, 10312979, 10313011, 10313021, 10313057 ], [ 10313063, 10313117, 10313123, 10313129, 10313161, 10313179, 10313189, 10313201 ] ] }, "qk_head_dim": 256, "swa_qk_head_dim": 256, "head_dim": 64, "model_type": "hy_v4_flash" }, "image_token_id": 120120, "image_start_token_id": 120118, "image_end_token_id": 120119, "video_token_id": 120124, "video_start_token_id": 120122, "video_end_token_id": 120123, "image_newline_token_id": 120121, "processor_class": "HYV4FlashProcessor" }