Hy4-FP8 / config.json
yiqichen01's picture
update
d063b58
Raw History Blame Contribute Delete
14 kB
{
"architectures": [
"HYV4VLForConditionalGeneration"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bitwise_backward_align": false,
"bos_token_id": 120000,
"enable_ihc": true,
"enable_lm_head_fp32": true,
"eos_token_id": 120025,
"first_k_dense_replace": 1,
"gated_mla": true,
"gating_type": "elementwise",
"hc_eps": 1e-06,
"hc_magnitude": 2.0,
"hc_mult": 4,
"head_dim": 64,
"hidden_act": "silu",
"hidden_size": 6144,
"image_end_token_id": 120119,
"image_newline_token_id": 120121,
"image_start_token_id": 120118,
"image_token_id": 120120,
"index_head_dim": 128,
"index_n_heads": 32,
"index_topk": 2048,
"indexer_types": [
"full",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full"
],
"initializer_range": 0.006,
"intermediate_size": 18432,
"kv_lora_rank": 512,
"learnable_sink": true,
"learnable_sink_init": 0.0,
"max_position_embeddings": 1048576,
"model_type": "hy_v4_vl",
"moe_intermediate_size": 2048,
"n_group": 1,
"n_routed_experts": 256,
"n_shared_experts": 1,
"norm_topk_prob": true,
"num_attention_heads": 64,
"num_experts_per_tok": 8,
"num_hidden_layers": 78,
"num_key_value_heads": 8,
"num_nextn_predict_layers": 0,
"pad_token_id": 120002,
"q_lora_rank": 2048,
"qk_head_dim": 256,
"qk_nope_head_dim": 192,
"qk_norm": true,
"qk_rope_head_dim": 64,
"rms_norm_eps": 1e-05,
"rope_parameters": {
"mrope_interleaved": true,
"mrope_section": [
12,
10,
10
],
"rope_theta": 10000000,
"rope_type": "default"
},
"routed_scaling_factor": 2.827,
"swiglu_limit": 10.0,
"text_config": {
"architectures": [
"HYV4VLForConditionalGeneration"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bitwise_backward_align": false,
"bos_token_id": 120000,
"enable_ihc": true,
"enable_lm_head_fp32": true,
"eos_token_id": 120025,
"first_k_dense_replace": 1,
"gated_mla": true,
"gating_type": "elementwise",
"hc_eps": 1e-06,
"hc_magnitude": 2.0,
"hc_mult": 4,
"head_dim": 64,
"hidden_act": "silu",
"hidden_size": 6144,
"image_newline_token_id": 120121,
"index_head_dim": 128,
"index_n_heads": 32,
"index_topk": 2048,
"indexer_types": [
"full",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full",
"shared",
"shared",
"shared",
"full"
],
"initializer_range": 0.006,
"intermediate_size": 18432,
"kv_lora_rank": 512,
"layer_types": [
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention",
"deepseek_sparse_attention"
],
"learnable_sink": true,
"learnable_sink_init": 0.0,
"max_position_embeddings": 1048576,
"mlp_layer_types": [
"dense",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse"
],
"model_type": "hy_v4",
"moe_intermediate_size": 2048,
"n_group": 1,
"n_routed_experts": 256,
"n_shared_experts": 1,
"norm_topk_prob": true,
"num_attention_heads": 64,
"num_experts_per_tok": 8,
"num_hidden_layers": 78,
"num_key_value_heads": 8,
"num_nextn_predict_layers": 0,
"pad_token_id": 120002,
"q_lora_rank": 2048,
"qk_head_dim": 256,
"qk_nope_head_dim": 192,
"qk_norm": true,
"qk_rope_head_dim": 64,
"rms_norm_eps": 1e-05,
"rope_parameters": {
"mrope_interleaved": true,
"mrope_section": [
12,
10,
10
],
"rope_theta": 10000000,
"rope_type": "default"
},
"routed_scaling_factor": 2.827,
"swiglu_limit": 10.0,
"tie_word_embeddings": false,
"topk_group": 1,
"use_cache": true,
"v_head_dim": 256,
"video_token_id": 120124,
"vocab_size": 120832
},
"tie_word_embeddings": false,
"topk_group": 1,
"transformers_version": "5.24.0",
"v_head_dim": 256,
"video_end_token_id": 120123,
"video_start_token_id": 120122,
"video_token_id": 120124,
"vision_config": {
"add_patchemb_bias": true,
"anyres_vit_max_image_size": 2048,
"attention_dropout": 0.0,
"cat_extra_token": 0,
"hidden_act": "gelu",
"hidden_dropout": 0.0,
"hidden_size": 1536,
"img_max_token_num": 4096,
"intermediate_size": 6144,
"interpolate_mode": "bilinear",
"learnable_mlp_pooling_size": 0,
"max_image_size": 2048,
"max_vit_seq_len": 16384,
"min_image_size": 512,
"model_type": "hy_v4_vl",
"num_attention_heads": 16,
"num_channels": 3,
"num_hidden_layers": 40,
"num_key_value_heads": 16,
"out_hidden_size": 6144,
"patch_size": 16,
"perceive_post_norm": false,
"perceive_pre_norm": true,
"projector_type": "rmsnorm_mlp_adapter",
"remove_prenorm": true,
"resize_resolution": 2048,
"rms_norm_eps": 1e-05,
"rope_2d_percent": 1.0,
"rope_2d_theta": 10000.0,
"spatial_merge_size": 2,
"spatial_patch_size": 1,
"temporal_patch_size": 1,
"text_hidden_size": 6144,
"video_max_image_size": 768,
"video_min_image_size": 256,
"video_spatial_compression": false,
"video_temporal_compression": false,
"vision_full_attention": false
},
"vocab_size": 120832,
"quantization_config": {
"quant_method": "compressed-tensors",
"format": "float-quantized",
"quantization_status": "compressed",
"global_compression_ratio": null,
"kv_cache_scheme": null,
"config_groups": {
"block_fp8_w8a8": {
"weights": {
"num_bits": 8,
"type": "float",
"strategy": "block",
"block_structure": [
128,
128
],
"symmetric": true,
"dynamic": false,
"observer": "minmax"
},
"input_activations": {
"num_bits": 8,
"type": "float",
"strategy": "group",
"group_size": 128,
"symmetric": true,
"dynamic": true,
"observer": null
},
"output_activations": null,
"targets": [
"re:(?:language_model\\.)?model\\.layers\\.\\d+\\.self_attn\\.(?:o_proj|linear_gate)$",
"re:(?:language_model\\.)?model\\.layers\\.\\d+\\.mlp\\.experts\\.\\d+\\.(?:gate_proj|up_proj|down_proj)$",
"re:(?:language_model\\.)?model\\.dfly_layers\\.\\d+\\.mlp\\.down_proj$",
"re:(?:language_model\\.)?model\\.dfly_layers\\.\\d+\\.mlp\\.gate_proj$",
"re:(?:language_model\\.)?model\\.dfly_layers\\.\\d+\\.mlp\\.up_proj$",
"re:(?:language_model\\.)?model\\.dfly_layers\\.\\d+\\.self_attn\\.kv_a_proj_with_mqa$",
"re:(?:language_model\\.)?model\\.dfly_layers\\.\\d+\\.self_attn\\.kv_b_proj$",
"re:(?:language_model\\.)?model\\.dfly_layers\\.\\d+\\.self_attn\\.linear_gate$",
"re:(?:language_model\\.)?model\\.dfly_layers\\.\\d+\\.self_attn\\.o_proj$",
"re:(?:language_model\\.)?model\\.dfly_layers\\.\\d+\\.self_attn\\.q_a_proj$",
"re:(?:language_model\\.)?model\\.dfly_layers\\.\\d+\\.self_attn\\.q_b_proj$",
"re:(?:language_model\\.)?model\\.dfly_layers\\.hidden_correction\\.down_proj$",
"re:(?:language_model\\.)?model\\.dfly_layers\\.hidden_correction\\.gate_proj$",
"re:(?:language_model\\.)?model\\.dfly_layers\\.hidden_correction\\.up_proj$"
]
}
},
"ignore": [
"re:.*lm_head$",
"re:(?:visual|vit)\\..*",
"re:(?:language_model\\.)?model\\.layers\\.0\\.self_attn\\..*",
"re:(?:language_model\\.)?model\\.layers\\.\\d+\\.self_attn\\.indexer\\.(?:wk|weights_proj)$",
"re:(?:language_model\\.)?model\\.dfly_layers\\.confidence_head\\.proj$",
"re:(?:language_model\\.)?model\\.dfly_layers\\.mtp_fusion\\.eh_proj$"
]
},
"dtype": "bfloat16",
"dfly_enable_ihc": false,
"dfly_mtp_fusion": "slot0",
"dfly_target_index_topk": 2048,
"dfly_context_local_window": null,
"dfly_enable_hidden_correction": true,
"dfly_num_hidden_layers": 5,
"dfly_mlp_type": "dense",
"dfly_mask_token_id": 120009,
"dfly_target_index_layer": [61, 65, 69, 73, 77],
"dfly_target_kv_layers": [61, 65, 69, 73, 77],
"dfly_attn_causal": true,
"dfly_block_size": 7
}