Hy4-flash-FP8 / config.json
yiqichen01's picture
update
678e774
Raw History Blame Contribute Delete
61 kB
{ "architectures": [ "HYV4FlashForConditionalGeneration" ], "dtype": "bfloat16", "image_end_token_id": 120119, "image_newline_token_id": 120121, "image_start_token_id": 120118, "image_token_id": 120120, "model_type": "hy_v4_flash", "processor_class": "HYV4FlashProcessor", "spatial_merge_size": 2, "text_config": { "architectures": [ "HYV4FlashForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "bitwise_backward_align": true, "bos_token_id": 120000, "compressed_indexer_candidate_topk": 2048, "compressed_indexer_compress_ratio": 4, "compressed_indexer_group_topk": 512, "compressed_indexer_head_dim": 128, "compressed_indexer_key_pooling": "gated", "compressed_indexer_num_attention_heads": 16, "compressed_indexer_reuse_fine_indexer": false, "compressed_indexer_skip_fine_select": true, "compressed_indexer_stage2_query_block_size": 8192, "compressed_indexer_teacher_pooling_type": "max", "compressed_indexer_triton_pooling_min_groups": 1, "compressed_indexer_use_triton_pooling": true, "dsa_skip_attn_post_norm": false, "enable_ihc": true, "enable_lm_head_fp32": true, "engram_embed_random_seed": 1234, "engram_fp8_embedding": true, "engram_isolate_fffd_namespace": false, "engram_layer_head_vocab_sizes": { "12": [ [ 10312277, 10312283, 10312301, 10312343, 10312373, 10312397, 10312409, 10312453 ], [ 10312457, 10312483, 10312507, 10312513, 10312531, 10312543, 10312567, 10312571 ] ], "2": [ [ 10312007, 10312021, 10312039, 10312069, 10312091, 10312121, 10312123, 10312147 ], [ 10312177, 10312187, 10312207, 10312213, 10312229, 10312241, 10312249, 10312259 ] ], "22": [ [ 10312613, 10312619, 10312633, 10312657, 10312681, 10312699, 10312723, 10312747 ], [ 10312759, 10312789, 10312811, 10312823, 10312867, 10312873, 10312889, 10312903 ] ], "32": [ [ 10312909, 10312919, 10312927, 10312957, 10312979, 10313011, 10313021, 10313057 ], [ 10313063, 10313117, 10313123, 10313129, 10313161, 10313179, 10313189, 10313201 ] ] }, "engram_layer_ids": [ 2, 12, 22, 32 ], "engram_max_ngram_size": 3, "engram_n_embed_per_ngram": 768, "engram_n_head_per_ngram": 8, "engram_pad_id": 120002, "engram_seed": 0, "engram_tokenizer_name_or_path": null, "engram_vocab_map_path": null, "engram_vocab_size": [ 10312000, 10312000 ], "eos_token_id": 120025, "fast_op_fp32_linear_gemm": true, "first_k_dense_replace": 1, "full_nope": false, "gate_groups": 1, "gate_lora_op": "none", "gate_lora_rank": null, "gated_mla": true, "gating_type": "elementwise", "hc_eps": 1e-06, "hc_magnitude": 2.0, "hc_mult": 4, "head_dim": 64, "hidden_act": "silu", "hidden_size": 3072, "ihc_kernel_backend": "naive", "index_head_dim": 128, "index_n_heads": 16, "index_topk": 2048, "indexer_types": [ "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full" ], "initializer_range": 0.006, "intermediate_size": 13824, "kv_lora_rank": 512, "layer_types": [ "deepseek_sparse_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention", "sliding_attention", "sliding_attention", "sliding_attention", "deepseek_sparse_attention" ], "learnable_sink": true, "learnable_sink_init": 0.0, "max_position_embeddings": 1048576, "mlp_layer_types": [ "dense", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse" ], "model_type": "hy_v4_flash", "moe_aux_loss_coeff": 0.0, "moe_intermediate_size": 1536, "moe_router_enable_expert_bias": true, "moe_router_use_sigmoid": true, "n_group": 1, "n_routed_experts": 512, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 32, "num_experts_per_tok": 16, "num_hidden_layers": 42, "num_key_value_heads": 32, "num_nextn_predict_layers": 0, "o_groups": 1, "o_lora_op": "none", "o_lora_rank": null, "output_router_logits": false, "pad_token_id": 120002, "q_lora_rank": 1024, "qk_head_dim": 256, "qk_nope_head_dim": 192, "qk_norm": true, "qk_rope_head_dim": 64, "return_bos_score": true, "return_max_logit": true, "rms_norm_eps": 1e-05, "rope_parameters": { "sliding_attention": { "mrope_interleaved": true, "mrope_section": [ 12, 10, 10 ], "rope_theta": 10000, "rope_type": "default" }, "deepseek_sparse_attention": { "mrope_interleaved": true, "mrope_section": [ 12, 10, 10 ], "rope_theta": 10000000, "rope_type": "default" } }, "routed_expert_hidden_size": 1536, "routed_scaling_factor": 3.998, "sandwich_norm": true, "sliding_window": 512, "swa_num_attention_heads": 64, "swa_pattern": [ 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0 ], "swa_qk_head_dim": 256, "swa_qk_nope_head_dim": 192, "swa_qk_rope_head_dim": 64, "swa_v_head_dim": 256, "swiglu_limit": 10.0, "tie_word_embeddings": false, "topk_group": 1, "use_cache": true, "use_compressed_indexer": true, "use_dsa": true, "use_grouped_mm": false, "use_mla": true, "v_head_dim": 256, "vocab_size": 120832, "window_size": [ 511, 0 ], "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "weight_block_size": [ 128, 128 ], "modules_to_not_convert": [ "language_model.model.lm_head", "language_model.model.model.embed_tokens", "language_model.model.model.hc_head.hc_head_fn", "language_model.model.model.layers.0.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.0.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.0.mlp.down_proj", "language_model.model.model.layers.0.mlp.gate_proj", "language_model.model.model.layers.0.mlp.up_proj", "language_model.model.model.layers.0.self_attn.compressed_indexer.head_weight_projection", "language_model.model.model.layers.0.self_attn.compressed_indexer.intra_group_ape", "language_model.model.model.layers.0.self_attn.compressed_indexer.key_gate_projection", "language_model.model.model.layers.0.self_attn.compressed_indexer.key_value_projection", "language_model.model.model.layers.0.self_attn.compressed_indexer.query_projection", "language_model.model.model.layers.0.self_attn.compressed_indexer.token_key_projection", "language_model.model.model.layers.0.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.0.self_attn.kv_b_proj", "language_model.model.model.layers.0.self_attn.linear_gate", "language_model.model.model.layers.0.self_attn.o_proj", "language_model.model.model.layers.0.self_attn.q_a_proj", "language_model.model.model.layers.0.self_attn.q_b_proj", "language_model.model.model.layers.1.engram_layer.key_projs.0", "language_model.model.model.layers.1.engram_layer.key_projs.1", "language_model.model.model.layers.1.engram_layer.key_projs.2", "language_model.model.model.layers.1.engram_layer.key_projs.3", "language_model.model.model.layers.1.engram_layer.multi_head_embedding.embedding", "language_model.model.model.layers.1.engram_layer.value_proj", "language_model.model.model.layers.1.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.1.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.1.mlp.gate", "language_model.model.model.layers.1.mlp.routed_expert_down_proj", "language_model.model.model.layers.1.mlp.routed_expert_up_proj", "language_model.model.model.layers.1.mlp.shared_experts.down_proj", "language_model.model.model.layers.1.mlp.shared_experts.gate_proj", "language_model.model.model.layers.1.mlp.shared_experts.up_proj", "language_model.model.model.layers.1.self_attn.compressed_indexer.head_weight_projection", "language_model.model.model.layers.1.self_attn.compressed_indexer.intra_group_ape", "language_model.model.model.layers.1.self_attn.compressed_indexer.key_gate_projection", "language_model.model.model.layers.1.self_attn.compressed_indexer.key_value_projection", "language_model.model.model.layers.1.self_attn.compressed_indexer.query_projection", "language_model.model.model.layers.1.self_attn.compressed_indexer.token_key_projection", "language_model.model.model.layers.1.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.1.self_attn.kv_b_proj", "language_model.model.model.layers.1.self_attn.q_a_proj", "language_model.model.model.layers.10.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.10.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.10.mlp.gate", "language_model.model.model.layers.10.mlp.routed_expert_down_proj", "language_model.model.model.layers.10.mlp.routed_expert_up_proj", "language_model.model.model.layers.10.mlp.shared_experts.down_proj", "language_model.model.model.layers.10.mlp.shared_experts.gate_proj", "language_model.model.model.layers.10.mlp.shared_experts.up_proj", "language_model.model.model.layers.10.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.10.self_attn.kv_b_proj", "language_model.model.model.layers.10.self_attn.q_a_proj", "language_model.model.model.layers.11.engram_layer.key_projs.0", "language_model.model.model.layers.11.engram_layer.key_projs.1", "language_model.model.model.layers.11.engram_layer.key_projs.2", "language_model.model.model.layers.11.engram_layer.key_projs.3", "language_model.model.model.layers.11.engram_layer.multi_head_embedding.embedding", "language_model.model.model.layers.11.engram_layer.value_proj", "language_model.model.model.layers.11.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.11.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.11.mlp.gate", "language_model.model.model.layers.11.mlp.routed_expert_down_proj", "language_model.model.model.layers.11.mlp.routed_expert_up_proj", "language_model.model.model.layers.11.mlp.shared_experts.down_proj", "language_model.model.model.layers.11.mlp.shared_experts.gate_proj", "language_model.model.model.layers.11.mlp.shared_experts.up_proj", "language_model.model.model.layers.11.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.11.self_attn.kv_b_proj", "language_model.model.model.layers.11.self_attn.q_a_proj", "language_model.model.model.layers.12.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.12.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.12.mlp.gate", "language_model.model.model.layers.12.mlp.routed_expert_down_proj", "language_model.model.model.layers.12.mlp.routed_expert_up_proj", "language_model.model.model.layers.12.mlp.shared_experts.down_proj", "language_model.model.model.layers.12.mlp.shared_experts.gate_proj", "language_model.model.model.layers.12.mlp.shared_experts.up_proj", "language_model.model.model.layers.12.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.12.self_attn.kv_b_proj", "language_model.model.model.layers.12.self_attn.q_a_proj", "language_model.model.model.layers.13.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.13.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.13.mlp.gate", "language_model.model.model.layers.13.mlp.routed_expert_down_proj", "language_model.model.model.layers.13.mlp.routed_expert_up_proj", "language_model.model.model.layers.13.mlp.shared_experts.down_proj", "language_model.model.model.layers.13.mlp.shared_experts.gate_proj", "language_model.model.model.layers.13.mlp.shared_experts.up_proj", "language_model.model.model.layers.13.self_attn.compressed_indexer.head_weight_projection", "language_model.model.model.layers.13.self_attn.compressed_indexer.intra_group_ape", "language_model.model.model.layers.13.self_attn.compressed_indexer.key_gate_projection", "language_model.model.model.layers.13.self_attn.compressed_indexer.key_value_projection", "language_model.model.model.layers.13.self_attn.compressed_indexer.query_projection", "language_model.model.model.layers.13.self_attn.compressed_indexer.token_key_projection", "language_model.model.model.layers.13.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.13.self_attn.kv_b_proj", "language_model.model.model.layers.13.self_attn.q_a_proj", "language_model.model.model.layers.14.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.14.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.14.mlp.gate", "language_model.model.model.layers.14.mlp.routed_expert_down_proj", "language_model.model.model.layers.14.mlp.routed_expert_up_proj", "language_model.model.model.layers.14.mlp.shared_experts.down_proj", "language_model.model.model.layers.14.mlp.shared_experts.gate_proj", "language_model.model.model.layers.14.mlp.shared_experts.up_proj", "language_model.model.model.layers.14.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.14.self_attn.kv_b_proj", "language_model.model.model.layers.14.self_attn.q_a_proj", "language_model.model.model.layers.15.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.15.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.15.mlp.gate", "language_model.model.model.layers.15.mlp.routed_expert_down_proj", "language_model.model.model.layers.15.mlp.routed_expert_up_proj", "language_model.model.model.layers.15.mlp.shared_experts.down_proj", "language_model.model.model.layers.15.mlp.shared_experts.gate_proj", "language_model.model.model.layers.15.mlp.shared_experts.up_proj", "language_model.model.model.layers.15.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.15.self_attn.kv_b_proj", "language_model.model.model.layers.15.self_attn.q_a_proj", "language_model.model.model.layers.16.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.16.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.16.mlp.gate", "language_model.model.model.layers.16.mlp.routed_expert_down_proj", "language_model.model.model.layers.16.mlp.routed_expert_up_proj", "language_model.model.model.layers.16.mlp.shared_experts.down_proj", "language_model.model.model.layers.16.mlp.shared_experts.gate_proj", "language_model.model.model.layers.16.mlp.shared_experts.up_proj", "language_model.model.model.layers.16.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.16.self_attn.kv_b_proj", "language_model.model.model.layers.16.self_attn.q_a_proj", "language_model.model.model.layers.17.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.17.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.17.mlp.gate", "language_model.model.model.layers.17.mlp.routed_expert_down_proj", "language_model.model.model.layers.17.mlp.routed_expert_up_proj", "language_model.model.model.layers.17.mlp.shared_experts.down_proj", "language_model.model.model.layers.17.mlp.shared_experts.gate_proj", "language_model.model.model.layers.17.mlp.shared_experts.up_proj", "language_model.model.model.layers.17.self_attn.compressed_indexer.head_weight_projection", "language_model.model.model.layers.17.self_attn.compressed_indexer.intra_group_ape", "language_model.model.model.layers.17.self_attn.compressed_indexer.key_gate_projection", "language_model.model.model.layers.17.self_attn.compressed_indexer.key_value_projection", "language_model.model.model.layers.17.self_attn.compressed_indexer.query_projection", "language_model.model.model.layers.17.self_attn.compressed_indexer.token_key_projection", "language_model.model.model.layers.17.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.17.self_attn.kv_b_proj", "language_model.model.model.layers.17.self_attn.q_a_proj", "language_model.model.model.layers.18.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.18.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.18.mlp.gate", "language_model.model.model.layers.18.mlp.routed_expert_down_proj", "language_model.model.model.layers.18.mlp.routed_expert_up_proj", "language_model.model.model.layers.18.mlp.shared_experts.down_proj", "language_model.model.model.layers.18.mlp.shared_experts.gate_proj", "language_model.model.model.layers.18.mlp.shared_experts.up_proj", "language_model.model.model.layers.18.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.18.self_attn.kv_b_proj", "language_model.model.model.layers.18.self_attn.q_a_proj", "language_model.model.model.layers.19.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.19.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.19.mlp.gate", "language_model.model.model.layers.19.mlp.routed_expert_down_proj", "language_model.model.model.layers.19.mlp.routed_expert_up_proj", "language_model.model.model.layers.19.mlp.shared_experts.down_proj", "language_model.model.model.layers.19.mlp.shared_experts.gate_proj", "language_model.model.model.layers.19.mlp.shared_experts.up_proj", "language_model.model.model.layers.19.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.19.self_attn.kv_b_proj", "language_model.model.model.layers.19.self_attn.q_a_proj", "language_model.model.model.layers.2.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.2.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.2.mlp.gate", "language_model.model.model.layers.2.mlp.routed_expert_down_proj", "language_model.model.model.layers.2.mlp.routed_expert_up_proj", "language_model.model.model.layers.2.mlp.shared_experts.down_proj", "language_model.model.model.layers.2.mlp.shared_experts.gate_proj", "language_model.model.model.layers.2.mlp.shared_experts.up_proj", "language_model.model.model.layers.2.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.2.self_attn.kv_b_proj", "language_model.model.model.layers.2.self_attn.q_a_proj", "language_model.model.model.layers.20.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.20.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.20.mlp.gate", "language_model.model.model.layers.20.mlp.routed_expert_down_proj", "language_model.model.model.layers.20.mlp.routed_expert_up_proj", "language_model.model.model.layers.20.mlp.shared_experts.down_proj", "language_model.model.model.layers.20.mlp.shared_experts.gate_proj", "language_model.model.model.layers.20.mlp.shared_experts.up_proj", "language_model.model.model.layers.20.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.20.self_attn.kv_b_proj", "language_model.model.model.layers.20.self_attn.q_a_proj", "language_model.model.model.layers.21.engram_layer.key_projs.0", "language_model.model.model.layers.21.engram_layer.key_projs.1", "language_model.model.model.layers.21.engram_layer.key_projs.2", "language_model.model.model.layers.21.engram_layer.key_projs.3", "language_model.model.model.layers.21.engram_layer.multi_head_embedding.embedding", "language_model.model.model.layers.21.engram_layer.value_proj", "language_model.model.model.layers.21.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.21.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.21.mlp.gate", "language_model.model.model.layers.21.mlp.routed_expert_down_proj", "language_model.model.model.layers.21.mlp.routed_expert_up_proj", "language_model.model.model.layers.21.mlp.shared_experts.down_proj", "language_model.model.model.layers.21.mlp.shared_experts.gate_proj", "language_model.model.model.layers.21.mlp.shared_experts.up_proj", "language_model.model.model.layers.21.self_attn.compressed_indexer.head_weight_projection", "language_model.model.model.layers.21.self_attn.compressed_indexer.intra_group_ape", "language_model.model.model.layers.21.self_attn.compressed_indexer.key_gate_projection", "language_model.model.model.layers.21.self_attn.compressed_indexer.key_value_projection", "language_model.model.model.layers.21.self_attn.compressed_indexer.query_projection", "language_model.model.model.layers.21.self_attn.compressed_indexer.token_key_projection", "language_model.model.model.layers.21.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.21.self_attn.kv_b_proj", "language_model.model.model.layers.21.self_attn.q_a_proj", "language_model.model.model.layers.22.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.22.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.22.mlp.gate", "language_model.model.model.layers.22.mlp.routed_expert_down_proj", "language_model.model.model.layers.22.mlp.routed_expert_up_proj", "language_model.model.model.layers.22.mlp.shared_experts.down_proj", "language_model.model.model.layers.22.mlp.shared_experts.gate_proj", "language_model.model.model.layers.22.mlp.shared_experts.up_proj", "language_model.model.model.layers.22.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.22.self_attn.kv_b_proj", "language_model.model.model.layers.22.self_attn.q_a_proj", "language_model.model.model.layers.23.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.23.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.23.mlp.gate", "language_model.model.model.layers.23.mlp.routed_expert_down_proj", "language_model.model.model.layers.23.mlp.routed_expert_up_proj", "language_model.model.model.layers.23.mlp.shared_experts.down_proj", "language_model.model.model.layers.23.mlp.shared_experts.gate_proj", "language_model.model.model.layers.23.mlp.shared_experts.up_proj", "language_model.model.model.layers.23.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.23.self_attn.kv_b_proj", "language_model.model.model.layers.23.self_attn.q_a_proj", "language_model.model.model.layers.24.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.24.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.24.mlp.gate", "language_model.model.model.layers.24.mlp.routed_expert_down_proj", "language_model.model.model.layers.24.mlp.routed_expert_up_proj", "language_model.model.model.layers.24.mlp.shared_experts.down_proj", "language_model.model.model.layers.24.mlp.shared_experts.gate_proj", "language_model.model.model.layers.24.mlp.shared_experts.up_proj", "language_model.model.model.layers.24.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.24.self_attn.kv_b_proj", "language_model.model.model.layers.24.self_attn.q_a_proj", "language_model.model.model.layers.25.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.25.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.25.mlp.gate", "language_model.model.model.layers.25.mlp.routed_expert_down_proj", "language_model.model.model.layers.25.mlp.routed_expert_up_proj", "language_model.model.model.layers.25.mlp.shared_experts.down_proj", "language_model.model.model.layers.25.mlp.shared_experts.gate_proj", "language_model.model.model.layers.25.mlp.shared_experts.up_proj", "language_model.model.model.layers.25.self_attn.compressed_indexer.head_weight_projection", "language_model.model.model.layers.25.self_attn.compressed_indexer.intra_group_ape", "language_model.model.model.layers.25.self_attn.compressed_indexer.key_gate_projection", "language_model.model.model.layers.25.self_attn.compressed_indexer.key_value_projection", "language_model.model.model.layers.25.self_attn.compressed_indexer.query_projection", "language_model.model.model.layers.25.self_attn.compressed_indexer.token_key_projection", "language_model.model.model.layers.25.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.25.self_attn.kv_b_proj", "language_model.model.model.layers.25.self_attn.q_a_proj", "language_model.model.model.layers.26.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.26.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.26.mlp.gate", "language_model.model.model.layers.26.mlp.routed_expert_down_proj", "language_model.model.model.layers.26.mlp.routed_expert_up_proj", "language_model.model.model.layers.26.mlp.shared_experts.down_proj", "language_model.model.model.layers.26.mlp.shared_experts.gate_proj", "language_model.model.model.layers.26.mlp.shared_experts.up_proj", "language_model.model.model.layers.26.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.26.self_attn.kv_b_proj", "language_model.model.model.layers.26.self_attn.q_a_proj", "language_model.model.model.layers.27.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.27.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.27.mlp.gate", "language_model.model.model.layers.27.mlp.routed_expert_down_proj", "language_model.model.model.layers.27.mlp.routed_expert_up_proj", "language_model.model.model.layers.27.mlp.shared_experts.down_proj", "language_model.model.model.layers.27.mlp.shared_experts.gate_proj", "language_model.model.model.layers.27.mlp.shared_experts.up_proj", "language_model.model.model.layers.27.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.27.self_attn.kv_b_proj", "language_model.model.model.layers.27.self_attn.q_a_proj", "language_model.model.model.layers.28.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.28.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.28.mlp.gate", "language_model.model.model.layers.28.mlp.routed_expert_down_proj", "language_model.model.model.layers.28.mlp.routed_expert_up_proj", "language_model.model.model.layers.28.mlp.shared_experts.down_proj", "language_model.model.model.layers.28.mlp.shared_experts.gate_proj", "language_model.model.model.layers.28.mlp.shared_experts.up_proj", "language_model.model.model.layers.28.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.28.self_attn.kv_b_proj", "language_model.model.model.layers.28.self_attn.q_a_proj", "language_model.model.model.layers.29.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.29.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.29.mlp.gate", "language_model.model.model.layers.29.mlp.routed_expert_down_proj", "language_model.model.model.layers.29.mlp.routed_expert_up_proj", "language_model.model.model.layers.29.mlp.shared_experts.down_proj", "language_model.model.model.layers.29.mlp.shared_experts.gate_proj", "language_model.model.model.layers.29.mlp.shared_experts.up_proj", "language_model.model.model.layers.29.self_attn.compressed_indexer.head_weight_projection", "language_model.model.model.layers.29.self_attn.compressed_indexer.intra_group_ape", "language_model.model.model.layers.29.self_attn.compressed_indexer.key_gate_projection", "language_model.model.model.layers.29.self_attn.compressed_indexer.key_value_projection", "language_model.model.model.layers.29.self_attn.compressed_indexer.query_projection", "language_model.model.model.layers.29.self_attn.compressed_indexer.token_key_projection", "language_model.model.model.layers.29.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.29.self_attn.kv_b_proj", "language_model.model.model.layers.29.self_attn.q_a_proj", "language_model.model.model.layers.3.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.3.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.3.mlp.gate", "language_model.model.model.layers.3.mlp.routed_expert_down_proj", "language_model.model.model.layers.3.mlp.routed_expert_up_proj", "language_model.model.model.layers.3.mlp.shared_experts.down_proj", "language_model.model.model.layers.3.mlp.shared_experts.gate_proj", "language_model.model.model.layers.3.mlp.shared_experts.up_proj", "language_model.model.model.layers.3.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.3.self_attn.kv_b_proj", "language_model.model.model.layers.3.self_attn.q_a_proj", "language_model.model.model.layers.30.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.30.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.30.mlp.gate", "language_model.model.model.layers.30.mlp.routed_expert_down_proj", "language_model.model.model.layers.30.mlp.routed_expert_up_proj", "language_model.model.model.layers.30.mlp.shared_experts.down_proj", "language_model.model.model.layers.30.mlp.shared_experts.gate_proj", "language_model.model.model.layers.30.mlp.shared_experts.up_proj", "language_model.model.model.layers.30.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.30.self_attn.kv_b_proj", "language_model.model.model.layers.30.self_attn.q_a_proj", "language_model.model.model.layers.31.engram_layer.key_projs.0", "language_model.model.model.layers.31.engram_layer.key_projs.1", "language_model.model.model.layers.31.engram_layer.key_projs.2", "language_model.model.model.layers.31.engram_layer.key_projs.3", "language_model.model.model.layers.31.engram_layer.multi_head_embedding.embedding", "language_model.model.model.layers.31.engram_layer.value_proj", "language_model.model.model.layers.31.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.31.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.31.mlp.gate", "language_model.model.model.layers.31.mlp.routed_expert_down_proj", "language_model.model.model.layers.31.mlp.routed_expert_up_proj", "language_model.model.model.layers.31.mlp.shared_experts.down_proj", "language_model.model.model.layers.31.mlp.shared_experts.gate_proj", "language_model.model.model.layers.31.mlp.shared_experts.up_proj", "language_model.model.model.layers.31.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.31.self_attn.kv_b_proj", "language_model.model.model.layers.31.self_attn.q_a_proj", "language_model.model.model.layers.32.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.32.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.32.mlp.gate", "language_model.model.model.layers.32.mlp.routed_expert_down_proj", "language_model.model.model.layers.32.mlp.routed_expert_up_proj", "language_model.model.model.layers.32.mlp.shared_experts.down_proj", "language_model.model.model.layers.32.mlp.shared_experts.gate_proj", "language_model.model.model.layers.32.mlp.shared_experts.up_proj", "language_model.model.model.layers.32.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.32.self_attn.kv_b_proj", "language_model.model.model.layers.32.self_attn.q_a_proj", "language_model.model.model.layers.33.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.33.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.33.mlp.gate", "language_model.model.model.layers.33.mlp.routed_expert_down_proj", "language_model.model.model.layers.33.mlp.routed_expert_up_proj", "language_model.model.model.layers.33.mlp.shared_experts.down_proj", "language_model.model.model.layers.33.mlp.shared_experts.gate_proj", "language_model.model.model.layers.33.mlp.shared_experts.up_proj", "language_model.model.model.layers.33.self_attn.compressed_indexer.head_weight_projection", "language_model.model.model.layers.33.self_attn.compressed_indexer.intra_group_ape", "language_model.model.model.layers.33.self_attn.compressed_indexer.key_gate_projection", "language_model.model.model.layers.33.self_attn.compressed_indexer.key_value_projection", "language_model.model.model.layers.33.self_attn.compressed_indexer.query_projection", "language_model.model.model.layers.33.self_attn.compressed_indexer.token_key_projection", "language_model.model.model.layers.33.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.33.self_attn.kv_b_proj", "language_model.model.model.layers.33.self_attn.q_a_proj", "language_model.model.model.layers.34.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.34.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.34.mlp.gate", "language_model.model.model.layers.34.mlp.routed_expert_down_proj", "language_model.model.model.layers.34.mlp.routed_expert_up_proj", "language_model.model.model.layers.34.mlp.shared_experts.down_proj", "language_model.model.model.layers.34.mlp.shared_experts.gate_proj", "language_model.model.model.layers.34.mlp.shared_experts.up_proj", "language_model.model.model.layers.34.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.34.self_attn.kv_b_proj", "language_model.model.model.layers.34.self_attn.q_a_proj", "language_model.model.model.layers.35.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.35.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.35.mlp.gate", "language_model.model.model.layers.35.mlp.routed_expert_down_proj", "language_model.model.model.layers.35.mlp.routed_expert_up_proj", "language_model.model.model.layers.35.mlp.shared_experts.down_proj", "language_model.model.model.layers.35.mlp.shared_experts.gate_proj", "language_model.model.model.layers.35.mlp.shared_experts.up_proj", "language_model.model.model.layers.35.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.35.self_attn.kv_b_proj", "language_model.model.model.layers.35.self_attn.q_a_proj", "language_model.model.model.layers.36.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.36.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.36.mlp.gate", "language_model.model.model.layers.36.mlp.routed_expert_down_proj", "language_model.model.model.layers.36.mlp.routed_expert_up_proj", "language_model.model.model.layers.36.mlp.shared_experts.down_proj", "language_model.model.model.layers.36.mlp.shared_experts.gate_proj", "language_model.model.model.layers.36.mlp.shared_experts.up_proj", "language_model.model.model.layers.36.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.36.self_attn.kv_b_proj", "language_model.model.model.layers.36.self_attn.q_a_proj", "language_model.model.model.layers.37.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.37.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.37.mlp.gate", "language_model.model.model.layers.37.mlp.routed_expert_down_proj", "language_model.model.model.layers.37.mlp.routed_expert_up_proj", "language_model.model.model.layers.37.mlp.shared_experts.down_proj", "language_model.model.model.layers.37.mlp.shared_experts.gate_proj", "language_model.model.model.layers.37.mlp.shared_experts.up_proj", "language_model.model.model.layers.37.self_attn.compressed_indexer.head_weight_projection", "language_model.model.model.layers.37.self_attn.compressed_indexer.intra_group_ape", "language_model.model.model.layers.37.self_attn.compressed_indexer.key_gate_projection", "language_model.model.model.layers.37.self_attn.compressed_indexer.key_value_projection", "language_model.model.model.layers.37.self_attn.compressed_indexer.query_projection", "language_model.model.model.layers.37.self_attn.compressed_indexer.token_key_projection", "language_model.model.model.layers.37.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.37.self_attn.kv_b_proj", "language_model.model.model.layers.37.self_attn.q_a_proj", "language_model.model.model.layers.38.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.38.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.38.mlp.gate", "language_model.model.model.layers.38.mlp.routed_expert_down_proj", "language_model.model.model.layers.38.mlp.routed_expert_up_proj", "language_model.model.model.layers.38.mlp.shared_experts.down_proj", "language_model.model.model.layers.38.mlp.shared_experts.gate_proj", "language_model.model.model.layers.38.mlp.shared_experts.up_proj", "language_model.model.model.layers.38.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.38.self_attn.kv_b_proj", "language_model.model.model.layers.38.self_attn.q_a_proj", "language_model.model.model.layers.39.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.39.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.39.mlp.gate", "language_model.model.model.layers.39.mlp.routed_expert_down_proj", "language_model.model.model.layers.39.mlp.routed_expert_up_proj", "language_model.model.model.layers.39.mlp.shared_experts.down_proj", "language_model.model.model.layers.39.mlp.shared_experts.gate_proj", "language_model.model.model.layers.39.mlp.shared_experts.up_proj", "language_model.model.model.layers.39.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.39.self_attn.kv_b_proj", "language_model.model.model.layers.39.self_attn.q_a_proj", "language_model.model.model.layers.4.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.4.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.4.mlp.gate", "language_model.model.model.layers.4.mlp.routed_expert_down_proj", "language_model.model.model.layers.4.mlp.routed_expert_up_proj", "language_model.model.model.layers.4.mlp.shared_experts.down_proj", "language_model.model.model.layers.4.mlp.shared_experts.gate_proj", "language_model.model.model.layers.4.mlp.shared_experts.up_proj", "language_model.model.model.layers.4.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.4.self_attn.kv_b_proj", "language_model.model.model.layers.4.self_attn.q_a_proj", "language_model.model.model.layers.40.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.40.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.40.mlp.gate", "language_model.model.model.layers.40.mlp.routed_expert_down_proj", "language_model.model.model.layers.40.mlp.routed_expert_up_proj", "language_model.model.model.layers.40.mlp.shared_experts.down_proj", "language_model.model.model.layers.40.mlp.shared_experts.gate_proj", "language_model.model.model.layers.40.mlp.shared_experts.up_proj", "language_model.model.model.layers.40.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.40.self_attn.kv_b_proj", "language_model.model.model.layers.40.self_attn.q_a_proj", "language_model.model.model.layers.41.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.41.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.41.mlp.gate", "language_model.model.model.layers.41.mlp.routed_expert_down_proj", "language_model.model.model.layers.41.mlp.routed_expert_up_proj", "language_model.model.model.layers.41.mlp.shared_experts.down_proj", "language_model.model.model.layers.41.mlp.shared_experts.gate_proj", "language_model.model.model.layers.41.mlp.shared_experts.up_proj", "language_model.model.model.layers.41.self_attn.compressed_indexer.head_weight_projection", "language_model.model.model.layers.41.self_attn.compressed_indexer.intra_group_ape", "language_model.model.model.layers.41.self_attn.compressed_indexer.key_gate_projection", "language_model.model.model.layers.41.self_attn.compressed_indexer.key_value_projection", "language_model.model.model.layers.41.self_attn.compressed_indexer.query_projection", "language_model.model.model.layers.41.self_attn.compressed_indexer.token_key_projection", "language_model.model.model.layers.41.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.41.self_attn.kv_b_proj", "language_model.model.model.layers.41.self_attn.q_a_proj", "language_model.model.model.layers.5.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.5.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.5.mlp.gate", "language_model.model.model.layers.5.mlp.routed_expert_down_proj", "language_model.model.model.layers.5.mlp.routed_expert_up_proj", "language_model.model.model.layers.5.mlp.shared_experts.down_proj", "language_model.model.model.layers.5.mlp.shared_experts.gate_proj", "language_model.model.model.layers.5.mlp.shared_experts.up_proj", "language_model.model.model.layers.5.self_attn.compressed_indexer.head_weight_projection", "language_model.model.model.layers.5.self_attn.compressed_indexer.intra_group_ape", "language_model.model.model.layers.5.self_attn.compressed_indexer.key_gate_projection", "language_model.model.model.layers.5.self_attn.compressed_indexer.key_value_projection", "language_model.model.model.layers.5.self_attn.compressed_indexer.query_projection", "language_model.model.model.layers.5.self_attn.compressed_indexer.token_key_projection", "language_model.model.model.layers.5.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.5.self_attn.kv_b_proj", "language_model.model.model.layers.5.self_attn.q_a_proj", "language_model.model.model.layers.6.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.6.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.6.mlp.gate", "language_model.model.model.layers.6.mlp.routed_expert_down_proj", "language_model.model.model.layers.6.mlp.routed_expert_up_proj", "language_model.model.model.layers.6.mlp.shared_experts.down_proj", "language_model.model.model.layers.6.mlp.shared_experts.gate_proj", "language_model.model.model.layers.6.mlp.shared_experts.up_proj", "language_model.model.model.layers.6.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.6.self_attn.kv_b_proj", "language_model.model.model.layers.6.self_attn.q_a_proj", "language_model.model.model.layers.7.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.7.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.7.mlp.gate", "language_model.model.model.layers.7.mlp.routed_expert_down_proj", "language_model.model.model.layers.7.mlp.routed_expert_up_proj", "language_model.model.model.layers.7.mlp.shared_experts.down_proj", "language_model.model.model.layers.7.mlp.shared_experts.gate_proj", "language_model.model.model.layers.7.mlp.shared_experts.up_proj", "language_model.model.model.layers.7.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.7.self_attn.kv_b_proj", "language_model.model.model.layers.7.self_attn.q_a_proj", "language_model.model.model.layers.8.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.8.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.8.mlp.gate", "language_model.model.model.layers.8.mlp.routed_expert_down_proj", "language_model.model.model.layers.8.mlp.routed_expert_up_proj", "language_model.model.model.layers.8.mlp.shared_experts.down_proj", "language_model.model.model.layers.8.mlp.shared_experts.gate_proj", "language_model.model.model.layers.8.mlp.shared_experts.up_proj", "language_model.model.model.layers.8.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.8.self_attn.kv_b_proj", "language_model.model.model.layers.8.self_attn.q_a_proj", "language_model.model.model.layers.9.hc_attn_layer.hc_pre.hc_fn", "language_model.model.model.layers.9.hc_mlp_layer.hc_pre.hc_fn", "language_model.model.model.layers.9.mlp.gate", "language_model.model.model.layers.9.mlp.routed_expert_down_proj", "language_model.model.model.layers.9.mlp.routed_expert_up_proj", "language_model.model.model.layers.9.mlp.shared_experts.down_proj", "language_model.model.model.layers.9.mlp.shared_experts.gate_proj", "language_model.model.model.layers.9.mlp.shared_experts.up_proj", "language_model.model.model.layers.9.self_attn.compressed_indexer.head_weight_projection", "language_model.model.model.layers.9.self_attn.compressed_indexer.intra_group_ape", "language_model.model.model.layers.9.self_attn.compressed_indexer.key_gate_projection", "language_model.model.model.layers.9.self_attn.compressed_indexer.key_value_projection", "language_model.model.model.layers.9.self_attn.compressed_indexer.query_projection", "language_model.model.model.layers.9.self_attn.compressed_indexer.token_key_projection", "language_model.model.model.layers.9.self_attn.kv_a_proj_with_mqa", "language_model.model.model.layers.9.self_attn.kv_b_proj", "language_model.model.model.layers.9.self_attn.q_a_proj", "vit.embeddings.position_embedding", "vit.layers.0.mlp.dense_4h_to_h", "vit.layers.0.mlp.dense_h_to_4h", "vit.layers.0.self_attn.k_proj", "vit.layers.0.self_attn.o_proj", "vit.layers.0.self_attn.q_proj", "vit.layers.0.self_attn.v_proj", "vit.layers.1.mlp.dense_4h_to_h", "vit.layers.1.mlp.dense_h_to_4h", "vit.layers.1.self_attn.k_proj", "vit.layers.1.self_attn.o_proj", "vit.layers.1.self_attn.q_proj", "vit.layers.1.self_attn.v_proj", "vit.layers.10.mlp.dense_4h_to_h", "vit.layers.10.mlp.dense_h_to_4h", "vit.layers.10.self_attn.k_proj", "vit.layers.10.self_attn.o_proj", "vit.layers.10.self_attn.q_proj", "vit.layers.10.self_attn.v_proj", "vit.layers.11.mlp.dense_4h_to_h", "vit.layers.11.mlp.dense_h_to_4h", "vit.layers.11.self_attn.k_proj", "vit.layers.11.self_attn.o_proj", "vit.layers.11.self_attn.q_proj", "vit.layers.11.self_attn.v_proj", "vit.layers.12.mlp.dense_4h_to_h", "vit.layers.12.mlp.dense_h_to_4h", "vit.layers.12.self_attn.k_proj", "vit.layers.12.self_attn.o_proj", "vit.layers.12.self_attn.q_proj", "vit.layers.12.self_attn.v_proj", "vit.layers.13.mlp.dense_4h_to_h", "vit.layers.13.mlp.dense_h_to_4h", "vit.layers.13.self_attn.k_proj", "vit.layers.13.self_attn.o_proj", "vit.layers.13.self_attn.q_proj", "vit.layers.13.self_attn.v_proj", "vit.layers.14.mlp.dense_4h_to_h", "vit.layers.14.mlp.dense_h_to_4h", "vit.layers.14.self_attn.k_proj", "vit.layers.14.self_attn.o_proj", "vit.layers.14.self_attn.q_proj", "vit.layers.14.self_attn.v_proj", "vit.layers.15.mlp.dense_4h_to_h", "vit.layers.15.mlp.dense_h_to_4h", "vit.layers.15.self_attn.k_proj", "vit.layers.15.self_attn.o_proj", "vit.layers.15.self_attn.q_proj", "vit.layers.15.self_attn.v_proj", "vit.layers.16.mlp.dense_4h_to_h", "vit.layers.16.mlp.dense_h_to_4h", "vit.layers.16.self_attn.k_proj", "vit.layers.16.self_attn.o_proj", "vit.layers.16.self_attn.q_proj", "vit.layers.16.self_attn.v_proj", "vit.layers.17.mlp.dense_4h_to_h", "vit.layers.17.mlp.dense_h_to_4h", "vit.layers.17.self_attn.k_proj", "vit.layers.17.self_attn.o_proj", "vit.layers.17.self_attn.q_proj", "vit.layers.17.self_attn.v_proj", "vit.layers.18.mlp.dense_4h_to_h", "vit.layers.18.mlp.dense_h_to_4h", "vit.layers.18.self_attn.k_proj", "vit.layers.18.self_attn.o_proj", "vit.layers.18.self_attn.q_proj", "vit.layers.18.self_attn.v_proj", "vit.layers.19.mlp.dense_4h_to_h", "vit.layers.19.mlp.dense_h_to_4h", "vit.layers.19.self_attn.k_proj", "vit.layers.19.self_attn.o_proj", "vit.layers.19.self_attn.q_proj", "vit.layers.19.self_attn.v_proj", "vit.layers.2.mlp.dense_4h_to_h", "vit.layers.2.mlp.dense_h_to_4h", "vit.layers.2.self_attn.k_proj", "vit.layers.2.self_attn.o_proj", "vit.layers.2.self_attn.q_proj", "vit.layers.2.self_attn.v_proj", "vit.layers.20.mlp.dense_4h_to_h", "vit.layers.20.mlp.dense_h_to_4h", "vit.layers.20.self_attn.k_proj", "vit.layers.20.self_attn.o_proj", "vit.layers.20.self_attn.q_proj", "vit.layers.20.self_attn.v_proj", "vit.layers.21.mlp.dense_4h_to_h", "vit.layers.21.mlp.dense_h_to_4h", "vit.layers.21.self_attn.k_proj", "vit.layers.21.self_attn.o_proj", "vit.layers.21.self_attn.q_proj", "vit.layers.21.self_attn.v_proj", "vit.layers.22.mlp.dense_4h_to_h", "vit.layers.22.mlp.dense_h_to_4h", "vit.layers.22.self_attn.k_proj", "vit.layers.22.self_attn.o_proj", "vit.layers.22.self_attn.q_proj", "vit.layers.22.self_attn.v_proj", "vit.layers.23.mlp.dense_4h_to_h", "vit.layers.23.mlp.dense_h_to_4h", "vit.layers.23.self_attn.k_proj", "vit.layers.23.self_attn.o_proj", "vit.layers.23.self_attn.q_proj", "vit.layers.23.self_attn.v_proj", "vit.layers.24.mlp.dense_4h_to_h", "vit.layers.24.mlp.dense_h_to_4h", "vit.layers.24.self_attn.k_proj", "vit.layers.24.self_attn.o_proj", "vit.layers.24.self_attn.q_proj", "vit.layers.24.self_attn.v_proj", "vit.layers.25.mlp.dense_4h_to_h", "vit.layers.25.mlp.dense_h_to_4h", "vit.layers.25.self_attn.k_proj", "vit.layers.25.self_attn.o_proj", "vit.layers.25.self_attn.q_proj", "vit.layers.25.self_attn.v_proj", "vit.layers.26.mlp.dense_4h_to_h", "vit.layers.26.mlp.dense_h_to_4h", "vit.layers.26.self_attn.k_proj", "vit.layers.26.self_attn.o_proj", "vit.layers.26.self_attn.q_proj", "vit.layers.26.self_attn.v_proj", "vit.layers.3.mlp.dense_4h_to_h", "vit.layers.3.mlp.dense_h_to_4h", "vit.layers.3.self_attn.k_proj", "vit.layers.3.self_attn.o_proj", "vit.layers.3.self_attn.q_proj", "vit.layers.3.self_attn.v_proj", "vit.layers.4.mlp.dense_4h_to_h", "vit.layers.4.mlp.dense_h_to_4h", "vit.layers.4.self_attn.k_proj", "vit.layers.4.self_attn.o_proj", "vit.layers.4.self_attn.q_proj", "vit.layers.4.self_attn.v_proj", "vit.layers.5.mlp.dense_4h_to_h", "vit.layers.5.mlp.dense_h_to_4h", "vit.layers.5.self_attn.k_proj", "vit.layers.5.self_attn.o_proj", "vit.layers.5.self_attn.q_proj", "vit.layers.5.self_attn.v_proj", "vit.layers.6.mlp.dense_4h_to_h", "vit.layers.6.mlp.dense_h_to_4h", "vit.layers.6.self_attn.k_proj", "vit.layers.6.self_attn.o_proj", "vit.layers.6.self_attn.q_proj", "vit.layers.6.self_attn.v_proj", "vit.layers.7.mlp.dense_4h_to_h", "vit.layers.7.mlp.dense_h_to_4h", "vit.layers.7.self_attn.k_proj", "vit.layers.7.self_attn.o_proj", "vit.layers.7.self_attn.q_proj", "vit.layers.7.self_attn.v_proj", "vit.layers.8.mlp.dense_4h_to_h", "vit.layers.8.mlp.dense_h_to_4h", "vit.layers.8.self_attn.k_proj", "vit.layers.8.self_attn.o_proj", "vit.layers.8.self_attn.q_proj", "vit.layers.8.self_attn.v_proj", "vit.layers.9.mlp.dense_4h_to_h", "vit.layers.9.mlp.dense_h_to_4h", "vit.layers.9.self_attn.k_proj", "vit.layers.9.self_attn.o_proj", "vit.layers.9.self_attn.q_proj", "vit.layers.9.self_attn.v_proj", "model.dfly_layers.0.self_attn.kv_a_proj_with_mqa", "model.dfly_layers.0.self_attn.kv_b_proj", "model.dfly_layers.0.self_attn.q_a_proj", "model.dfly_layers.1.self_attn.kv_a_proj_with_mqa", "model.dfly_layers.1.self_attn.kv_b_proj", "model.dfly_layers.1.self_attn.q_a_proj", "model.dfly_layers.2.self_attn.kv_a_proj_with_mqa", "model.dfly_layers.2.self_attn.kv_b_proj", "model.dfly_layers.2.self_attn.q_a_proj", "model.dfly_layers.3.self_attn.kv_a_proj_with_mqa", "model.dfly_layers.3.self_attn.kv_b_proj", "model.dfly_layers.3.self_attn.q_a_proj", "model.dfly_layers.4.self_attn.kv_a_proj_with_mqa", "model.dfly_layers.4.self_attn.kv_b_proj", "model.dfly_layers.4.self_attn.q_a_proj", "model.dfly_layers.confidence_head.proj", "model.dfly_layers.hidden_correction.down_proj", "model.dfly_layers.hidden_correction.gate_proj", "model.dfly_layers.hidden_correction.up_proj", "model.dfly_layers.mtp_fusion.eh_proj" ] } }, "transformers_version": "5.23.0", "video_end_token_id": 120123, "video_start_token_id": 120122, "video_token_id": 120124, "vision_config": { "add_patchemb_bias": true, "anyres_vit_max_image_size": 2048, "attention_backend": "hf_align_sdpa", "attention_dropout": 0.0, "bias_activation_fusion": true, "cat_extra_token": 0, "hidden_act": "gelu", "hidden_dropout": 0.0, "hidden_size": 1152, "img_max_token_num": 4096, "intermediate_size": 4304, "interpolate_mode": "bilinear", "learnable_mlp_pooling_size": 0, "max_image_size": 2048, "max_vit_seq_len": 16384, "min_image_size": 512, "model_type": "hy_v4_flash", "num_attention_heads": 16, "num_channels": 3, "num_hidden_layers": 27, "num_key_value_heads": 16, "out_hidden_size": 3072, "packed_seq_alignment": 128, "patch_size": 16, "perceive_post_norm": false, "perceive_pre_norm": true, "projector_type": "rmsnorm_mlp_adapter", "remove_prenorm": true, "resize_resolution": 2048, "rms_norm_eps": 1e-05, "rope_2d_percent": 1.0, "rope_2d_theta": 10000.0, "spatial_merge_size": 2, "spatial_patch_size": 1, "temporal_patch_size": 1, "text_hidden_size": 3072, "video_max_image_size": 768, "video_min_image_size": 256, "video_spatial_compression": false, "video_temporal_compression": false, "vision_full_attention": false }, "vision_start_token_id": 120118, "dfly_mtp_fusion": "slot0", "dfly_target_index_topk": 2048, "dfly_context_local_window": null, "dfly_enable_hidden_correction": true, "dfly_num_hidden_layers": 5, "dfly_mask_token_id": 120009, "dfly_target_index_layer": [ 25, 29, 33, 37, 41 ], "dfly_target_kv_layers": [ 25, 29, 33, 37, 41 ] }
{
"architectures": [
"HYV4FlashForConditionalGeneration"
],
"dtype": "bfloat16",
"image_end_token_id": 120119,
"image_newline_token_id": 120121,
"image_start_token_id": 120118,
"image_token_id": 120120,
"model_type": "hy_v4_flash",
"processor_class": "HYV4FlashProcessor",
"spatial_merge_size": 2,
"text_config": {
"architectures": [
"HYV4FlashForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bitwise_backward_align": true,
"bos_token_id": 120000,
"compressed_indexer_candidate_topk": 2048,
"compressed_indexer_compress_ratio": 4,
"compressed_indexer_group_topk": 512,
"compressed_indexer_head_dim": 128,
"compressed_indexer_key_pooling": "gated",
"compressed_indexer_num_attention_heads": 16,
"compressed_indexer_reuse_fine_indexer": false,
"compressed_indexer_skip_fine_select": true,
"compressed_indexer_stage2_query_block_size": 8192,
"compressed_indexer_teacher_pooling_type": "max",
"compressed_indexer_triton_pooling_min_groups": 1,
"compressed_indexer_use_triton_pooling": true,
"dsa_skip_attn_post_norm": false,
"enable_ihc": true,
"enable_lm_head_fp32": true,
"engram_embed_random_seed": 1234,
"engram_fp8_embedding": true,
"engram_isolate_fffd_namespace": false,
"engram_layer_head_vocab_sizes": {
"12": [
[
10312277,
10312283,
10312301,
10312343,
10312373,
10312397,
10312409,
10312453
],
[
10312457,
10312483,
10312507,
10312513,
10312531,
10312543,
10312567,
10312571
]
],
"2": [
[
10312007,
10312021,
10312039,
10312069,
10312091,
10312121,
10312123,
10312147
],
[
10312177,
10312187,
10312207,
10312213,
10312229,
10312241,
10312249,
10312259
]
],
"22": [
[
10312613,
10312619,
10312633,
10312657,
10312681,
10312699,
10312723,
10312747
],
[
10312759,
10312789,
10312811,
10312823,
10312867,