| { | |
| "architectures": [ | |
| "NumbatForCausalLM" | |
| ], | |
| "attention_bias": false, | |
| "auto_map": { | |
| "AutoConfig": "configuration_numbat.NumbatConfig", | |
| "AutoModel": "modeling_numbat.NumbatModel", | |
| "AutoModelForCausalLM": "modeling_numbat.NumbatForCausalLM" | |
| }, | |
| "bos_token_id": 0, | |
| "dense_layer_indices": [ | |
| 0, | |
| 1, | |
| 109, | |
| 110, | |
| 111, | |
| 112, | |
| 113 | |
| ], | |
| "dtype": "bfloat16", | |
| "eos_token_id": 1, | |
| "fp32_buffer_suffixes": [ | |
| "mlp.gate.e_score_correction_bias" | |
| ], | |
| "fp32_parameter_suffixes": [ | |
| "self_attn.A_log", | |
| "self_attn.dt_bias" | |
| ], | |
| "gdn_config": { | |
| "chunk_size": 64, | |
| "conv_bias": false, | |
| "head_dim_k": 128, | |
| "head_dim_v": 256, | |
| "num_heads": 48, | |
| "qk_l2norm": true, | |
| "short_conv_kernel": 4, | |
| "state_dtype": "float32" | |
| }, | |
| "hidden_act": "silu", | |
| "hidden_size": 12288, | |
| "intermediate_size": 24576, | |
| "is_encoder_decoder": false, | |
| "max_position_embeddings": 131072, | |
| "mla_config": { | |
| "kv_lora_rank": 512, | |
| "num_heads": 96, | |
| "q_lora_rank": 1536, | |
| "qk_nope_head_dim": 128, | |
| "qk_rope_head_dim": 64, | |
| "rope_theta": 1000000.0, | |
| "v_head_dim": 128 | |
| }, | |
| "mla_layer_indices": [ | |
| 3, | |
| 7, | |
| 11, | |
| 15, | |
| 19, | |
| 23, | |
| 27, | |
| 31, | |
| 35, | |
| 39, | |
| 43, | |
| 47, | |
| 51, | |
| 55, | |
| 59, | |
| 63, | |
| 67, | |
| 71, | |
| 75, | |
| 79, | |
| 83, | |
| 87, | |
| 91, | |
| 95, | |
| 99, | |
| 103, | |
| 107, | |
| 111 | |
| ], | |
| "mlp_bias": false, | |
| "model_type": "numbat", | |
| "moe_config": { | |
| "group_limited_routing": { | |
| "n_group": 72, | |
| "topk_group": 8 | |
| }, | |
| "load_balancing": { | |
| "dropless": true, | |
| "freeze_correction_at_token_fraction": 0.98 | |
| }, | |
| "moe_intermediate_size": 2816, | |
| "n_routed_experts": 1224, | |
| "n_shared_experts": 1, | |
| "norm_topk_prob": true, | |
| "num_experts_per_tok": 48, | |
| "num_moe_layers": 107, | |
| "routed_scaling_factor": 1.0, | |
| "router_dtype": "bfloat16", | |
| "router_score_dtype": "float32", | |
| "scoring_func": "sigmoid" | |
| }, | |
| "mtp_config": { | |
| "depth": 1, | |
| "inference_use": "self-speculative drafter (1 draft token)", | |
| "layer_index_in_checkpoint": 114, | |
| "loss_weight": 0.1 | |
| }, | |
| "num_hidden_layers": 114, | |
| "pad_token_id": 2, | |
| "rms_norm_eps": 1e-06, | |
| "tie_word_embeddings": false, | |
| "transformers_version": "5.19.0", | |
| "use_cache": false, | |
| "vocab_size": 131072, | |
| "quantization_config": { | |
| "config_groups": { | |
| "group_0": { | |
| "input_activations": { | |
| "dynamic": false, | |
| "group_size": 16, | |
| "num_bits": 4, | |
| "type": "float" | |
| }, | |
| "targets": [ | |
| "Linear" | |
| ], | |
| "weights": { | |
| "dynamic": false, | |
| "group_size": 16, | |
| "num_bits": 4, | |
| "type": "float" | |
| } | |
| } | |
| }, | |
| "group_size": 16, | |
| "ignore": [ | |
| "lm_head", | |
| "model.embed_tokens", | |
| "model.norm", | |
| "model.layers.0.*", | |
| "model.layers.1.*", | |
| "model.layers.109.*", | |
| "model.layers.110.*", | |
| "model.layers.111.*", | |
| "model.layers.112.*", | |
| "model.layers.113.*", | |
| "model.layers.114.*", | |
| "model.layers.*.mlp.gate", | |
| "model.layers.*.self_attn.a_proj", | |
| "model.layers.*.self_attn.b_proj", | |
| "model.layers.*.self_attn.*conv1d", | |
| "model.layers.*.*layernorm", | |
| "model.layers.*.self_attn.*layernorm", | |
| "model.layers.*.self_attn.o_norm" | |
| ], | |
| "producer": { | |
| "name": "modelopt", | |
| "version": "0.47.0" | |
| }, | |
| "quant_algo": "NVFP4", | |
| "quant_method": "modelopt" | |
| } | |
| } |
Xet Storage Details
- Size:
- 4.19 kB
- Xet hash:
- 5f69e5f68f88fe518909374829e1575e6480da67bc98f4d9f905e520ee3de5ec
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.