Buckets:
| """numbat model configuration.""" | |
| from transformers import PretrainedConfig | |
| class NumbatConfig(PretrainedConfig): | |
| """Configuration class for the numbat architecture. | |
| The architecture interleaves two mixer blocks: gated delta rules with | |
| a short causal convolution (``gdn_config``), and full multi-head | |
| latent attention with decoupled rope and a sigmoid output gate | |
| (``mla_config``). Feed-forward blocks are either dense SwiGLU or, for | |
| the MoE block, one shared expert plus routed experts with grouped | |
| top-48 routing (``moe_config``). Layer indices for each block are | |
| listed in ``dense_layer_indices`` and ``mla_layer_indices``. | |
| A single multi-token-prediction layer is stored after the decoder | |
| stack, at index ``num_hidden_layers``. It shares ``embed_tokens``, | |
| ``norm`` and ``lm_head`` with the trunk; ``enorm`` / ``hnorm`` / | |
| ``eh_proj`` implement the embedding drop and output projection | |
| (``mtp_config``). | |
| Quantized linears follow the NVFP4 quad layout: a U8 packed E2M1 | |
| payload, an F8_E4M3 block scale per 16 input elements, and F32 | |
| global weight/input scales. Dequantization happens in | |
| ``modeling_numbat``; tensors whose names match | |
| ``fp32_parameter_suffixes`` or ``fp32_buffer_suffixes`` must be loaded | |
| in FP32 regardless of the checkpoint's default dtype. | |
| """ | |
| model_type = "numbat" | |
| def __init__( | |
| self, | |
| vocab_size=131072, | |
| hidden_size=12288, | |
| num_hidden_layers=114, | |
| hidden_act="silu", | |
| rms_norm_eps=1e-06, | |
| max_position_embeddings=131072, | |
| tie_word_embeddings=False, | |
| gdn_config=None, | |
| mla_config=None, | |
| moe_config=None, | |
| mtp_config=None, | |
| dense_layer_indices=None, | |
| mla_layer_indices=None, | |
| fp32_parameter_suffixes=None, | |
| fp32_buffer_suffixes=None, | |
| **kwargs, | |
| ): | |
| self.gdn_config = gdn_config or {} | |
| self.mla_config = mla_config or {} | |
| self.moe_config = moe_config or {} | |
| self.mtp_config = mtp_config or {} | |
| self.dense_layer_indices = ( | |
| dense_layer_indices if dense_layer_indices is not None else [] | |
| ) | |
| self.mla_layer_indices = ( | |
| mla_layer_indices if mla_layer_indices is not None else [] | |
| ) | |
| self.fp32_parameter_suffixes = fp32_parameter_suffixes or [] | |
| self.fp32_buffer_suffixes = fp32_buffer_suffixes or [] | |
| self.quantization_config = kwargs.pop("quantization_config", None) | |
| super().__init__(**kwargs) | |
| self.vocab_size = vocab_size | |
| self.hidden_size = hidden_size | |
| self.num_hidden_layers = num_hidden_layers | |
| self.hidden_act = hidden_act | |
| self.rms_norm_eps = rms_norm_eps | |
| self.max_position_embeddings = max_position_embeddings | |
| self.tie_word_embeddings = tie_word_embeddings | |
| self.intermediate_size = kwargs.pop("intermediate_size", None) | |
| self.mtp_config.setdefault("layer_index_in_checkpoint", 114) | |
| def num_moe_layers(self): | |
| return sum(1 for i in range(self.num_hidden_layers) | |
| if i not in self.dense_layer_indices) | |
Xet Storage Details
- Size:
- 3.17 kB
- Xet hash:
- 4dcd2bca3456f2347df51d0887900cf01cc2cd3e0b0154bfa92f450f59a1edbf
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.