Update configuration_neollm.py

configuration_neollm.py

@@ -2,12 +2,7 @@
 import math
 
 from transformers.configuration_utils import PretrainedConfig
-try:
-    from transformers.modeling_rope_utils import (  # type: ignore[attr-defined]
-        rope_config_validation as _legacy_rope_config_validation,
-    )
-except ImportError:  # pragma: no cover - only older Transformers releases.
-    _legacy_rope_config_validation = None
+from transformers.modeling_rope_utils import rope_config_validation
 from transformers.utils import logging
 
 logger = logging.get_logger(__name__)
@@ -16,17 +11,14 @@ logger = logging.get_logger(__name__)
 class NeoLLMConfig(PretrainedConfig):
     r"""
     Configuration class for the NeoLLM model architecture.
-
     Instantiates a NeoLLM model according to the specified arguments, defining the
     full architecture including attention mechanisms, normalization, periodicity
     modeling, an optional Leviathan continuous token embedding generator (LEV
     layer, Batley & Saha 2026), and optional Spelling Bee character-level
     embedding augmentation.
-
     Configuration objects inherit from :class:`~transformers.PretrainedConfig` and
     can be used to control the model outputs. Read the documentation from
     :class:`~transformers.PretrainedConfig` for more information.
-
     Args:
         vocab_size (:obj:`int`, *optional*, defaults to 200005):
             Vocabulary size of the NeoLLM model. Defines the number of different
@@ -45,22 +37,7 @@ class NeoLLMConfig(PretrainedConfig):
         hidden_act (:obj:`str`, *optional*, defaults to ``"xielu"``):
             Non-linear activation function used in the MLP layers.
         max_position_embeddings (:obj:`int`, *optional*, defaults to 32768):
-            Training/pretraining sequence length.  It is kept separate from
-            the opt-in inference context limit so a training checkpoint can
-            retain its original curriculum length.
-        inference_max_position_embeddings (:obj:`int`, *optional*, defaults
-            to 32768): Maximum sequence length permitted by the explicit long
-            context inference mode.  This is a runtime guardrail, not a
-            training setting.
-        allow_long_context_inference (:obj:`bool`, *optional*, defaults to
-            ``False``): If true, ``NeoLLMForCausalLM.generate`` temporarily
-            exposes ``inference_max_position_embeddings`` to Transformers'
-            generation length checks.  Training is never affected.
-        strict_cache_parity (:obj:`bool`, *optional*, defaults to ``False``):
-            Disable low-precision incremental cache decoding and use the full
-            sequence fallback when exact greedy parity is required. This is an
-            inference-only safety switch; it takes priority over the default
-            performance mode enabled by ``allow_cache_numerical_drift``.
+            Maximum sequence length supported by the positional encoding.
         initializer_range (:obj:`float`, *optional*, defaults to 0.02):
             Standard deviation of the truncated-normal weight initializer.
         rms_norm_eps (:obj:`float`, *optional*, defaults to 1e-6):
@@ -123,9 +100,8 @@ class NeoLLMConfig(PretrainedConfig):
             Base period for Rotary Position Embeddings (RoPE).
         rope_scaling (:obj:`dict`, *optional*):
             Dictionary containing the RoPE scaling configuration. Must contain at
-             least the key ``"rope_type"`` (or ``"type"``). Validated by the
-             current ``PretrainedConfig.validate_rope`` API, with a compatibility
-             fallback for older Transformers releases.
+            least the key ``"rope_type"`` (or ``"type"``). Validated by
+            :func:`~transformers.modeling_rope_utils.rope_config_validation`.
         partial_rotary_factor (:obj:`float`, *optional*, defaults to 0.25):
             Fraction of each attention head's dimension to rotate with RoPE.
         attention_bias (:obj:`bool`, *optional*, defaults to ``False``):
@@ -184,9 +160,7 @@ class NeoLLMConfig(PretrainedConfig):
             input-dependent per-head scaling factor α and a moving-average bias β
             directly to the softmax-normalized attention weights before the
             weighted sum with V:
-
                 [α(X) · softmax(QKᵀ/√dk) + β(X)] V
-
             This relaxes the unit-sum constraint of softmax, reduces first-token
             bias, increases attention entropy, and promotes head diversity.
             Orthogonal to Gated Attention: the gate modulates the post-SDPA
@@ -204,15 +178,12 @@ class NeoLLMConfig(PretrainedConfig):
             along the direction of the token's own value vector, forcing the
             attention layer to carry only contextual information orthogonal to
             self-position.
-
             Two paths depending on active components:
-
             - **MEA active or LUCID active**: ``v_ref`` is the value vector
               after MEA mixing and after LUCID preconditioning — the vector that
               actually participated in the SDPA aggregation.
             - **MEA and LUCID inactive**: ``v_ref`` is the raw value projection
               — standard XSA as described in the paper.
-
             Applied after MEAHeadRMSNorm and before the Gated Attention gate.
             Gains increase with sequence length.
         xsa_eps (:obj:`float`, *optional*, defaults to 1e-6):
@@ -236,7 +207,6 @@ class NeoLLMConfig(PretrainedConfig):
         stack_memory_cache_size (:obj:`int`, *optional*, defaults to ``2048``):
             Cache length kept for parity with the released StackTrans source.
             The default training path keeps this cache disabled.
-
         fan_ratio (:obj:`float`, *optional*, defaults to 0.125):
             Ratio controlling the periodic-dimension size in FANformer attention.
             The transformed representation has dimension
@@ -297,9 +267,7 @@ class NeoLLMConfig(PretrainedConfig):
         use_token_generator (:obj:`bool`, *optional*, defaults to ``False``):
             Replace the discrete vocabulary embedding lookup table with a
             **Leviathan** continuous token generator (Batley & Saha, 2026).
-
             When enabled:
-
             - ``tie_word_embeddings`` is forced to ``False``.
             - ``model.embed_tokens`` is replaced by ``model.token_generator``
               (:class:`LeviathanGenerator`).
@@ -307,7 +275,6 @@ class NeoLLMConfig(PretrainedConfig):
               ``O(k · ⌈V^{1/k}⌉ · d_seed)`` instead of ``O(V · D)``.
             - The output head stays fully untied (``lm_head``), with a total
               parameter overhead below 0.2% at 1.2B scale.
-
             See :class:`LeviathanGenerator` in ``modeling_neollm.py``.
         generator_d_seed (:obj:`int`, *optional*, defaults to 128):
             Dimensionality of the latent seed space ``z̃ ∈ [0,1]^{d_seed}``,
@@ -341,93 +308,73 @@ class NeoLLMConfig(PretrainedConfig):
         use_spelling_bee_embeddings (:obj:`bool`, *optional*, defaults to ``False``):
             Augment token embeddings with character-level byte information
             (Rabe, Clymo & Dong, 2026).
-
             Each token's UTF-8 encoding (up to 16 bytes) is embedded through a
             shared ``nn.Embedding(256, d)`` table. Byte embeddings are
             position-encoded with RoPE using intra-token byte positions (not
             sequence positions), averaged over the fixed 16 byte slots, then
             averaged with the standard token embedding:
-
             .. math::
                 e_{\\text{bee}}(t) = \\frac{1}{2}\\left(e_{\\text{tok}}(t) +
                 \\frac{1}{16}\\sum_{i=1}^{16}
                 \\text{RoPE}(e_{\\text{byte}}[b_i], i)\\right)
-
             The active configuration follows the reference fixed-length mean.
             A separate theoretical variant would mask padded positions and use
             ``1 / sqrt(|t|)`` over valid bytes; it is intentionally not active
             in this training configuration.
-
             Adds ``256 × hidden_size`` parameters (≈0.13M for d=512).
             Zero inference overhead when ``bake_inference_table()`` is called
             after training.
-
             Compatible with all four combinations of ``use_token_generator``
             and ``use_spelling_bee_embeddings``.
-
             **Setup required**: call
             ``model.model.spelling_bee.set_byte_table(tokenizer)`` once after
             model instantiation (handled automatically by ``setup_model`` in
             ``train.py``).
-
             Reference: Rabe, Clymo & Dong (2026). *Spelling Bee Embeddings for
             Language Modeling.* arXiv:2601.18030.
-
         use_hadamard_o_proj (:obj:`bool`, *optional*, defaults to ``False``):
             Replace the dense ``W_O ∈ R^{d×d}`` output projection in every
             multi-head attention block with a fixed Walsh–Hadamard Transform
             followed by a learnable per-channel affine rescaling
             ``α ⊙ FWHT(x)/√d + β``.
-
             The WHT is a parameter-free orthogonal matrix whose singular values
             are all identically 1, so the effective condition number is
             ``κ = 1`` by construction and cannot grow during training. This
             directly addresses the high-κ pathology (κ up to 10^5) observed in
             the dense ``o_proj`` matrices, which causes FP8 per-tensor
             quantisation to lose low-magnitude directions entirely.
-
             Parameter reduction: replaces ``d²`` weights with ``2d``
             (``α`` and ``β``), saving ≈25% of attention parameters per block.
             Requires ``hidden_size`` to be a power of 2 (512 ✓, 1024 ✓,
             768 ✗).
-
             Reference: Aggarwal & Kumar (2026). *Rethinking Attention Output
             Projection: Structured Hadamard Transforms for Efficient
             Transformers.* arXiv:2603.08343.
-
         use_repo (:obj:`bool`, *optional*, defaults to ``False``):
             Enable Context Re-Positioning (REPO) in attention layers at or
             above ``repo_start_layer`` (Li et al., 2026).
-
             REPO replaces the fixed linear position indices ``0…L-1`` fed to
             RoPE with continuous, data-dependent positions ``z_i = f_ϕ(h_i)``
             learned end-to-end. The attention score between tokens ``i`` and
             ``j`` becomes:
-
             .. math::
                 A^{\\text{REPO}}_{i,j} = q_i^\\top\\, g_\\theta(z_j - z_i)\\, k_j
-
             where ``g_θ`` is the standard RoPE rotation and ``z_i`` is
             predicted from the hidden state ``h_i`` by a lightweight SwiGLU
             sub-layer ``f_ϕ``:
-
             .. math::
                 r_i = \\text{Swish}(h_i W_g) \\odot (h_i W_c), \\quad
                 z_i^{(h)} = r_i w_z^{(h)}
-
             ``W_g, W_c \\in \\mathbb{R}^{d \\times d_p}`` are shared across
             all query heads within a layer; ``w_z^{(h)} \\in \\mathbb{R}^{d_p}``
             is learned independently per head. The assigned positions are
             real-valued and unconstrained — the model may learn constant
             (NoPE-like), monotonic (RoPE-like), or hybrid patterns as needed.
-
             Lower layers (``layer_idx < repo_start_layer``) retain the
             standard integer RoPE positions because they primarily capture
             surface-level, locally-dependent features that benefit less from
             re-positioning (Li et al., 2026, §3).
-
             Overhead: +0.9% parameters; inference latency negligible.
-
         repo_start_layer (:obj:`int`, *optional*, defaults to
             ``num_hidden_layers // 3``):
             Index of the first decoder layer to which REPO is applied.
@@ -435,7 +382,6 @@ class NeoLLMConfig(PretrainedConfig):
             integer RoPE positions. Must satisfy
             ``0 <= repo_start_layer < num_hidden_layers``.
             Ignored when ``use_repo=False``.
-
         repo_d_p (:obj:`int`, *optional*, defaults to
             ``hidden_size // 8``):
             Dimensionality of the intermediate position representation
@@ -443,79 +389,55 @@ class NeoLLMConfig(PretrainedConfig):
             ``d_p = d/8`` on the assumption that positional information
             is less rich than the full hidden representation. Ignored
             when ``use_repo=False``.
-
         use_repo_grape (:obj:`bool`, *optional*, defaults to ``False``):
             Enable the proposed REPO-GRAPE positional operator. This reuses
             the REPO coordinate module ``f_ϕ`` and replaces the RoPE action
             with a GRAPE-M action using a learned per-head/per-plane angular
             spectrum:
-
             .. math::
                 u_i^{(h)} = z_i^{(h)}, \\quad
                 \\theta_{h,r}=\\theta^0_r\\exp(s_{h,r})
-
             When enabled, REPO-GRAPE activates the REPO coordinate path at
             ``repo_start_layer`` even if ``use_repo=False``. Lower layers keep
             standard integer RoPE.
-
-        use_repo_grape_triton (:obj:`bool`, *optional*, defaults to ``False``):
-            Enable the optional ``cut-cross-entropy`` Triton acceleration for
-            REPO-GRAPE during inference. The model unconditionally keeps the
-            reference PyTorch route for training or gradient-enabled forwards;
-            unsupported shapes also fall back to that route.
-
         use_laurel (:obj:`bool`, *optional*, defaults to ``False``):
             Enable the Learned Augmented Residual Layer (LAUREL) framework
             (Menghani, Kumar & Kumar, ICML 2025). LAUREL generalises the
             canonical residual connection:
-
             .. math::
                 x_{i+1} = \\alpha \\cdot f(x_i) + g(x_i)
-
             where :math:`g` is a learned linear function operating on the
             residual stream.  Applied independently to both the attention
             and MLP sublayers of every decoder layer.
-
             At least one of ``use_laurel_rw`` or ``use_laurel_lr`` must be
             ``True`` when this flag is active; both may be active
             simultaneously, producing the combined **LAUREL-RW+LR** variant
             (paper eq. 5).
-
             Incompatible with ``use_attn_res=True`` — both methods modify
             the residual stream and their interaction is undefined.
-
             Reference: Menghani, G., Kumar, R. & Kumar, S. (2025).
             *LAUREL: Learned Augmented Residual Layer.* ICML 2025.
-
         use_laurel_rw (:obj:`bool`, *optional*, defaults to ``True``):
             Enable the **LAUREL-RW** (Residual Weights) variant.  Assigns
             independent learned scalars :math:`\\alpha, \\beta` to the
             sublayer output and residual respectively:
-
             .. math::
                 x_{i+1} = \\alpha_s \\cdot f(x_i) + \\beta_s \\cdot x_i
-
             :math:`\\alpha_s, \\beta_s = \\text{softmax}([\\tilde{\\alpha},
             \\tilde{\\beta}])` so that they are non-negative and sum to 1,
             preventing unbounded growth (paper §2.1).  Adds **2 parameters
             per sublayer** (4 per decoder layer).
-
             When combined with ``use_laurel_lr=True`` (LAUREL-RW+LR,
             paper eq. 5):
-
             .. math::
                 x_{i+1} = \\alpha_s \\cdot f(x_i)
                          + \\beta_s \\cdot (B A x_i + x_i)
-
             Ignored when ``use_laurel=False``.
-
         use_laurel_lr (:obj:`bool`, *optional*, defaults to ``False``):
             Enable the **LAUREL-LR** (Low-Rank) variant.  Augments the
             residual with a rank-``laurel_lr_rank`` correction:
-
             .. math::
                 x_{i+1} = f(x_i) + B A x_i + x_i
-
             where :math:`A \\in \\mathbb{R}^{D \\times r}` and
             :math:`B \\in \\mathbb{R}^{r \\times D}` are learnable matrices
             (paper eq. 3).  :math:`A` is initialised with column-orthogonal
@@ -524,27 +446,22 @@ class NeoLLMConfig(PretrainedConfig):
             convention and ensuring the residual starts as identity
             (paper §3.3).  Adds **2·r·D parameters per sublayer**
             (4·r·D per decoder layer).
-
             Ignored when ``use_laurel=False``.
-
         laurel_lr_rank (:obj:`int`, *optional*, defaults to ``32``):
             Rank ``r`` of the low-rank matrices in LAUREL-LR.  The paper
             recommends :math:`r \\in \\{32, 48, 64\\}` for LLMs
             (paper §3.3).  Ignored when ``use_laurel=False`` or
             ``use_laurel_lr=False``.
-
         use_tweo (:obj:`bool`, *optional*, defaults to ``False``):
             Enable Transformers Without Extreme Outliers (TWEO), a
             training-only activation regularizer from Liang et al. (2026).
             TWEO adds a scaled :math:`L_p` penalty over the final output
             activations of every Transformer block:
-
             .. math::
                 L_{total} = L_{task} + \lambda L_{TWEO}, \quad
                 L_{TWEO} = \frac{1}{L}\sum_l
                 \mathbb{E}\left[\left(\frac{|A^{(l)}|}
                 {\tau + \epsilon}\right)^p\right]
-
             This is non-invasive: no architecture, inference path, or
             quantization backend is changed. It is intended for FP8/low-bit
             training runs where extreme activation outliers are harmful.
@@ -560,7 +477,6 @@ class NeoLLMConfig(PretrainedConfig):
             small activations while sharply penalizing extreme outliers.
         tweo_eps (:obj:`float`, *optional*, defaults to ``1e-6``):
             Numerical stability epsilon added to the TWEO denominator.
-
         use_nitp_temporal (:obj:`bool`, *optional*, defaults to ``False``):
             Instantiate the causal NITP temporal-transition extension.  The
             module is a separate bias-free multiplicative residual updater and
@@ -613,7 +529,6 @@ class NeoLLMConfig(PretrainedConfig):
             Temperature controlling temporal label smoothing.  Smaller values
             make the correct diagonal step more nearly one-hot, while retaining
             limited tolerance for genuinely similar adjacent states.
-
     Constraints:
         - ``use_spelling_bee_embeddings=True`` requires calling
           ``model.model.spelling_bee.set_byte_table(tokenizer)`` before
@@ -629,17 +544,13 @@ class NeoLLMConfig(PretrainedConfig):
         - ``stack_d_model`` must be divisible by ``num_mem_heads`` when
           ``use_stack_memory=True``.
         - ``use_nitp_temporal=True`` requires ``use_nitp=True``.
-
     Examples::
-
         >>> from configuration_neollm import NeoLLMConfig
         >>> from modeling_neollm import NeoLLMForCausalLM
-
         >>> # Standard dense-embedding model
         >>> config = NeoLLMConfig(use_token_generator=False,
         ...                       tie_word_embeddings=True)
         >>> model  = NeoLLMForCausalLM(config)
-
         >>> # Full attention stack
         >>> config_full = NeoLLMConfig(
         ...     use_affine_scaled_attention=True,
@@ -647,59 +558,45 @@ class NeoLLMConfig(PretrainedConfig):
         ...     use_lucid_attention=True,
         ... )
         >>> model_full = NeoLLMForCausalLM(config_full)
-
         >>> # Leviathan generator (LEV layer, untied output head)
         >>> config_lev = NeoLLMConfig(
         ...     use_token_generator=True,
         ...     tie_word_embeddings=False,
         ... )
-
         >>> # REPO: context re-positioning from layer 4 onward (default for 12 layers)
         >>> config_repo = NeoLLMConfig(
         ...     use_repo=True,
         ...     # repo_start_layer defaults to num_hidden_layers // 3 = 4
         ...     # repo_d_p          defaults to hidden_size // 8       = 64
         ... )
-
         >>> # Proposed REPO-GRAPE-M: REPO coordinates + learned GRAPE spectrum
         >>> config_repo_grape = NeoLLMConfig(
         ...     use_repo_grape=True,
         ... )
-
         >>> # REPO-GRAPE-M + GOAT-style factorised log-prior
         >>> config_repo_grape_goat = NeoLLMConfig(
         ...     use_repo_grape=True,
         ...     use_repo_goat_prior=True,
         ...     repo_goat_num_frequencies=3,
         ... )
-
     References:
         Bae, J. et al. (2026). *Affine-Scaled Attention: Towards Flexible and
         Stable Transformer Attention.* arXiv:2602.23057.
-
         Duvvuri, S. et al. (2026). *LUCID: Attention with Preconditioned
         Representations.* arXiv:2602.10410.
-
         Zhai, S. (2026). *Exclusive Self Attention.* arXiv:2603.09078.
-
         Batley, R. T. & Saha, S. (2026). *A Separable Architecture for Continuous
         Token Representation in Language Models.* arXiv:2601.22040.
-
         Robinson, M. et al. (2025). *Token Embeddings Violate the Manifold
         Hypothesis.* arXiv:2504.01002.
-
         Rabe, M. N., Clymo, J. & Dong, Z. (2026). *Spelling Bee Embeddings for
         Language Modeling.* arXiv:2601.18030.
-
         Li, H., Zhao, T., Cai, D. & Sproat, R. (2026). *REPO: Language Models
         with Context Re-Positioning.* arXiv:2512.14391.
-
         Zhang, Y. et al. (2026). *Group Representational Position Encoding.*
         ICLR 2026 / arXiv:2512.07805.
-
         Litman, E. & Guo, G. (2026). *You Need Better Attention Priors.*
         arXiv:2601.15380.
-
         Menghani, G., Kumar, R. & Kumar, S. (2025). *LAUREL: Learned Augmented
         Residual Layer.* ICML 2025. arXiv:2411.07501.
     """
@@ -717,8 +614,6 @@ class NeoLLMConfig(PretrainedConfig):
         num_key_value_heads=4,
         hidden_act="xielu",
         max_position_embeddings=32768,
-        inference_max_position_embeddings=32768,
-        allow_long_context_inference=False,
         initializer_range=0.02,
         rms_norm_eps=1e-6,
         tie_word_embeddings=False,
@@ -761,11 +656,6 @@ class NeoLLMConfig(PretrainedConfig):
         mea_groupnorm_eps=1e-6,
         use_lucid_attention=False,
         lucid_attention_eps=1e-6,
-        # Eager BF16/FP16 can round reductions differently for q_len=S and
-        # q_len=1. The optimized inference path is enabled by default; users
-        # requiring strict parity can set this to False or use strict_cache_parity.
-        allow_cache_numerical_drift=True,
-        strict_cache_parity=False,
         use_affine_scaled_attention=False,
         affine_momentum=0.9,
         use_xsa=True,
@@ -819,7 +709,6 @@ class NeoLLMConfig(PretrainedConfig):
         repo_start_layer=None,
         repo_d_p=None,
         use_repo_grape=True,
-        use_repo_grape_triton=False,
         use_repo_goat_prior=False,
         repo_goat_num_frequencies=3,
         repo_goat_sink_decay=4.0,
@@ -1230,21 +1119,6 @@ class NeoLLMConfig(PretrainedConfig):
         # ── Core Transformer ──────────────────────────────────────────────
         self.vocab_size = vocab_size
         self.max_position_embeddings = max_position_embeddings
-        # ``max_position_embeddings`` is the checkpoint's training limit.  Do
-        # not overwrite it when enabling the inference-only long-context
-        # context manager in modeling_neollm.py.
-        self.training_max_position_embeddings = int(max_position_embeddings)
-        if inference_max_position_embeddings is None:
-            inference_max_position_embeddings = max_position_embeddings
-        if int(inference_max_position_embeddings) <= 0:
-            raise ValueError(
-                "`inference_max_position_embeddings` must be positive, got "
-                f"{inference_max_position_embeddings}."
-            )
-        self.inference_max_position_embeddings = int(
-            inference_max_position_embeddings
-        )
-        self.allow_long_context_inference = bool(allow_long_context_inference)
         self.hidden_size = hidden_size
         self.intermediate_size = intermediate_size
         self.num_hidden_layers = num_hidden_layers
@@ -1299,8 +1173,6 @@ class NeoLLMConfig(PretrainedConfig):
         self.mea_groupnorm_eps = mea_groupnorm_eps
         self.use_lucid_attention = use_lucid_attention
         self.lucid_attention_eps = lucid_attention_eps
-        self.allow_cache_numerical_drift = bool(allow_cache_numerical_drift)
-        self.strict_cache_parity = bool(strict_cache_parity)
         self.use_affine_scaled_attention = use_affine_scaled_attention
         self.affine_momentum = affine_momentum
         self.use_xsa = use_xsa
@@ -1328,14 +1200,7 @@ class NeoLLMConfig(PretrainedConfig):
         self.num_mem_heads = num_mem_heads
         self.stack_slots = stack_slots
         self.stack_memory_cache_size = stack_memory_cache_size
-        # Transformers 5.x moved RoPE validation onto PretrainedConfig and the
-        # old module-level helper now emits a deprecation warning. Prefer the
-        # new method while retaining a path for older installed versions.
-        validate_rope = getattr(self, "validate_rope", None)
-        if callable(validate_rope):
-            validate_rope()
-        elif _legacy_rope_config_validation is not None:
-            _legacy_rope_config_validation(self)
+        rope_config_validation(self)
 
         # ── FANformer periodicity ─────────────────────────────────────────
         self.use_fan_residual = use_fan_residual
@@ -1389,7 +1254,6 @@ class NeoLLMConfig(PretrainedConfig):
         self.repo_start_layer = repo_start_layer
         self.repo_d_p = repo_d_p
         self.use_repo_grape = use_repo_grape
-        self.use_repo_grape_triton = bool(use_repo_grape_triton)
         self.use_repo_goat_prior = use_repo_goat_prior
         self.repo_goat_num_frequencies = int(repo_goat_num_frequencies)
         self.repo_goat_sink_decay = float(repo_goat_sink_decay)
@@ -1480,4 +1344,4 @@ class NeoLLMConfig(PretrainedConfig):
         }
 
 
-__all__ = ["NeoLLMConfig"]
+__all__ = ["NeoLLMConfig"]