Upload EchoChat model components

Add inference weights, tokenizer, audio encoder, configuration, and applicable third-party notices.

LICENSE

@@ -0,0 +1,4 @@
+EchoChat components retain their applicable upstream licenses.
+Model weights and model-specific components: MIT, as declared by their upstream distribution.
+Derived framework code and audio encoder: Apache License 2.0.
+See licenses/ and THIRD_PARTY_NOTICES.md; source-file copyright notices remain applicable.

README.md

@@ -0,0 +1,55 @@
+---
+pretty_name: EchoChat
+language:
+- en
+tags:
+- audio
+- speech
+- conversational
+library_name: transformers
+license: other
+license_name: echochat-component-licenses
+license_link: LICENSE
+---
+
+# EchoChat
+
+EchoChat is an audio-conditioned conversational model with text and speech-token
+outputs. This repository provides the language model, tokenizer, and audio
+encoder weights.
+
+## Files and loading
+
+- Root: language model weights, tokenizer, configuration and custom model code.
+- `audio_encoder/`: complete audio encoder weights, configuration and its custom implementation.
+- `LICENSE`, `licenses/`, `THIRD_PARTY_NOTICES.md`: applicable licenses and notices.
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo = "ddwang2000/EchoChat"
+tokenizer = AutoTokenizer.from_pretrained(repo, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(
+    repo, trust_remote_code=True,
+    torch_dtype=torch.bfloat16,
+    attn_implementation="flash_attention_2",
+).to("cuda").eval()
+```
+
+The custom audio encoder class is `WhisperEncoder` in
+`audio_encoder/modeling_audio_encoder.py`. Instantiate it with the supplied
+configuration and strictly load `audio_encoder/model.safetensors`. Its custom
+attention implementation requires a compatible CUDA/FlashAttention runtime.
+
+This is a component checkpoint, not a standard Transformers text-generation
+pipeline. End-to-end audio use requires compatible feature extraction, prompting,
+and text/speech-token generation logic. Waveform synthesis additionally requires
+a separately licensed compatible decoder, which is not included here.
+
+## Dependencies
+
+The component configuration targets Transformers 4.51.3. Runtime dependencies
+include PyTorch, Transformers, safetensors, tiktoken, einops, and a compatible
+FlashAttention build for CUDA execution. Inspect the custom code before enabling
+`trust_remote_code=True`.

THIRD_PARTY_NOTICES.md

@@ -0,0 +1,7 @@
+# Third-party notices
+
+Model components include code and materials from Moonshot AI / Kimi-Audio, Qwen and Hugging Face. Original source notices are preserved. The upstream distribution declares MIT for its model-specific materials and Apache 2.0 for code derived from Qwen.
+
+The audio encoder uses OpenAI Whisper materials and Hugging Face implementation components under Apache 2.0. Tokenizer code preserves its NVIDIA copyright notice.
+
+Public Python entry points were renamed for this distribution; numeric weight and token interfaces are retained. No waveform decoder is included.

audio_encoder/config.json

@@ -0,0 +1,98 @@
+{
+  "vocab_size": 51866,
+  "num_mel_bins": 128,
+  "d_model": 1280,
+  "encoder_layers": 32,
+  "encoder_attention_heads": 20,
+  "decoder_layers": 32,
+  "decoder_attention_heads": 20,
+  "decoder_ffn_dim": 5120,
+  "encoder_ffn_dim": 5120,
+  "dropout": 0.0,
+  "attention_dropout": 0.0,
+  "activation_dropout": 0.0,
+  "activation_function": "gelu",
+  "init_std": 0.02,
+  "encoder_layerdrop": 0.0,
+  "decoder_layerdrop": 0.0,
+  "use_cache": true,
+  "num_hidden_layers": 32,
+  "scale_embedding": false,
+  "max_source_positions": 1500,
+  "max_target_positions": 448,
+  "classifier_proj_size": 256,
+  "use_weighted_layer_sum": false,
+  "apply_spec_augment": false,
+  "mask_time_prob": 0.05,
+  "mask_time_length": 10,
+  "mask_time_min_masks": 2,
+  "mask_feature_prob": 0.0,
+  "mask_feature_length": 10,
+  "mask_feature_min_masks": 0,
+  "median_filter_width": 7,
+  "return_dict": true,
+  "output_hidden_states": false,
+  "output_attentions": false,
+  "torchscript": false,
+  "torch_dtype": "bfloat16",
+  "use_bfloat16": false,
+  "tf_legacy_loss": false,
+  "pruned_heads": {},
+  "tie_word_embeddings": true,
+  "chunk_size_feed_forward": 0,
+  "is_encoder_decoder": true,
+  "is_decoder": false,
+  "cross_attention_hidden_size": null,
+  "add_cross_attention": false,
+  "tie_encoder_decoder": false,
+  "max_length": 448,
+  "min_length": 0,
+  "do_sample": false,
+  "early_stopping": false,
+  "num_beams": 1,
+  "num_beam_groups": 1,
+  "diversity_penalty": 0.0,
+  "temperature": 1.0,
+  "top_k": 50,
+  "top_p": 1.0,
+  "typical_p": 1.0,
+  "repetition_penalty": 1.0,
+  "length_penalty": 1.0,
+  "no_repeat_ngram_size": 0,
+  "encoder_no_repeat_ngram_size": 0,
+  "bad_words_ids": null,
+  "num_return_sequences": 1,
+  "output_scores": false,
+  "return_dict_in_generate": false,
+  "forced_bos_token_id": null,
+  "forced_eos_token_id": null,
+  "remove_invalid_values": false,
+  "exponential_decay_length_penalty": null,
+  "suppress_tokens": null,
+  "begin_suppress_tokens": [
+    220,
+    50257
+  ],
+  "architectures": [
+    "WhisperEncoder"
+  ],
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1"
+  },
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1
+  },
+  "tokenizer_class": null,
+  "prefix": null,
+  "bos_token_id": 50257,
+  "pad_token_id": 50256,
+  "eos_token_id": 50257,
+  "sep_token_id": null,
+  "decoder_start_token_id": 50258,
+  "problem_type": null,
+  "_attn_implementation_autoset": true,
+  "transformers_version": "4.51.3",
+  "model_type": "whisper"
+}

audio_encoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8365fdce897a4853c454bbabfce6fcfa16b87233d90d419c6a74a7b3e1269f3
+size 1273988176

audio_encoder/modeling_audio_encoder.py

@@ -0,0 +1,1641 @@
+# This script is based on https://github.com/huggingface/transformers/blob/v4.29.1/src/transformers/models/whisper/modeling_whisper.py
+
+# coding=utf-8
+# Copyright 2022 The OpenAI Authors and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""PyTorch Whisper model."""
+
+import math
+import random
+from typing import Optional, Tuple, Union
+
+import numpy as np
+import torch
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn import CrossEntropyLoss
+from torch.nn.utils.rnn import pad_sequence
+from flash_attn import (
+    flash_attn_qkvpacked_func,
+    flash_attn_func,
+    flash_attn_varlen_func,
+    flash_attn_varlen_qkvpacked_func,
+)
+
+try:
+    is_fa3_ready = True
+    from flash_attn_interface import (
+        flash_attn_varlen_func as fa3_varlen_func,
+        flash_attn_func as fa3_func,
+    )
+except:
+    is_fa3_ready = False
+
+from transformers.activations import ACT2FN
+from transformers.generation.logits_process import WhisperTimeStampLogitsProcessor
+from transformers.modeling_outputs import (
+    BaseModelOutput,
+    BaseModelOutputWithPastAndCrossAttentions,
+    Seq2SeqLMOutput,
+    Seq2SeqModelOutput,
+    SequenceClassifierOutput,
+)
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import (
+    add_start_docstrings,
+    add_start_docstrings_to_model_forward,
+    logging,
+    replace_return_docstrings,
+)
+from transformers.models.whisper.configuration_whisper import WhisperConfig
+from transformers.models.whisper.tokenization_whisper import TASK_IDS, TO_LANGUAGE_CODE
+
+
+logger = logging.get_logger(__name__)
+
+_CONFIG_FOR_DOC = "WhisperConfig"
+_CHECKPOINT_FOR_DOC = "openai/whisper-tiny"
+
+
+WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST = [
+    "openai/whisper-base",
+    # See all Whisper models at https://huggingface.co/models?filter=whisper
+]
+
+
+# Copied from transformers.models.bart.modeling_bart.shift_tokens_right
+def shift_tokens_right(
+    input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int
+):
+    """
+    Shift input ids one token to the right.
+    """
+    shifted_input_ids = input_ids.new_zeros(input_ids.shape)
+    shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()
+    shifted_input_ids[:, 0] = decoder_start_token_id
+
+    if pad_token_id is None:
+        raise ValueError("self.model.config.pad_token_id has to be defined.")
+    # replace possible -100 values in labels by `pad_token_id`
+    shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)
+
+    return shifted_input_ids
+
+
+# Copied from transformers.models.bart.modeling_bart._make_causal_mask
+def _make_causal_mask(
+    input_ids_shape: torch.Size,
+    dtype: torch.dtype,
+    device: torch.device,
+    past_key_values_length: int = 0,
+):
+    """
+    Make causal mask used for bi-directional self-attention.
+    """
+    bsz, tgt_len = input_ids_shape
+    mask = torch.full(
+        (tgt_len, tgt_len),
+        torch.tensor(torch.finfo(dtype).min, device=device),
+        device=device,
+    )
+    mask_cond = torch.arange(mask.size(-1), device=device)
+    mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
+    mask = mask.to(dtype)
+
+    if past_key_values_length > 0:
+        mask = torch.cat(
+            [
+                torch.zeros(
+                    tgt_len, past_key_values_length, dtype=dtype, device=device
+                ),
+                mask,
+            ],
+            dim=-1,
+        )
+    return mask[None, None, :, :].expand(
+        bsz, 1, tgt_len, tgt_len + past_key_values_length
+    )
+
+
+# Copied from transformers.models.bart.modeling_bart._expand_mask
+def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):
+    """
+    Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
+    """
+    bsz, src_len = mask.size()
+    tgt_len = tgt_len if tgt_len is not None else src_len
+
+    expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)
+
+    inverted_mask = 1.0 - expanded_mask
+
+    return inverted_mask.masked_fill(
+        inverted_mask.to(torch.bool), torch.finfo(dtype).min
+    )
+
+
+# Copied from transformers.models.wav2vec2.modeling_wav2vec2._compute_mask_indices
+def _compute_mask_indices(
+    shape: Tuple[int, int],
+    mask_prob: float,
+    mask_length: int,
+    attention_mask: Optional[torch.LongTensor] = None,
+    min_masks: int = 0,
+) -> np.ndarray:
+    """
+    Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
+    ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on
+    CPU as part of the preprocessing during training.
+
+    Args:
+        shape: The shape for which to compute masks. This should be of a tuple of size 2 where
+               the first element is the batch size and the second element is the length of the axis to span.
+        mask_prob:  The percentage of the whole axis (between 0 and 1) which will be masked. The number of
+                    independently generated mask spans of length `mask_length` is computed by
+                    `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the
+                    actual percentage will be smaller.
+        mask_length: size of the mask
+        min_masks: minimum number of masked spans
+        attention_mask: A (right-padded) attention mask which independently shortens the feature axis of
+                        each batch dimension.
+    """
+    batch_size, sequence_length = shape
+
+    if mask_length < 1:
+        raise ValueError("`mask_length` has to be bigger than 0.")
+
+    if mask_length > sequence_length:
+        raise ValueError(
+            f"`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: {mask_length}"
+            f" and `sequence_length`: {sequence_length}`"
+        )
+
+    # epsilon is used for probabilistic rounding
+    epsilon = np.random.rand(1).item()
+
+    def compute_num_masked_span(input_length):
+        """Given input length, compute how many spans should be masked"""
+        num_masked_span = int(mask_prob * input_length / mask_length + epsilon)
+        num_masked_span = max(num_masked_span, min_masks)
+
+        # make sure num masked span <= sequence_length
+        if num_masked_span * mask_length > sequence_length:
+            num_masked_span = sequence_length // mask_length
+
+        # make sure num_masked span is also <= input_length - (mask_length - 1)
+        if input_length - (mask_length - 1) < num_masked_span:
+            num_masked_span = max(input_length - (mask_length - 1), 0)
+
+        return num_masked_span
+
+    # compute number of masked spans in batch
+    input_lengths = (
+        attention_mask.sum(-1).detach().tolist()
+        if attention_mask is not None
+        else [sequence_length for _ in range(batch_size)]
+    )
+
+    # SpecAugment mask to fill
+    spec_aug_mask = np.zeros((batch_size, sequence_length), dtype=bool)
+    spec_aug_mask_idxs = []
+
+    max_num_masked_span = compute_num_masked_span(sequence_length)
+
+    if max_num_masked_span == 0:
+        return spec_aug_mask
+
+    for input_length in input_lengths:
+        # compute num of masked spans for this input
+        num_masked_span = compute_num_masked_span(input_length)
+
+        # get random indices to mask
+        spec_aug_mask_idx = np.random.choice(
+            np.arange(input_length - (mask_length - 1)), num_masked_span, replace=False
+        )
+
+        # pick first sampled index that will serve as a dummy index to pad vector
+        # to ensure same dimension for all batches due to probabilistic rounding
+        # Picking first sample just pads those vectors twice.
+        if len(spec_aug_mask_idx) == 0:
+            # this case can only happen if `input_length` is strictly smaller then
+            # `sequence_length` in which case the last token has to be a padding
+            # token which we can use as a dummy mask id
+            dummy_mask_idx = sequence_length - 1
+        else:
+            dummy_mask_idx = spec_aug_mask_idx[0]
+
+        spec_aug_mask_idx = np.concatenate(
+            [
+                spec_aug_mask_idx,
+                np.ones(max_num_masked_span - num_masked_span, dtype=np.int32)
+                * dummy_mask_idx,
+            ]
+        )
+        spec_aug_mask_idxs.append(spec_aug_mask_idx)
+
+    spec_aug_mask_idxs = np.array(spec_aug_mask_idxs)
+
+    # expand masked indices to masked spans
+    spec_aug_mask_idxs = np.broadcast_to(
+        spec_aug_mask_idxs[:, :, None], (batch_size, max_num_masked_span, mask_length)
+    )
+    spec_aug_mask_idxs = spec_aug_mask_idxs.reshape(
+        batch_size, max_num_masked_span * mask_length
+    )
+
+    # add offset to the starting indexes so that indexes now create a span
+    offsets = np.arange(mask_length)[None, None, :]
+    offsets = np.broadcast_to(
+        offsets, (batch_size, max_num_masked_span, mask_length)
+    ).reshape(batch_size, max_num_masked_span * mask_length)
+    spec_aug_mask_idxs = spec_aug_mask_idxs + offsets
+
+    # ensure that we cannot have indices larger than sequence_length
+    if spec_aug_mask_idxs.max() > sequence_length - 1:
+        spec_aug_mask_idxs[spec_aug_mask_idxs > sequence_length - 1] = (
+            sequence_length - 1
+        )
+
+    # scatter indices to mask
+    np.put_along_axis(spec_aug_mask, spec_aug_mask_idxs, 1, -1)
+
+    return spec_aug_mask
+
+
+class WhisperPositionalEmbedding(nn.Embedding):
+    def __init__(
+        self, num_positions: int, embedding_dim: int, padding_idx: Optional[int] = None
+    ):
+        super().__init__(num_positions, embedding_dim)
+
+    def forward(self, input_ids, past_key_values_length=0):
+        return self.weight[
+            past_key_values_length : past_key_values_length + input_ids.shape[1]
+        ]
+
+
+class WhisperAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    def __init__(
+        self,
+        embed_dim: int,
+        num_heads: int,
+        dropout: float = 0.0,
+        is_decoder: bool = False,
+        bias: bool = True,
+    ):
+        super().__init__()
+        self.embed_dim = embed_dim
+        self.num_heads = num_heads
+        self.dropout = dropout
+        self.head_dim = embed_dim // num_heads
+
+        if (self.head_dim * num_heads) != self.embed_dim:
+            raise ValueError(
+                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
+                f" and `num_heads`: {num_heads})."
+            )
+        self.scaling = self.head_dim**-0.5
+        self.is_decoder = is_decoder
+
+        self.k_proj = nn.Linear(embed_dim, embed_dim, bias=False)
+        self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
+        self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
+        self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
+
+    # Copied from transformers.models.bart.modeling_bart.BartAttention._shape with BART->whisper
+    def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
+        return (
+            tensor.view(bsz, seq_len, self.num_heads, self.head_dim)
+            .transpose(1, 2)
+            .contiguous()
+        )
+
+    def forward_flash_attn(
+        self,
+        hidden_states,
+        cu_seqlens_q=None,
+        cu_seqlens_kv=None,
+        max_seqlen_q=None,
+        max_seqlen_kv=None,
+        output_attentions=False,
+    ):
+        """
+        hidden_states: [bsz, tgt_len, embed_dim]
+        seq_lens: Optinal, [bsz]
+        """
+        ### use flash_attn_func when seq_lens is None
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+        if cu_seqlens_q is None:
+            bsz, tgt_len, _ = hidden_states.size()
+            query_states = query_states.view(
+                bsz, tgt_len, self.num_heads, self.head_dim
+            )
+            key_states = key_states.view(bsz, tgt_len, self.num_heads, self.head_dim)
+            value_states = value_states.view(
+                bsz, tgt_len, self.num_heads, self.head_dim
+            )
+
+            if is_fa3_ready:
+                attn_output, attn_probs_ret = fa3_func(
+                    query_states,
+                    key_states,
+                    value_states,
+                    softmax_scale=self.scaling,
+                    causal=self.is_decoder,
+                )
+                if output_attentions:
+                    attn_probs = attn_probs_ret
+                else:
+                    attn_probs = None
+            else:
+                attn_output = flash_attn_func(
+                    query_states,
+                    key_states,
+                    value_states,
+                    dropout_p=self.dropout,
+                    softmax_scale=self.scaling,
+                    causal=self.is_decoder,
+                    return_attn_probs=output_attentions,
+                )
+                if output_attentions:
+                    attn_output, attn_probs = attn_output
+                else:
+                    attn_probs = None
+            attn_output = attn_output.view(bsz, tgt_len, self.embed_dim)
+        else:
+            query_states = query_states.view(-1, self.num_heads, self.head_dim)
+            key_states = key_states.view(-1, self.num_heads, self.head_dim)
+            value_states = value_states.view(-1, self.num_heads, self.head_dim)
+            if is_fa3_ready:
+                attn_output, attn_probs_ret = fa3_varlen_func(
+                    q=query_states,
+                    k=key_states,
+                    v=value_states,
+                    cu_seqlens_q=cu_seqlens_q,
+                    cu_seqlens_k=cu_seqlens_kv,
+                    max_seqlen_q=max_seqlen_q,
+                    max_seqlen_k=max_seqlen_kv,
+                    softmax_scale=self.scaling,
+                    causal=self.is_decoder,
+                )
+                if output_attentions:
+                    attn_probs = attn_probs_ret
+                else:
+                    attn_probs = None
+            else:
+                attn_output = flash_attn_varlen_func(
+                    q=query_states,
+                    k=key_states,
+                    v=value_states,
+                    cu_seqlens_q=cu_seqlens_q,
+                    cu_seqlens_k=cu_seqlens_kv,
+                    max_seqlen_q=max_seqlen_q,
+                    max_seqlen_k=max_seqlen_kv,
+                    dropout_p=self.dropout,
+                    softmax_scale=self.scaling,
+                    causal=self.is_decoder,
+                    return_attn_probs=output_attentions,
+                )
+                if output_attentions:
+                    attn_output, attn_probs = attn_output
+                else:
+                    attn_probs = None
+            attn_output = attn_output.view(-1, self.embed_dim)
+
+        ret = self.out_proj(attn_output)
+
+        return ret, attn_probs
+
+    # Copied from transformers.models.bart.modeling_bart.BartAttention.forward with BART->whisper
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        key_value_states: Optional[torch.Tensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        layer_head_mask: Optional[torch.Tensor] = None,
+        output_attentions: bool = False,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        """Input shape: Batch x Time x Channel"""
+
+        # if key_value_states are provided this layer is used as a cross-attention layer
+        # for the decoder
+        is_cross_attention = key_value_states is not None
+
+        bsz, tgt_len, _ = hidden_states.size()
+
+        # get query proj
+        query_states = self.q_proj(hidden_states) * self.scaling
+        # get key, value proj
+        # `past_key_value[0].shape[2] == key_value_states.shape[1]`
+        # is checking that the `sequence_length` of the `past_key_value` is the same as
+        # the provided `key_value_states` to support prefix tuning
+        if (
+            is_cross_attention
+            and past_key_value is not None
+            and past_key_value[0].shape[2] == key_value_states.shape[1]
+        ):
+            # reuse k,v, cross_attentions
+            key_states = past_key_value[0]
+            value_states = past_key_value[1]
+        elif is_cross_attention:
+            # cross_attentions
+            key_states = self._shape(self.k_proj(key_value_states), -1, bsz)
+            value_states = self._shape(self.v_proj(key_value_states), -1, bsz)
+        elif past_key_value is not None:
+            # reuse k, v, self_attention
+            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
+            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
+            key_states = torch.cat([past_key_value[0], key_states], dim=2)
+            value_states = torch.cat([past_key_value[1], value_states], dim=2)
+        else:
+            # self_attention
+            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
+            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
+
+        if self.is_decoder:
+            # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states.
+            # Further calls to cross_attention layer can then reuse all cross-attention
+            # key/value_states (first "if" case)
+            # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of
+            # all previous decoder key/value_states. Further calls to uni-directional self-attention
+            # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
+            # if encoder bi-directional self-attention `past_key_value` is always `None`
+            past_key_value = (key_states, value_states)
+
+        proj_shape = (bsz * self.num_heads, -1, self.head_dim)
+        query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape)
+        key_states = key_states.reshape(*proj_shape)
+        value_states = value_states.reshape(*proj_shape)
+
+        src_len = key_states.size(1)
+        attn_weights = torch.bmm(query_states, key_states.transpose(1, 2))
+
+        if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len):
+            raise ValueError(
+                f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is"
+                f" {attn_weights.size()}"
+            )
+
+        if attention_mask is not None:
+            if attention_mask.size() != (bsz, 1, tgt_len, src_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}"
+                )
+            attn_weights = (
+                attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
+                + attention_mask
+            )
+            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)
+
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1)
+
+        if layer_head_mask is not None:
+            if layer_head_mask.size() != (self.num_heads,):
+                raise ValueError(
+                    f"Head mask for a single layer should be of size {(self.num_heads,)}, but is"
+                    f" {layer_head_mask.size()}"
+                )
+            attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(
+                bsz, self.num_heads, tgt_len, src_len
+            )
+            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)
+
+        if output_attentions:
+            # this operation is a bit awkward, but it's required to
+            # make sure that attn_weights keeps its gradient.
+            # In order to do so, attn_weights have to be reshaped
+            # twice and have to be reused in the following
+            attn_weights_reshaped = attn_weights.view(
+                bsz, self.num_heads, tgt_len, src_len
+            )
+            attn_weights = attn_weights_reshaped.view(
+                bsz * self.num_heads, tgt_len, src_len
+            )
+        else:
+            attn_weights_reshaped = None
+
+        attn_probs = nn.functional.dropout(
+            attn_weights, p=self.dropout, training=self.training
+        )
+
+        attn_output = torch.bmm(attn_probs, value_states)
+
+        if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim):
+            raise ValueError(
+                f"`attn_output` should be of size {(bsz * self.num_heads, tgt_len, self.head_dim)}, but is"
+                f" {attn_output.size()}"
+            )
+
+        attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim)
+        attn_output = attn_output.transpose(1, 2)
+
+        # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be
+        # partitioned across GPUs when using tensor-parallelism.
+        attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim)
+
+        attn_output = self.out_proj(attn_output)
+
+        return attn_output, attn_weights_reshaped, past_key_value
+
+
+# Copied from transformers.models.mbart.modeling_mbart.MBartEncoderLayer with MBart->Whisper
+class WhisperEncoderLayer(nn.Module):
+    def __init__(self, config: WhisperConfig):
+        super().__init__()
+        self.embed_dim = config.d_model
+        self.self_attn = WhisperAttention(
+            embed_dim=self.embed_dim,
+            num_heads=config.encoder_attention_heads,
+            dropout=config.attention_dropout,
+        )
+        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
+        self.dropout = config.dropout
+        self.activation_fn = ACT2FN[config.activation_function]
+        self.activation_dropout = config.activation_dropout
+        self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim)
+        self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim)
+        self.final_layer_norm = nn.LayerNorm(self.embed_dim)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        cu_seqlens_q: torch.Tensor = None,
+        cu_seqlens_kv: torch.Tensor = None,
+        max_seqlen_q: int = None,
+        max_seqlen_kv: int = None,
+        output_attentions: bool = False,
+    ) -> torch.Tensor:
+        """
+        Args:
+            hidden_states (`torch.FloatTensor`): input to the layer of shape `(seq_len, batch, embed_dim)`
+            attention_mask (`torch.FloatTensor`): attention mask of size
+                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
+            layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
+                `(encoder_attention_heads,)`.
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+        """
+        residual = hidden_states
+        hidden_states = self.self_attn_layer_norm(hidden_states)
+
+        hidden_states, attn_weights = self.self_attn.forward_flash_attn(
+            hidden_states=hidden_states,
+            cu_seqlens_q=cu_seqlens_q,
+            cu_seqlens_kv=cu_seqlens_kv,
+            max_seqlen_q=max_seqlen_q,
+            max_seqlen_kv=max_seqlen_kv,
+            output_attentions=output_attentions,
+        )
+        # ret_vallina_attn, attn_weights, _ = self.self_attn(
+        #     hidden_states=hidden_states,
+        #     attention_mask=attention_mask,
+        #     layer_head_mask=layer_head_mask,
+        #     output_attentions=output_attentions,
+        # )
+
+        # hidden_states, attn_weights, _ = self.self_attn(
+        #     hidden_states=hidden_states,
+        #     attention_mask=attention_mask,
+        #     layer_head_mask=layer_head_mask,
+        #     output_attentions=output_attentions,
+        # )
+        hidden_states = nn.functional.dropout(
+            hidden_states, p=self.dropout, training=self.training
+        )
+        hidden_states = residual + hidden_states
+
+        residual = hidden_states
+        hidden_states = self.final_layer_norm(hidden_states)
+        hidden_states = self.activation_fn(self.fc1(hidden_states))
+        hidden_states = nn.functional.dropout(
+            hidden_states, p=self.activation_dropout, training=self.training
+        )
+        hidden_states = self.fc2(hidden_states)
+        hidden_states = nn.functional.dropout(
+            hidden_states, p=self.dropout, training=self.training
+        )
+        hidden_states = residual + hidden_states
+
+        if hidden_states.dtype == torch.float16 and (
+            torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any()
+        ):
+            clamp_value = torch.finfo(hidden_states.dtype).max - 1000
+            hidden_states = torch.clamp(
+                hidden_states, min=-clamp_value, max=clamp_value
+            )
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (attn_weights,)
+
+        return outputs
+
+
+# Copied from transformers.models.mbart.modeling_mbart.MBartDecoderLayer with MBart->Whisper
+class WhisperDecoderLayer(nn.Module):
+    def __init__(self, config: WhisperConfig):
+        super().__init__()
+        self.embed_dim = config.d_model
+
+        self.self_attn = WhisperAttention(
+            embed_dim=self.embed_dim,
+            num_heads=config.decoder_attention_heads,
+            dropout=config.attention_dropout,
+            is_decoder=True,
+        )
+        self.dropout = config.dropout
+        self.activation_fn = ACT2FN[config.activation_function]
+        self.activation_dropout = config.activation_dropout
+
+        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
+        self.encoder_attn = WhisperAttention(
+            self.embed_dim,
+            config.decoder_attention_heads,
+            dropout=config.attention_dropout,
+            is_decoder=True,
+        )
+        self.encoder_attn_layer_norm = nn.LayerNorm(self.embed_dim)
+        self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim)
+        self.fc2 = nn.Linear(config.decoder_ffn_dim, self.embed_dim)
+        self.final_layer_norm = nn.LayerNorm(self.embed_dim)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.Tensor] = None,
+        layer_head_mask: Optional[torch.Tensor] = None,
+        cross_attn_layer_head_mask: Optional[torch.Tensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: Optional[bool] = False,
+        use_cache: Optional[bool] = True,
+    ) -> torch.Tensor:
+        """
+        Args:
+            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+            attention_mask (`torch.FloatTensor`): attention mask of size
+                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
+            encoder_hidden_states (`torch.FloatTensor`):
+                cross attention input to the layer of shape `(batch, seq_len, embed_dim)`
+            encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
+                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
+            layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
+                `(encoder_attention_heads,)`.
+            cross_attn_layer_head_mask (`torch.FloatTensor`): mask for cross-attention heads in a given layer of
+                size `(decoder_attention_heads,)`.
+            past_key_value (`Tuple(torch.FloatTensor)`): cached past key and value projection states
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+        """
+        residual = hidden_states
+        hidden_states = self.self_attn_layer_norm(hidden_states)
+
+        # Self Attention
+        # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
+        self_attn_past_key_value = (
+            past_key_value[:2] if past_key_value is not None else None
+        )
+        # add present self-attn cache to positions 1,2 of present_key_value tuple
+        hidden_states, self_attn_weights, present_key_value = self.self_attn(
+            hidden_states=hidden_states,
+            past_key_value=self_attn_past_key_value,
+            attention_mask=attention_mask,
+            layer_head_mask=layer_head_mask,
+            output_attentions=output_attentions,
+        )
+        hidden_states = nn.functional.dropout(
+            hidden_states, p=self.dropout, training=self.training
+        )
+        hidden_states = residual + hidden_states
+
+        # Cross-Attention Block
+        cross_attn_present_key_value = None
+        cross_attn_weights = None
+        if encoder_hidden_states is not None:
+            residual = hidden_states
+            hidden_states = self.encoder_attn_layer_norm(hidden_states)
+
+            # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple
+            cross_attn_past_key_value = (
+                past_key_value[-2:] if past_key_value is not None else None
+            )
+            hidden_states, cross_attn_weights, cross_attn_present_key_value = (
+                self.encoder_attn(
+                    hidden_states=hidden_states,
+                    key_value_states=encoder_hidden_states,
+                    attention_mask=encoder_attention_mask,
+                    layer_head_mask=cross_attn_layer_head_mask,
+                    past_key_value=cross_attn_past_key_value,
+                    output_attentions=output_attentions,
+                )
+            )
+            hidden_states = nn.functional.dropout(
+                hidden_states, p=self.dropout, training=self.training
+            )
+            hidden_states = residual + hidden_states
+
+            # add cross-attn to positions 3,4 of present_key_value tuple
+            present_key_value = present_key_value + cross_attn_present_key_value
+
+        # Fully Connected
+        residual = hidden_states
+        hidden_states = self.final_layer_norm(hidden_states)
+        hidden_states = self.activation_fn(self.fc1(hidden_states))
+        hidden_states = nn.functional.dropout(
+            hidden_states, p=self.activation_dropout, training=self.training
+        )
+        hidden_states = self.fc2(hidden_states)
+        hidden_states = nn.functional.dropout(
+            hidden_states, p=self.dropout, training=self.training
+        )
+        hidden_states = residual + hidden_states
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (self_attn_weights, cross_attn_weights)
+
+        if use_cache:
+            outputs += (present_key_value,)
+
+        return outputs
+
+
+class WhisperPreTrainedModel(PreTrainedModel):
+    config_class = WhisperConfig
+    base_model_prefix = "model"
+    main_input_name = "input_features"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["WhisperEncoderLayer", "WhisperDecoderLayer"]
+
+    def _init_weights(self, module):
+        std = self.config.init_std
+        if isinstance(module, (nn.Linear, nn.Conv1d)):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+
+    def _set_gradient_checkpointing(self, module, value=False):
+        if isinstance(module, (WhisperDecoder, WhisperEncoder)):
+            module.gradient_checkpointing = value
+
+    def _get_feat_extract_output_lengths(self, input_lengths: torch.LongTensor):
+        """
+        Computes the output length of the convolutional layers
+        """
+        input_lengths = (input_lengths - 1) // 2 + 1
+
+        return input_lengths
+
+
+WHISPER_START_DOCSTRING = r"""
+    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+    etc.)
+
+    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+    and behavior.
+
+    Parameters:
+        config ([`WhisperConfig`]):
+            Model configuration class with all the parameters of the model. Initializing with a config file does not
+            load the weights associated with the model, only the configuration. Check out the
+            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+WHISPER_INPUTS_DOCSTRING = r"""
+    Args:
+        input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, sequence_length)`):
+            Float values mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by
+            loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via
+            the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the
+            [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a
+            tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`]
+        attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Mask to avoid performing *SpecAugment* data augmentation on padding token indices. Mask values selected in
+            `[0, 1]`:
+
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+
+            [What are attention masks?](../glossary#attention-mask)
+        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
+            Indices of decoder input sequence tokens in the vocabulary.
+
+            Indices can be obtained using [`WhisperTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            [What are decoder input IDs?](../glossary#decoder-input-ids)
+
+            Whisper uses the `decoder_start_token_id` as the starting token for `decoder_input_ids` generation. If
+            `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
+            `past_key_values`).
+        decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
+            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+            be used by default.
+
+            If you want to change padding behavior, you should read
+            [`modeling_whisper._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in [the BART
+            paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
+        head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
+            Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`:
+
+            - 1 indicates the head is **not masked**,
+            - 0 indicates the head is **masked**.
+
+        decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
+            Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`:
+
+            - 1 indicates the head is **not masked**,
+            - 0 indicates the head is **masked**.
+
+        cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
+            Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`:
+
+            - 1 indicates the head is **not masked**,
+            - 0 indicates the head is **masked**.
+
+        encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*):
+            Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+            `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+            hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
+        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+            `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+            `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+
+            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
+            blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
+
+            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+        decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
+            Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
+            representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be
+            input (see `past_key_values`). This is useful if you want more control over how to convert
+            `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix.
+        use_cache (`bool`, *optional*):
+            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+            `past_key_values`).
+        output_attentions (`bool`, *optional*):
+            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+            tensors for more detail.
+        output_hidden_states (`bool`, *optional*):
+            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+            more detail.
+        return_dict (`bool`, *optional*):
+            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+WHISPER_ENCODER_INPUTS_DOCSTRING = r"""
+    Args:
+        input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, sequence_length)`):
+            Float values mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by
+            loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via
+            the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the
+            [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a
+            tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`]
+        head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
+            Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`:
+
+            - 1 indicates the head is **not masked**,
+            - 0 indicates the head is **masked**.
+        encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*):
+            Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+            `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+            hidden-states at the output of the last layer of the encoder.
+        output_attentions (`bool`, *optional*):
+            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+            tensors for more detail.
+        output_hidden_states (`bool`, *optional*):
+            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+            more detail.
+        return_dict (`bool`, *optional*):
+            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+class WhisperEncoder(WhisperPreTrainedModel):
+    """
+    Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a
+    [`WhisperEncoderLayer`].
+
+    Args:
+        config: WhisperConfig
+    """
+
+    def __init__(self, config: WhisperConfig):
+        super().__init__(config)
+        self.dropout = config.dropout
+        self.layerdrop = config.encoder_layerdrop
+
+        embed_dim = config.d_model
+        self.num_mel_bins = 128
+        self.padding_idx = config.pad_token_id
+        self.max_source_positions = config.max_source_positions
+        self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0
+
+        self.conv1 = nn.Conv1d(self.num_mel_bins, embed_dim, kernel_size=3, padding=1)
+        self.conv2 = nn.Conv1d(embed_dim, embed_dim, kernel_size=3, stride=2, padding=1)
+
+        self.embed_positions = nn.Embedding(self.max_source_positions, embed_dim)
+
+        self.layers = nn.ModuleList(
+            [WhisperEncoderLayer(config) for _ in range(config.encoder_layers)]
+        )
+        self.layer_norm = nn.LayerNorm(config.d_model)
+
+        # args = get_args()
+        self.gradient_checkpointing = False  # args.kimia_whisper_gradient_checkpointing
+        self.gradient_checkpointing_first_layer = (
+            False  # args.kimia_whisper_gradient_checkpointing_first_layer
+        )
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def _freeze_parameters(self):
+        for param in self.parameters():
+            param.requires_grad = False
+        self._requires_grad = False
+
+    def get_input_embeddings(self) -> nn.Module:
+        return self.conv1
+
+    def set_input_embeddings(self, value: nn.Module):
+        self.conv1 = value
+
+    def forward(
+        self,
+        input_features,
+        input_seq_lens=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+    ):
+        r"""
+        Args:
+            input_features (`torch.LongTensor` of shape `(batch_size, feature_size, sequence_length)`):
+                Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be
+                obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a
+                `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into
+                `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding
+                and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`]
+            input_seq_lens (`torch.Tensor`)`, *optional*):
+                The sequence lengths of the input mel features.
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            output_hidden_states (`bool`, *optional*):
+                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
+                for more detail.
+            return_dict (`bool`, *optional*):
+                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+        """
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
+        output_hidden_states = (
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
+        )
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+        inputs_embeds = nn.functional.gelu(self.conv1(input_features))
+        inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds))
+
+        inputs_embeds = inputs_embeds.permute(0, 2, 1)
+        embed_pos = self.embed_positions.weight
+
+        hidden_states = inputs_embeds + embed_pos
+        hidden_states = nn.functional.dropout(
+            hidden_states, p=self.dropout, training=self.training
+        )
+
+        encoder_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+
+        # if input_seq_lens is None:
+        #     input_seq_lens = torch.zeros(hidden_states.shape[0], dtype=torch.int32).fill_(hidden_states.shape[1]).to(hidden_states.device).long()
+
+        if input_seq_lens is not None:
+            # build cu_seqlens_q, cu_seqlens_kv, max_seqlen_q, max_seqlen_kv and q, k, v by seq_lens
+            B, S, D = hidden_states.shape
+            cu_seqlens_q = torch.zeros(B + 1, dtype=torch.int32).to(
+                hidden_states.device
+            )
+            cu_seqlens_kv = torch.zeros(B + 1, dtype=torch.int32).to(
+                hidden_states.device
+            )
+            max_seqlen_q = torch.max(input_seq_lens).to(hidden_states.device)
+            max_seqlen_kv = torch.max(input_seq_lens).to(hidden_states.device)
+
+            hidden_states_collect = []
+            for i in range(B):
+                hidden_states_collect.append(hidden_states[i, : input_seq_lens[i], :])
+                cu_seqlens_q[i + 1] = cu_seqlens_q[i] + input_seq_lens[i]
+                cu_seqlens_kv[i + 1] = cu_seqlens_kv[i] + input_seq_lens[i]
+                max_seqlen_q = max(max_seqlen_q, input_seq_lens[i])
+                max_seqlen_kv = max(max_seqlen_kv, input_seq_lens[i])
+
+            hidden_states = torch.cat(hidden_states_collect, dim=0)
+        else:
+            cu_seqlens_q = None
+            cu_seqlens_kv = None
+            max_seqlen_q = None
+            max_seqlen_kv = None
+
+        for idx, encoder_layer in enumerate(self.layers):
+            if output_hidden_states:
+                encoder_states = encoder_states + (hidden_states,)
+            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
+            dropout_probability = random.uniform(0, 1)
+            if self.training and (
+                dropout_probability < self.layerdrop
+            ):  # skip the layer
+                layer_outputs = (None, None)
+            else:
+                if (
+                    self.gradient_checkpointing
+                    and self.training
+                    and (
+                        self.gradient_checkpointing_first_layer == -1
+                        or idx <= self.gradient_checkpointing_first_layer
+                    )
+                ):
+
+                    def create_custom_forward(module):
+                        def custom_forward(*inputs):
+                            return module(*inputs, output_attentions)
+
+                        return custom_forward
+
+                    layer_outputs = torch.utils.checkpoint.checkpoint(
+                        create_custom_forward(encoder_layer),
+                        hidden_states,
+                        cu_seqlens_q,
+                        cu_seqlens_kv,
+                        max_seqlen_q,
+                        max_seqlen_kv,
+                    )
+                else:
+                    layer_outputs = encoder_layer(
+                        hidden_states,
+                        cu_seqlens_q,
+                        cu_seqlens_kv,
+                        max_seqlen_q,
+                        max_seqlen_kv,
+                        output_attentions=output_attentions,
+                    )
+
+                hidden_states = layer_outputs[0]
+
+            if output_attentions:
+                all_attentions = all_attentions + (layer_outputs[1],)
+
+        if input_seq_lens is not None:
+            # recover the hidden_states, split by seq_lens and then pad&cat
+            hidden_states = torch.split(
+                hidden_states, input_seq_lens.detach().cpu().numpy().tolist(), dim=0
+            )
+            hidden_states = pad_sequence(hidden_states, batch_first=True)
+
+        hidden_states = self.layer_norm(hidden_states)
+        if output_hidden_states:
+            encoder_states = encoder_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v
+                for v in [hidden_states, encoder_states, all_attentions]
+                if v is not None
+            )
+        return BaseModelOutput(
+            last_hidden_state=hidden_states,
+            hidden_states=encoder_states,
+            attentions=all_attentions,
+        )
+
+
+class WhisperDecoder(WhisperPreTrainedModel):
+    """
+    Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`WhisperDecoderLayer`]
+
+    Args:
+        config: WhisperConfig
+    """
+
+    def __init__(self, config: WhisperConfig):
+        super().__init__(config)
+        self.dropout = config.dropout
+        self.layerdrop = config.decoder_layerdrop
+        self.padding_idx = config.pad_token_id
+        self.max_target_positions = config.max_target_positions
+        self.max_source_positions = config.max_source_positions
+        self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0
+
+        self.embed_tokens = nn.Embedding(
+            config.vocab_size, config.d_model, self.padding_idx
+        )
+        self.embed_positions = WhisperPositionalEmbedding(
+            self.max_target_positions, config.d_model
+        )
+
+        self.layers = nn.ModuleList(
+            [WhisperDecoderLayer(config) for _ in range(config.decoder_layers)]
+        )
+
+        self.layer_norm = nn.LayerNorm(config.d_model)
+
+        self.gradient_checkpointing = False
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.embed_tokens = value
+
+    def _prepare_decoder_attention_mask(
+        self, attention_mask, input_shape, inputs_embeds, past_key_values_length
+    ):
+        # create causal mask
+        # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
+        combined_attention_mask = None
+
+        if input_shape[-1] > 1:
+            combined_attention_mask = _make_causal_mask(
+                input_shape,
+                inputs_embeds.dtype,
+                device=inputs_embeds.device,
+                past_key_values_length=past_key_values_length,
+            )
+
+        if attention_mask is not None:
+            # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
+            expanded_attn_mask = _expand_mask(
+                attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]
+            )
+            combined_attention_mask = (
+                expanded_attn_mask
+                if combined_attention_mask is None
+                else expanded_attn_mask + combined_attention_mask
+            )
+
+        return combined_attention_mask
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        encoder_hidden_states=None,
+        head_mask=None,
+        cross_attn_head_mask=None,
+        past_key_values=None,
+        inputs_embeds=None,
+        use_cache=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+    ):
+        r"""
+        Args:
+            input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+                Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
+                provide it.
+
+                Indices can be obtained using [`WhisperTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+                [`PreTrainedTokenizer.__call__`] for details.
+
+                [What are input IDs?](../glossary#input-ids)
+            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+                - 1 for tokens that are **not masked**,
+                - 0 for tokens that are **masked**.
+
+                [What are attention masks?](../glossary#attention-mask)
+            encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
+                Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
+                of the decoder.
+            head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
+                Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
+
+                - 1 indicates the head is **not masked**,
+                - 0 indicates the head is **masked**.
+
+            cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
+                Mask to nullify selected heads of the attention modules in encoder to avoid performing cross-attention
+                on hidden heads. Mask values selected in `[0, 1]`:
+
+                - 1 indicates the head is **not masked**,
+                - 0 indicates the head is **masked**.
+
+            past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+                Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+                shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+                shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+
+                Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
+                cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
+
+                If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+                that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+                all `decoder_input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of
+                shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+                `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+                control over how to convert `input_ids` indices into associated vectors than the model's internal
+                embedding lookup matrix.
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            output_hidden_states (`bool`, *optional*):
+                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
+                for more detail.
+            return_dict (`bool`, *optional*):
+                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+        """
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
+        output_hidden_states = (
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+
+        # retrieve input_ids and inputs_embeds
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError(
+                "You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time"
+            )
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+            input_ids = input_ids.view(-1, input_shape[-1])
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+        else:
+            raise ValueError(
+                "You have to specify either decoder_input_ids or decoder_inputs_embeds"
+            )
+
+        # past_key_values_length
+        past_key_values_length = (
+            past_key_values[0][0].shape[2] if past_key_values is not None else 0
+        )
+
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+
+        attention_mask = self._prepare_decoder_attention_mask(
+            attention_mask, input_shape, inputs_embeds, past_key_values_length
+        )
+
+        # embed positions
+        if input_ids is not None:
+            positions = self.embed_positions(
+                input_ids, past_key_values_length=past_key_values_length
+            )
+        else:
+            positions = self.embed_positions(
+                inputs_embeds, past_key_values_length=past_key_values_length
+            )
+
+        hidden_states = inputs_embeds + positions
+        hidden_states = nn.functional.dropout(
+            hidden_states, p=self.dropout, training=self.training
+        )
+
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                logger.warning_once(
+                    "`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache = False`..."
+                )
+                use_cache = False
+        # decoder layers
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attns = () if output_attentions else None
+        all_cross_attentions = (
+            () if (output_attentions and encoder_hidden_states is not None) else None
+        )
+        next_decoder_cache = () if use_cache else None
+
+        # check if head_mask/cross_attn_head_mask has a correct number of layers specified if desired
+        for attn_mask, mask_name in zip(
+            [head_mask, cross_attn_head_mask], ["head_mask", "cross_attn_head_mask"]
+        ):
+            if attn_mask is not None:
+                assert attn_mask.size()[0] == (len(self.layers)), (
+                    f"The `{mask_name}` should be specified for {len(self.layers)} layers, but it is for"
+                    f" {head_mask.size()[0]}."
+                )
+        for idx, decoder_layer in enumerate(self.layers):
+            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
+            if output_hidden_states:
+                all_hidden_states += (hidden_states,)
+            dropout_probability = random.uniform(0, 1)
+            if self.training and (dropout_probability < self.layerdrop):
+                continue
+
+            past_key_value = (
+                past_key_values[idx] if past_key_values is not None else None
+            )
+
+            if self.gradient_checkpointing and self.training:
+
+                def create_custom_forward(module):
+                    def custom_forward(*inputs):
+                        # None for past_key_value
+                        return module(*inputs, output_attentions, use_cache)
+
+                    return custom_forward
+
+                layer_outputs = torch.utils.checkpoint.checkpoint(
+                    create_custom_forward(decoder_layer),
+                    hidden_states,
+                    attention_mask,
+                    encoder_hidden_states,
+                    None,  # encoder attention mask
+                    head_mask[idx] if head_mask is not None else None,
+                    (
+                        cross_attn_head_mask[idx]
+                        if cross_attn_head_mask is not None
+                        else None
+                    ),
+                    None,  # past_key_value
+                )
+            else:
+                layer_outputs = decoder_layer(
+                    hidden_states,
+                    attention_mask=attention_mask,
+                    encoder_hidden_states=encoder_hidden_states,
+                    layer_head_mask=(head_mask[idx] if head_mask is not None else None),
+                    cross_attn_layer_head_mask=(
+                        cross_attn_head_mask[idx]
+                        if cross_attn_head_mask is not None
+                        else None
+                    ),
+                    past_key_value=past_key_value,
+                    output_attentions=output_attentions,
+                    use_cache=use_cache,
+                )
+            hidden_states = layer_outputs[0]
+
+            if use_cache:
+                next_decoder_cache += (layer_outputs[3 if output_attentions else 1],)
+
+            if output_attentions:
+                all_self_attns += (layer_outputs[1],)
+
+                if encoder_hidden_states is not None:
+                    all_cross_attentions += (layer_outputs[2],)
+
+        hidden_states = self.layer_norm(hidden_states)
+        # add hidden states from the last decoder layer
+        if output_hidden_states:
+            all_hidden_states += (hidden_states,)
+
+        next_cache = next_decoder_cache if use_cache else None
+        if not return_dict:
+            return tuple(
+                v
+                for v in [
+                    hidden_states,
+                    next_cache,
+                    all_hidden_states,
+                    all_self_attns,
+                    all_cross_attentions,
+                ]
+                if v is not None
+            )
+        return BaseModelOutputWithPastAndCrossAttentions(
+            last_hidden_state=hidden_states,
+            past_key_values=next_cache,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attns,
+            cross_attentions=all_cross_attentions,
+        )
+
+
+@add_start_docstrings(
+    "The bare Whisper Model outputting raw hidden-states without any specific head on top.",
+    WHISPER_START_DOCSTRING,
+)
+class WhisperModel(WhisperPreTrainedModel):
+    _keys_to_ignore_on_load_missing = [r"proj_out.weight"]
+
+    def __init__(self, config: WhisperConfig):
+        super().__init__(config)
+
+        self.encoder = WhisperEncoder(config)
+        self.decoder = WhisperDecoder(config)
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.decoder.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.decoder.embed_tokens = value
+
+    def get_encoder(self):
+        return self.encoder
+
+    def get_decoder(self):
+        return self.decoder
+
+    def freeze_encoder(self):
+        """
+        Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will
+        not be updated during training.
+        """
+        self.encoder._freeze_parameters()
+
+    def _mask_input_features(
+        self,
+        input_features: torch.FloatTensor,
+        attention_mask: Optional[torch.LongTensor] = None,
+    ):
+        """
+        Masks extracted features along time axis and/or along feature axis according to
+        [SpecAugment](https://arxiv.org/abs/1904.08779).
+        """
+
+        # `config.apply_spec_augment` can set masking to False
+        if not getattr(self.config, "apply_spec_augment", True):
+            return input_features
+
+        # generate indices & apply SpecAugment along time axis
+        batch_size, hidden_size, sequence_length = input_features.size()
+
+        if self.config.mask_time_prob > 0 and self.training:
+            # generate indices & apply SpecAugment along time axis
+            mask_time_indices = _compute_mask_indices(
+                (batch_size, sequence_length),
+                mask_prob=self.config.mask_time_prob,
+                mask_length=self.config.mask_time_length,
+                attention_mask=attention_mask,
+                min_masks=self.config.mask_time_min_masks,
+            )
+            mask_time_indices = torch.tensor(
+                mask_time_indices, device=input_features.device, dtype=torch.bool
+            )
+            mask_time_indices = mask_time_indices[:, None].expand(-1, hidden_size, -1)
+            input_features[mask_time_indices] = 0
+
+        if self.config.mask_feature_prob > 0 and self.training:
+            # generate indices & apply SpecAugment along feature axis
+            mask_feature_indices = _compute_mask_indices(
+                (batch_size, hidden_size),
+                mask_prob=self.config.mask_feature_prob,
+                mask_length=self.config.mask_feature_length,
+                min_masks=self.config.mask_feature_min_masks,
+            )
+            mask_feature_indices = torch.tensor(
+                mask_feature_indices, device=input_features.device, dtype=torch.bool
+            )
+            input_features[mask_feature_indices] = 0
+
+        return input_features
+
+    @add_start_docstrings_to_model_forward(WHISPER_INPUTS_DOCSTRING)
+    @replace_return_docstrings(
+        output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC
+    )
+    def forward(
+        self,
+        input_features: Optional[torch.FloatTensor] = None,
+        attention_mask: Optional[torch.LongTensor] = None,
+        decoder_input_ids: Optional[torch.LongTensor] = None,
+        decoder_attention_mask: Optional[torch.LongTensor] = None,
+        head_mask: Optional[torch.Tensor] = None,
+        decoder_head_mask: Optional[torch.Tensor] = None,
+        cross_attn_head_mask: Optional[torch.Tensor] = None,
+        encoder_outputs: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
+        decoder_inputs_embeds: Optional[Tuple[torch.FloatTensor]] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple[torch.Tensor], Seq2SeqModelOutput]:
+        r"""
+        Returns:
+
+        Example:
+         ```python
+         >>> import torch
+         >>> from transformers import AutoFeatureExtractor, WhisperModel
+         >>> from datasets import load_dataset
+
+         >>> model = WhisperModel.from_pretrained("openai/whisper-base")
+         >>> feature_extractor = AutoFeatureExtractor.from_pretrained("openai/whisper-base")
+         >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+         >>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="pt")
+         >>> input_features = inputs.input_features
+         >>> decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id
+         >>> last_hidden_state = model(input_features, decoder_input_ids=decoder_input_ids).last_hidden_state
+         >>> list(last_hidden_state.shape)
+         [1, 2, 512]
+         ```"""
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
+        output_hidden_states = (
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+
+        if encoder_outputs is None:
+            input_features = self._mask_input_features(
+                input_features, attention_mask=attention_mask
+            )
+
+            encoder_outputs = self.encoder(
+                input_features,
+                head_mask=head_mask,
+                output_attentions=output_attentions,
+                output_hidden_states=output_hidden_states,
+                return_dict=return_dict,
+            )
+        # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True
+        elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
+            encoder_outputs = BaseModelOutput(
+                last_hidden_state=encoder_outputs[0],
+                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
+                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
+            )
+
+        # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn)
+        decoder_outputs = self.decoder(
+            input_ids=decoder_input_ids,
+            attention_mask=decoder_attention_mask,
+            encoder_hidden_states=encoder_outputs[0],
+            head_mask=decoder_head_mask,
+            cross_attn_head_mask=cross_attn_head_mask,
+            past_key_values=past_key_values,
+            inputs_embeds=decoder_inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        if not return_dict:
+            return decoder_outputs + encoder_outputs
+
+        return Seq2SeqModelOutput(
+            last_hidden_state=decoder_outputs.last_hidden_state,
+            past_key_values=decoder_outputs.past_key_values,
+            decoder_hidden_states=decoder_outputs.hidden_states,
+            decoder_attentions=decoder_outputs.attentions,
+            cross_attentions=decoder_outputs.cross_attentions,
+            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
+            encoder_hidden_states=encoder_outputs.hidden_states,
+            encoder_attentions=encoder_outputs.attentions,
+        )

config.json

@@ -0,0 +1,49 @@
+{
+  "architectures": [
+    "EchoChatForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_echochat.EchoChatConfig",
+    "AutoModel": "modeling_echochat.EchoChatModel",
+    "AutoModelForCausalLM": "modeling_echochat.EchoChatForCausalLM"
+  },
+  "bos_token_id": 151643,
+  "eos_token_ids": [
+    151644,
+    151645
+  ],
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "kimia_adaptor_input_dim": 5120,
+  "kimia_audio_output_vocab": 16896,
+  "kimia_media_begin": 151661,
+  "kimia_media_end": 151663,
+  "kimia_mimo_audiodelaytokens": 6,
+  "kimia_mimo_layers": 6,
+  "kimia_mimo_transformer_from_layer_index": 21,
+  "kimia_text_output_vocab": 152064,
+  "kimia_token_offset": 152064,
+  "max_position_embeddings": 8192,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_audio_special_tokens": 512,
+  "num_base_tokens": 151643,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "pad_token_id": 152063,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "use_whisper_feature": true,
+  "vocab_size": 168448
+}

configuration_echochat.py

@@ -0,0 +1,67 @@
+# Modified for EchoChat distribution: renamed public entry points; computation unchanged.
+from transformers.models.qwen2.configuration_qwen2 import Qwen2Config
+
+
+class EchoChatConfig(Qwen2Config):
+    def __init__(
+        self,
+        vocab_size=163840,
+        hidden_size=4096,
+        intermediate_size=11008,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=None,
+        hidden_act="silu",
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        rope_theta=10000.0,
+        rope_scaling=None,
+        tie_word_embeddings=False,
+        kimia_mimo_layers: int = 6,
+        kimia_mimo_audiodelaytokens: int = 5,
+        kimia_mimo_transformer_from_layer_index: int = 21,
+        kimia_audio_output_vocab: int = 16896,
+        kimia_text_output_vocab: int = 152064,
+        num_audio_special_tokens: int = 512,
+        num_base_tokens: int = 151643,
+        kimia_token_offset: int = 152064,
+        use_whisper_feature: bool = True,
+        kimia_adaptor_input_dim: int = 5120,
+        kimia_media_begin: int = 151661,
+        kimia_media_end: int = 151663,
+        **kwargs,
+    ):
+        super().__init__(
+            vocab_size=vocab_size,
+            hidden_size=hidden_size,
+            intermediate_size=intermediate_size,
+            num_hidden_layers=num_hidden_layers,
+            num_attention_heads=num_attention_heads,
+            num_key_value_heads=num_key_value_heads,
+            hidden_act=hidden_act,
+            initializer_range=initializer_range,
+            rms_norm_eps=rms_norm_eps,
+            use_cache=use_cache,
+            tie_word_embeddings=tie_word_embeddings,
+            rope_theta=rope_theta,
+            rope_scaling=rope_scaling,
+            **kwargs,
+        )
+
+        self.kimia_mimo_layers = kimia_mimo_layers
+        self.kimia_mimo_audiodelaytokens = kimia_mimo_audiodelaytokens
+        # vocab
+        self.kimia_mimo_transformer_from_layer_index = (
+            kimia_mimo_transformer_from_layer_index
+        )
+        self.kimia_audio_output_vocab = kimia_audio_output_vocab
+        self.kimia_text_output_vocab = kimia_text_output_vocab
+        self.num_audio_special_tokens = num_audio_special_tokens
+        self.num_base_tokens = num_base_tokens
+        self.kimia_token_offset = kimia_token_offset
+        self.use_whisper_feature = use_whisper_feature
+        self.kimia_adaptor_input_dim = kimia_adaptor_input_dim
+        # special tokens
+        self.kimia_media_begin = kimia_media_begin
+        self.kimia_media_end = kimia_media_end

licenses/Apache-2.0.txt

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

licenses/MIT.txt

@@ -0,0 +1,20 @@
+Copyright 2025 The Moonshot AI Team, Qwen Team, and HuggingFace Inc. team. All rights reserved.
+
+MIT License:
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

model-00001-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39f83ac6ca7cdf41590d96a47df721efdf4bd0f3436efb7fa215e23c3ce65065
+size 1868073328

model-00002-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca32847eb341cecbdaf9cd6e6615b0dab923e67f8abc436902903d9d59aed2cb
+size 1864467800

model-00003-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf780ca93e0f064657600e562edd9e4057a3f31c654691fd76c32e086dfa5dae
+size 1864467800

model-00004-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63988fd3109491d6af07b566a5af899bb44e051ae80fb51958fdf9dc0c0c2950
+size 1864467824

model-00005-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0417331bcd9804e51432f27434711e249c80ba830c1cbbafa49987820b3de312
+size 1864467848

model-00006-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:041e61c8399bb208d726e61e98ee2af13ad05e312ca583e71c93f118fd4215ac
+size 1864467848

model-00007-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e631fc93e13211d184a8ab2fdaab9b8355aae08b5154b6957b22348fbf5faf74
+size 1864467848

model-00008-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88d8f91e3fdee77496bd820ca1aeb51898b0e539deef038f72b1d679ac324eb3
+size 1864475184

model-00009-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a50385172a9aaca4b75aeb7fc73208a498cd5775ccb04e52a21b03997c57bc4d
+size 1864468040

model-00010-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec5e761bde74f16ee8fe8bbc913f50a014e6765335a76815e849cf0c5846b48b
+size 1541458240

model-00011-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95b6f8f4f0366eaafede3c5e8cbc03218b8fd789cb9d4c7f7f41ca6402e4bb2b
+size 1207435400

model.safetensors.index.json

@@ -0,0 +1,426 @@
+{
+  "metadata": {
+    "total_size": 19532668928
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00010-of-00011.safetensors",
+    "mimo_output.weight": "model-00011-of-00011.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00011.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00011.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00011.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00011.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00011.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00011.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00011.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00011.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00011.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00011.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00011.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00011.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00011.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00002-of-00011.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00011.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00002-of-00011.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00011.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00011.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00011.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00011.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00011.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00011.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00011.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00004-of-00011.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00004-of-00011.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00004-of-00011.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00004-of-00011.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00004-of-00011.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00004-of-00011.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00004-of-00011.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00004-of-00011.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00004-of-00011.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00004-of-00011.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00004-of-00011.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00004-of-00011.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00004-of-00011.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00004-of-00011.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00004-of-00011.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00005-of-00011.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00005-of-00011.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00004-of-00011.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00004-of-00011.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00004-of-00011.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00005-of-00011.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00005-of-00011.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00005-of-00011.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00005-of-00011.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00005-of-00011.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00005-of-00011.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00005-of-00011.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00005-of-00011.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00005-of-00011.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00005-of-00011.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00005-of-00011.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00005-of-00011.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00005-of-00011.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00005-of-00011.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00005-of-00011.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00006-of-00011.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00006-of-00011.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00005-of-00011.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00005-of-00011.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00005-of-00011.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00005-of-00011.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00006-of-00011.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00006-of-00011.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00006-of-00011.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00006-of-00011.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00006-of-00011.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00006-of-00011.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00006-of-00011.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00006-of-00011.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00006-of-00011.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00006-of-00011.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00002-of-00011.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00002-of-00011.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00002-of-00011.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00002-of-00011.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00002-of-00011.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00006-of-00011.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00006-of-00011.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00006-of-00011.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00006-of-00011.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00006-of-00011.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00007-of-00011.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00007-of-00011.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00006-of-00011.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00006-of-00011.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00006-of-00011.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00006-of-00011.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00007-of-00011.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00007-of-00011.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00007-of-00011.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00007-of-00011.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00007-of-00011.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00007-of-00011.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00007-of-00011.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00007-of-00011.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00007-of-00011.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00007-of-00011.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00007-of-00011.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00007-of-00011.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00007-of-00011.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00007-of-00011.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00007-of-00011.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00008-of-00011.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00008-of-00011.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00008-of-00011.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00008-of-00011.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00007-of-00011.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00007-of-00011.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00007-of-00011.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00007-of-00011.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00008-of-00011.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00008-of-00011.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00008-of-00011.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00008-of-00011.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00008-of-00011.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00008-of-00011.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00008-of-00011.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00008-of-00011.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00008-of-00011.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00008-of-00011.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00008-of-00011.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00008-of-00011.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00008-of-00011.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00008-of-00011.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00008-of-00011.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00008-of-00011.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00008-of-00011.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00008-of-00011.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00008-of-00011.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00008-of-00011.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00008-of-00011.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00008-of-00011.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00008-of-00011.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00008-of-00011.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00011.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00011.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00002-of-00011.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00002-of-00011.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00002-of-00011.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00011.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00011.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00002-of-00011.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00002-of-00011.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00002-of-00011.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00003-of-00011.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00003-of-00011.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00002-of-00011.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00002-of-00011.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00002-of-00011.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00011.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00003-of-00011.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00003-of-00011.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00003-of-00011.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00003-of-00011.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00003-of-00011.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00003-of-00011.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00003-of-00011.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00003-of-00011.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00003-of-00011.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00003-of-00011.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00011.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00011.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00003-of-00011.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00003-of-00011.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00003-of-00011.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00004-of-00011.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00004-of-00011.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00004-of-00011.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00003-of-00011.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00003-of-00011.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00011.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00003-of-00011.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00011.safetensors",
+    "model.mimo_layers.0.input_layernorm.weight": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.0.mlp.down_proj.weight": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.0.mlp.gate_proj.weight": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.0.mlp.up_proj.weight": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.0.post_attention_layernorm.weight": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.0.self_attn.k_proj.bias": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.0.self_attn.k_proj.weight": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.0.self_attn.o_proj.weight": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.0.self_attn.q_proj.bias": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.0.self_attn.q_proj.weight": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.0.self_attn.v_proj.bias": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.0.self_attn.v_proj.weight": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.1.input_layernorm.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.1.mlp.down_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.1.mlp.gate_proj.weight": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.1.mlp.up_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.1.post_attention_layernorm.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.1.self_attn.k_proj.bias": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.1.self_attn.k_proj.weight": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.1.self_attn.o_proj.weight": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.1.self_attn.q_proj.bias": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.1.self_attn.q_proj.weight": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.1.self_attn.v_proj.bias": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.1.self_attn.v_proj.weight": "model-00008-of-00011.safetensors",
+    "model.mimo_layers.2.input_layernorm.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.2.mlp.down_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.2.mlp.gate_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.2.mlp.up_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.2.post_attention_layernorm.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.2.self_attn.k_proj.bias": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.2.self_attn.k_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.2.self_attn.o_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.2.self_attn.q_proj.bias": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.2.self_attn.q_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.2.self_attn.v_proj.bias": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.2.self_attn.v_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.3.input_layernorm.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.3.mlp.down_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.3.mlp.gate_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.3.mlp.up_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.3.post_attention_layernorm.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.3.self_attn.k_proj.bias": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.3.self_attn.k_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.3.self_attn.o_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.3.self_attn.q_proj.bias": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.3.self_attn.q_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.3.self_attn.v_proj.bias": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.3.self_attn.v_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.4.input_layernorm.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.4.mlp.down_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.4.mlp.gate_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.4.mlp.up_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.4.post_attention_layernorm.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.4.self_attn.k_proj.bias": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.4.self_attn.k_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.4.self_attn.o_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.4.self_attn.q_proj.bias": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.4.self_attn.q_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.4.self_attn.v_proj.bias": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.4.self_attn.v_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.5.input_layernorm.weight": "model-00010-of-00011.safetensors",
+    "model.mimo_layers.5.mlp.down_proj.weight": "model-00010-of-00011.safetensors",
+    "model.mimo_layers.5.mlp.gate_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.5.mlp.up_proj.weight": "model-00010-of-00011.safetensors",
+    "model.mimo_layers.5.post_attention_layernorm.weight": "model-00010-of-00011.safetensors",
+    "model.mimo_layers.5.self_attn.k_proj.bias": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.5.self_attn.k_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.5.self_attn.o_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.5.self_attn.q_proj.bias": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.5.self_attn.q_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.5.self_attn.v_proj.bias": "model-00009-of-00011.safetensors",
+    "model.mimo_layers.5.self_attn.v_proj.weight": "model-00009-of-00011.safetensors",
+    "model.mimo_norm.weight": "model-00010-of-00011.safetensors",
+    "model.norm.weight": "model-00008-of-00011.safetensors",
+    "model.vq_adaptor.layers.0.bias": "model-00010-of-00011.safetensors",
+    "model.vq_adaptor.layers.0.weight": "model-00010-of-00011.safetensors",
+    "model.vq_adaptor.layers.3.bias": "model-00010-of-00011.safetensors",
+    "model.vq_adaptor.layers.3.weight": "model-00010-of-00011.safetensors",
+    "model.vq_adaptor.layers.4.bias": "model-00010-of-00011.safetensors",
+    "model.vq_adaptor.layers.4.weight": "model-00010-of-00011.safetensors"
+  }
+}

modeling_echochat.py

@@ -0,0 +1,955 @@
+# Modified for EchoChat distribution: renamed entry points and deferred the FlashAttention import error to execution; GPU computation unchanged.
+# coding=utf-8
+# Copyright 2025 The Moonshot AI Team, Qwen Team, and HuggingFace Inc. team. All rights reserved.
+#
+# The code is based on Qwen2.5-7B, but modified for KimiAudio.
+#
+# Licensing Information:
+# - Code derived from Qwen2.5-7B is licensed under the Apache License, Version 2.0.
+# - Other parts of the code are licensed under the MIT License.
+#
+# Apache License, Version 2.0:
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# MIT License:
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+"""PyTorch KimiAudio model."""
+
+import math
+from typing import List, Optional, Tuple, Union
+import torch
+import torch.utils.checkpoint
+from torch import nn
+
+import transformers
+from packaging import version
+
+assert version.parse(transformers.__version__) >= version.parse("4.34.1")
+
+from transformers.modeling_outputs import (
+    BaseModelOutputWithPast,
+    CausalLMOutputWithPast,
+)
+from transformers.utils import (
+    logging,
+)
+from .configuration_echochat import EchoChatConfig
+import torch.nn.functional as F
+from transformers.models.qwen2.modeling_qwen2 import (
+    Qwen2RMSNorm,
+    Qwen2MLP,
+    Qwen2PreTrainedModel,
+)
+from transformers.models.qwen2.modeling_qwen2 import apply_rotary_pos_emb
+
+if version.parse(transformers.__version__) >= version.parse("4.35.0"):
+    from transformers.utils import is_flash_attn_2_available as is_flash_attn_available
+else:
+    from transformers.utils import is_flash_attn_available
+
+if is_flash_attn_available():
+    from flash_attn import flash_attn_func, flash_attn_varlen_func
+    from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input  # noqa
+else:
+    # Permit CPU weight inspection; execution still requires FlashAttention.
+    def _flash_attention_required(*args, **kwargs):
+        raise RuntimeError("CUDA FlashAttention is required for model execution")
+    flash_attn_func = flash_attn_varlen_func = _flash_attention_required
+    index_first_axis = pad_input = unpad_input = _flash_attention_required
+
+
+logger = logging.get_logger(__name__)
+
+
+def _get_unpad_data(padding_mask):
+    seqlens_in_batch = padding_mask.sum(dim=-1, dtype=torch.int32)
+    indices = torch.nonzero(padding_mask.flatten(), as_tuple=False).flatten()
+    max_seqlen_in_batch = seqlens_in_batch.max().item()
+    cu_seqlens = F.pad(
+        torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0)
+    )
+    return (
+        indices,
+        cu_seqlens,
+        max_seqlen_in_batch,
+    )
+
+
+def _upad_input(query_layer, key_layer, value_layer, padding_mask, query_length):
+    indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(padding_mask)
+    batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
+    num_heads = query_layer.shape[2]
+
+    key_layer = index_first_axis(
+        key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim),
+        indices_k,
+    )
+    value_layer = index_first_axis(
+        value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim),
+        indices_k,
+    )
+    if query_length == kv_seq_len:
+        query_layer = index_first_axis(
+            query_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k
+        )
+        cu_seqlens_q = cu_seqlens_k
+        max_seqlen_in_batch_q = max_seqlen_in_batch_k
+        indices_q = indices_k
+    elif query_length == 1:
+        max_seqlen_in_batch_q = 1
+        cu_seqlens_q = torch.arange(
+            batch_size + 1, dtype=torch.int32, device=query_layer.device
+        )  # There is a memcpy here, that is very bad.
+        indices_q = cu_seqlens_q[:-1]
+        query_layer = query_layer.squeeze(1)
+    else:
+        # The -q_len: slice assumes left padding.
+        padding_mask = padding_mask[:, -query_length:]
+        query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(
+            query_layer, padding_mask
+        )
+
+    return (
+        query_layer,
+        key_layer,
+        value_layer,
+        indices_q,
+        (cu_seqlens_q, cu_seqlens_k),
+        (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
+    )
+
+
+# Copied from transformers.models.bart.modeling_bart._make_causal_mask
+def _make_causal_mask(
+    input_ids_shape: torch.Size,
+    dtype: torch.dtype,
+    device: torch.device,
+    past_key_values_length: int = 0,
+):
+    """
+    Make causal mask used for bi-directional self-attention.
+    """
+    bsz, tgt_len = input_ids_shape
+    mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device)
+    mask_cond = torch.arange(mask.size(-1), device=device)
+    mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
+    mask = mask.to(dtype)
+
+    if past_key_values_length > 0:
+        mask = torch.cat(
+            [
+                torch.zeros(
+                    tgt_len, past_key_values_length, dtype=dtype, device=device
+                ),
+                mask,
+            ],
+            dim=-1,
+        )
+    return mask[None, None, :, :].expand(
+        bsz, 1, tgt_len, tgt_len + past_key_values_length
+    )
+
+
+# Copied from transformers.models.bart.modeling_bart._expand_mask
+def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):
+    """
+    Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
+    """
+    bsz, src_len = mask.size()
+    tgt_len = tgt_len if tgt_len is not None else src_len
+
+    expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)
+
+    inverted_mask = 1.0 - expanded_mask
+
+    return inverted_mask.masked_fill(
+        inverted_mask.to(torch.bool), torch.finfo(dtype).min
+    )
+
+
+class RotaryEmbedding(nn.Module):
+    def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
+        super().__init__()
+
+        self.dim = dim
+        self.max_position_embeddings = max_position_embeddings
+        self.base = base
+        inv_freq = 1.0 / (
+            self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim)
+        )
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+        # Build here to make `torch.jit.trace` work.
+        self._set_cos_sin_cache(
+            seq_len=max_position_embeddings,
+            device=self.inv_freq.device,
+            dtype=torch.get_default_dtype(),
+        )
+
+    def _set_cos_sin_cache(self, seq_len, device, dtype):
+        self.max_seq_len_cached = seq_len
+        t = torch.arange(
+            self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype
+        )
+
+        freqs = torch.einsum("i,j->ij", t, self.inv_freq)
+        # Different from paper, but it uses a different permutation in order to obtain the same calculation
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
+        self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
+
+    def forward(self, x, seq_len=None):
+        # x: [bs, num_attention_heads, seq_len, head_size]
+        if seq_len > self.max_seq_len_cached:
+            self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype)
+
+        return (
+            self.cos_cached[:seq_len].to(dtype=x.dtype),
+            self.sin_cached[:seq_len].to(dtype=x.dtype),
+        )
+
+
+class MoonshotAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    def __init__(self, config: EchoChatConfig, layer_idx: int = 0, total_layers: int = 28):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        self.total_layers = total_layers
+        self.hidden_size = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.hidden_size // self.num_heads
+        self.num_key_value_heads = config.num_key_value_heads
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+        self.max_position_embeddings = config.max_position_embeddings
+        self.rope_theta = config.rope_theta
+        if (self.head_dim * self.num_heads) != self.hidden_size:
+            raise ValueError(
+                f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
+                f" and `num_heads`: {self.num_heads})."
+            )
+        self.q_proj = nn.Linear(
+            self.hidden_size, self.num_heads * self.head_dim, bias=True
+        )
+        self.k_proj = nn.Linear(
+            self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True
+        )
+        self.v_proj = nn.Linear(
+            self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True
+        )
+        self.o_proj = nn.Linear(
+            self.num_heads * self.head_dim, self.hidden_size, bias=False
+        )
+
+        self._init_rope()
+
+    def _init_rope(self):
+
+        self.rotary_emb = RotaryEmbedding(
+            self.head_dim,
+            max_position_embeddings=self.max_position_embeddings,
+            base=self.rope_theta,
+        )
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        padding_mask: Optional[torch.LongTensor] = None,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        # AAA: only the last AAA_EAGER_LAYERS layers use the eager attention path when
+        # output_attentions=True. All earlier layers silently fall back to flash attention.
+        # layer_idx=-1 (mimo layers) never enters the eager path.
+        AAA_EAGER_LAYERS = 2  # last 2 main transformer layers
+        use_eager = output_attentions and (self.layer_idx >= 0) and (self.layer_idx >= self.total_layers - AAA_EAGER_LAYERS)
+
+        bsz, q_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        # Shape: [bsz, num_heads, seq_len, head_dim]
+        query_states = query_states.view(
+            bsz, q_len, self.num_heads, self.head_dim
+        ).transpose(1, 2)
+        key_states = key_states.view(
+            bsz, q_len, self.num_key_value_heads, self.head_dim
+        ).transpose(1, 2)
+        value_states = value_states.view(
+            bsz, q_len, self.num_key_value_heads, self.head_dim
+        ).transpose(1, 2)
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            kv_seq_len += past_key_value[0].shape[-2]
+
+        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        cos = cos[position_ids]
+        sin = sin[position_ids]
+        query_states, key_states = apply_rotary_pos_emb(
+            query_states, key_states, cos, sin, position_ids
+        )
+
+        if past_key_value is not None:
+            key_states = torch.cat([past_key_value[0], key_states], dim=2)
+            value_states = torch.cat([past_key_value[1], value_states], dim=2)
+
+        past_key_value = (key_states, value_states) if use_cache else None
+
+        # --- Eager attention path (AAA: last N layers, output_attentions=True) ---
+        if use_eager:
+            # Expand KV heads for GQA: [bsz, num_heads, kv_len, head_dim]
+            if self.num_key_value_groups > 1:
+                key_states_e = key_states.repeat_interleave(self.num_key_value_groups, dim=1)
+                value_states_e = value_states.repeat_interleave(self.num_key_value_groups, dim=1)
+            else:
+                key_states_e = key_states
+                value_states_e = value_states
+
+            scale = 1.0 / math.sqrt(self.head_dim)
+            # [bsz, num_heads, q_len, kv_len]
+            attn_weights = torch.matmul(query_states, key_states_e.transpose(-2, -1)) * scale
+
+            # Build causal mask: positions i cannot attend to j > i
+            if q_len > 1:
+                causal = torch.full(
+                    (q_len, kv_seq_len), fill_value=float('-inf'),
+                    device=attn_weights.device, dtype=attn_weights.dtype,
+                )
+                # Upper-triangular (excluding the diagonal shifted by kv_seq_len - q_len)
+                causal = torch.triu(causal, diagonal=1 + kv_seq_len - q_len)
+                attn_weights = attn_weights + causal.unsqueeze(0).unsqueeze(0)
+
+            # Softmax in fp32 for numerical stability; keep fp32 copy for AAA loss gradients.
+            attn_weights_f32 = torch.nn.functional.softmax(attn_weights.float(), dim=-1)
+            # Cast to model dtype for the value matmul (keeps compute in bf16/fp16)
+            attn_weights_compute = attn_weights_f32.to(query_states.dtype)
+
+            # [bsz, num_heads, q_len, head_dim]
+            attn_output = torch.matmul(attn_weights_compute, value_states_e)
+            # [bsz, q_len, num_heads, head_dim] → [bsz, q_len, hidden_size]
+            attn_output = attn_output.transpose(1, 2).reshape(bsz, q_len, self.hidden_size).contiguous()
+            attn_output = self.o_proj(attn_output)
+            # Return fp32 attention weights so AAA loss can compute precise gradients
+            return attn_output, attn_weights_f32, past_key_value
+
+        # --- Flash attention path (all other layers, or when output_attentions=False) ---
+        # Transpose to [bsz, seq_len, heads, head_dim] for flash attention
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+
+        dropout_rate = 0.0
+
+        # Cast back to fp16 if upcast happened (PEFT layer-norm fp32 upcasting)
+        input_dtype = query_states.dtype
+        if input_dtype == torch.float32:
+            logger.warning_once(
+                "The input hidden states seems to be silently casted in float32, this might be related to"
+                " the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
+                " float16."
+            )
+            query_states = query_states.to(torch.float16)
+            key_states = key_states.to(torch.float16)
+            value_states = value_states.to(torch.float16)
+
+        attn_output = self._flash_attention_forward(
+            query_states,
+            key_states,
+            value_states,
+            padding_mask,
+            q_len,
+            dropout=dropout_rate,
+        )
+
+        if input_dtype == torch.float32:
+            attn_output = attn_output.to(torch.float32)
+
+        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()
+        attn_output = self.o_proj(attn_output)
+
+        # Flash attention does not expose attention weights
+        return attn_output, None, past_key_value
+
+    def _flash_attention_forward(
+        self,
+        query_states,
+        key_states,
+        value_states,
+        padding_mask,
+        query_length,
+        dropout=0.0,
+        softmax_scale=None,
+    ):
+        """
+        Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
+        first unpad the input, then computes the attention scores and pad the final attention scores.
+
+        Args:
+            query_states (`torch.Tensor`):
+                Input query states to be passed to Flash Attention API
+            key_states (`torch.Tensor`):
+                Input key states to be passed to Flash Attention API
+            value_states (`torch.Tensor`):
+                Input value states to be passed to Flash Attention API
+            padding_mask (`torch.Tensor`):
+                The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
+                position of padding tokens and 1 for the position of non-padding tokens.
+            dropout (`int`, *optional*):
+                Attention dropout
+            softmax_scale (`float`, *optional*):
+                The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
+        """
+        # Contains at least one padding token in the sequence
+        if padding_mask is not None:
+            batch_size = query_states.shape[0]
+            (
+                query_states,
+                key_states,
+                value_states,
+                indices_q,
+                cu_seq_lens,
+                max_seq_lens,
+            ) = _upad_input(
+                query_states, key_states, value_states, padding_mask, query_length
+            )
+
+            cu_seqlens_q, cu_seqlens_k = cu_seq_lens
+            max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
+
+            attn_output_unpad = flash_attn_varlen_func(
+                query_states,
+                key_states,
+                value_states,
+                cu_seqlens_q=cu_seqlens_q,
+                cu_seqlens_k=cu_seqlens_k,
+                max_seqlen_q=max_seqlen_in_batch_q,
+                max_seqlen_k=max_seqlen_in_batch_k,
+                dropout_p=dropout,
+                softmax_scale=softmax_scale,
+                causal=True,
+            )
+
+            attn_output = pad_input(
+                attn_output_unpad, indices_q, batch_size, query_length
+            )
+        else:
+            attn_output = flash_attn_func(
+                query_states,
+                key_states,
+                value_states,
+                dropout,
+                softmax_scale=softmax_scale,
+                causal=True,
+            )
+
+        return attn_output
+
+
+class MoonshotDecoderLayer(nn.Module):
+    def __init__(self, config: EchoChatConfig, layer_idx: int = 0, total_layers: int = 28):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+        self.config = config
+
+        logger.warning_once("using normal flash attention")
+        self.self_attn = MoonshotAttention(config=config, layer_idx=layer_idx, total_layers=total_layers)
+
+        self.mlp = Qwen2MLP(config)
+        self.input_layernorm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.post_attention_layernorm = Qwen2RMSNorm(
+            config.hidden_size, eps=config.rms_norm_eps
+        )
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: Optional[bool] = False,
+        use_cache: Optional[bool] = False,
+        padding_mask: Optional[torch.LongTensor] = None,
+    ) -> Tuple[
+        torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]
+    ]:
+        """
+        Args:
+            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
+                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            use_cache (`bool`, *optional*):
+                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+                (see `past_key_values`).
+            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
+        """
+
+        residual = hidden_states
+
+        hidden_states = self.input_layernorm(hidden_states)
+
+        # Self Attention
+        hidden_states, self_attn_weights, present_key_value = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_value=past_key_value,
+            output_attentions=output_attentions,
+            use_cache=use_cache,
+            padding_mask=padding_mask,
+        )
+        hidden_states = residual + hidden_states
+
+        # Fully Connected
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (self_attn_weights,)
+
+        if use_cache:
+            outputs += (present_key_value,)
+
+        return outputs
+
+
+class VQAdaptor(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.layers = nn.Sequential(
+            nn.Linear(config.kimia_adaptor_input_dim, config.hidden_size, bias=True),
+            nn.SiLU(),
+            nn.Dropout(0.0),
+            nn.Linear(config.hidden_size, config.hidden_size, bias=True),
+            nn.LayerNorm(config.hidden_size, eps=config.rms_norm_eps, bias=True),
+        )
+
+    def forward(self, x):
+        return self.layers(x)
+
+
+class EchoChatModel(Qwen2PreTrainedModel):
+    """
+    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`QwenDecoderLayer`]
+
+    Args:
+        config: EchoChatConfig
+    """
+
+    config_class = EchoChatConfig
+
+    def __init__(self, config: EchoChatConfig):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+        self.kimia_mimo_transformer_from_layer_index = (
+            config.kimia_mimo_transformer_from_layer_index
+        )
+
+        self.embed_tokens = nn.Embedding(
+            config.vocab_size, config.hidden_size, self.padding_idx
+        )
+        _total = config.num_hidden_layers
+        self.layers = nn.ModuleList(
+            [MoonshotDecoderLayer(config, layer_idx=i, total_layers=_total)
+             for i in range(_total)]
+        )
+        self.norm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+        # extra 1B audio transformers (mimo layers are not used for AAA; use flash always)
+        self.mimo_layers = nn.ModuleList(
+            [MoonshotDecoderLayer(config, layer_idx=-1, total_layers=_total)
+             for _ in range(config.kimia_mimo_layers)]
+        )
+        self.mimo_norm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.use_whisper_feature = config.use_whisper_feature
+        if self.use_whisper_feature:
+            self.vq_adaptor = VQAdaptor(config)
+        self.kimia_media_begin = config.kimia_media_begin
+        self.kimia_media_end = config.kimia_media_end
+
+        self.gradient_checkpointing = False
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.embed_tokens = value
+
+    # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask
+    def _prepare_decoder_attention_mask(
+        self, attention_mask, input_shape, inputs_embeds, past_key_values_length
+    ):
+        # create causal mask
+        # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
+        combined_attention_mask = None
+        if input_shape[-1] > 1:
+            combined_attention_mask = _make_causal_mask(
+                input_shape,
+                inputs_embeds.dtype,
+                device=inputs_embeds.device,
+                past_key_values_length=past_key_values_length,
+            )
+
+        if attention_mask is not None:
+            # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
+            expanded_attn_mask = _expand_mask(
+                attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]
+            ).to(inputs_embeds.device)
+            combined_attention_mask = (
+                expanded_attn_mask
+                if combined_attention_mask is None
+                else expanded_attn_mask + combined_attention_mask
+            )
+
+        return combined_attention_mask
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        text_input_ids: torch.LongTensor = None,
+        whisper_input_feature: Optional[torch.FloatTensor] = None,
+        is_continuous_mask: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutputWithPast]:
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
+        output_hidden_states = (
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+
+        # retrieve input_ids and inputs_embeds
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError(
+                "You cannot specify both input_ids and inputs_embeds at the same time"
+            )
+        elif input_ids is not None:
+            batch_size, seq_length = input_ids.shape
+        elif inputs_embeds is not None:
+            batch_size, seq_length, _ = inputs_embeds.shape
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+        seq_length_with_past = seq_length
+        past_key_values_length = 0
+
+        if past_key_values is not None:
+            past_key_values_length = past_key_values[0][0].shape[2]
+            seq_length_with_past = seq_length_with_past + past_key_values_length
+        if position_ids is None:
+            device = input_ids.device if input_ids is not None else inputs_embeds.device
+            position_ids = torch.arange(
+                past_key_values_length,
+                seq_length + past_key_values_length,
+                dtype=torch.long,
+                device=device,
+            )
+            position_ids = position_ids.unsqueeze(0)
+
+        if inputs_embeds is None:
+            # shape: batch, seq_len, hidden_size
+            input_ids = input_ids.to(torch.cuda.current_device())
+            text_input_ids = text_input_ids.to(torch.cuda.current_device())
+            audio_emb = self.embed_tokens(input_ids)
+            if self.use_whisper_feature and whisper_input_feature is not None:
+                if not isinstance(whisper_input_feature, list):
+                    whisper_input_feature = whisper_input_feature.squeeze(0)
+                    whisper_input_feature = [whisper_input_feature]
+
+                media_start_idx = (input_ids == self.kimia_media_begin).nonzero()
+                media_end_idx = (input_ids == self.kimia_media_end).nonzero()
+                # shape: batch, seq_len, hidden_size
+                whisper_input_dim = whisper_input_feature[0].shape[-1]
+                whisper_dtype = whisper_input_feature[0].dtype
+                expanded_whisper = (
+                    torch.zeros(audio_emb.shape[1], whisper_input_dim)
+                    .to(torch.cuda.current_device())
+                    .to(whisper_dtype)
+                )
+                assert (media_end_idx - media_start_idx).sum() - media_start_idx.shape[0] == is_continuous_mask.sum()
+                for seg_idx, ((batch_idx, start_idx), (_, end_idx)) in enumerate(zip(
+                    media_start_idx, media_end_idx
+                )):
+
+                    feat_len = end_idx - (start_idx + 1)
+                    whisper_input_feature_i = whisper_input_feature[seg_idx].squeeze(0)
+                    expanded_whisper[start_idx + 1 : end_idx, :] = (
+                        whisper_input_feature_i[:feat_len, :]
+                    )
+
+                expanded_whisper = expanded_whisper.unsqueeze(0)
+                whisper_emb = self.vq_adaptor(
+                    expanded_whisper.transpose(0, 1)
+                ).transpose(0, 1)
+                is_continuous_mask = is_continuous_mask.to(torch.cuda.current_device())
+                whisper_emb = whisper_emb.to(torch.cuda.current_device())
+                whisper_emb = whisper_emb * is_continuous_mask[:, :, None]
+
+                encoder_input_addwith_discrete_token = (
+                    audio_emb + whisper_emb
+                ) * torch.sqrt(
+                    torch.tensor(
+                        2.0, dtype=whisper_emb.dtype, device=torch.cuda.current_device()
+                    )
+                )
+                audio_emb = (
+                    audio_emb * (~is_continuous_mask[:, :, None])
+                    + encoder_input_addwith_discrete_token
+                    * is_continuous_mask[:, :, None]
+                )
+            if text_input_ids is not None and text_input_ids.sum() != 0:
+                inputs_embeds = audio_emb + self.embed_tokens(text_input_ids)
+            else:
+                inputs_embeds = audio_emb
+        # embed positions
+        # TODO kill attention_mask for prefill
+        padding_mask = attention_mask
+
+        hidden_states = inputs_embeds
+
+        # decoder layers
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attns = () if output_attentions else None
+        next_decoder_cache = () if use_cache else None
+        for idx, decoder_layer in enumerate(self.layers):
+            if output_hidden_states:
+                all_hidden_states += (hidden_states,)
+
+            past_key_value = (
+                past_key_values[idx] if past_key_values is not None else None
+            )
+            layer_outputs = decoder_layer(
+                hidden_states,
+                attention_mask=attention_mask,
+                position_ids=position_ids,
+                past_key_value=past_key_value,
+                output_attentions=output_attentions,
+                use_cache=use_cache,
+                padding_mask=padding_mask,
+            )
+
+            hidden_states = layer_outputs[0]
+            if idx == self.kimia_mimo_transformer_from_layer_index:
+                mimo_hidden_states = hidden_states.clone()
+
+            if use_cache:
+                next_decoder_cache += (layer_outputs[2 if output_attentions else 1],)
+
+            if output_attentions:
+                all_self_attns += (layer_outputs[1],)
+
+        hidden_states = self.norm(hidden_states)
+        if output_hidden_states:
+            all_hidden_states += (hidden_states,)
+
+        # apply audio transformer layers
+        for idx, decoder_layer in enumerate(self.mimo_layers):
+            if output_hidden_states:
+                all_hidden_states += (mimo_hidden_states,)
+
+            past_key_value = (
+                past_key_values[idx + len(self.layers)]
+                if past_key_values is not None
+                else None
+            )
+            layer_outputs = decoder_layer(
+                mimo_hidden_states,
+                attention_mask=attention_mask,
+                position_ids=position_ids,
+                past_key_value=past_key_value,
+                output_attentions=output_attentions,
+                use_cache=use_cache,
+                padding_mask=padding_mask,
+            )
+
+            mimo_hidden_states = layer_outputs[0]
+
+            if use_cache:
+                next_decoder_cache += (layer_outputs[2 if output_attentions else 1],)
+
+        mimo_hidden_states = self.mimo_norm(mimo_hidden_states)
+
+        # add hidden states from the last decoder layer
+        if output_hidden_states:
+            all_hidden_states += (mimo_hidden_states,)
+
+        next_cache = next_decoder_cache if use_cache else None
+        if not return_dict:
+            return tuple(
+                v
+                for v in [
+                    hidden_states,
+                    mimo_hidden_states,
+                    next_cache,
+                    all_hidden_states,
+                    all_hidden_states,
+                    all_self_attns,
+                ]
+                if v is not None
+            )
+        return BaseModelOutputWithPast(
+            last_hidden_state=(hidden_states, mimo_hidden_states),
+            past_key_values=next_cache,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attns,
+        )
+
+
+class EchoChatForCausalLM(Qwen2PreTrainedModel):
+    _tied_weights_keys = ["lm_head.weight", "mimo_output.weight"]
+    config_class = EchoChatConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = EchoChatModel(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        self.mimo_output = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.model.embed_tokens = value
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def set_decoder(self, decoder):
+        self.model = decoder
+
+    def get_decoder(self):
+        return self.model
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        text_input_ids: torch.LongTensor = None,
+        whisper_input_feature: Optional[torch.FloatTensor] = None,
+        is_continuous_mask: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        generation_mode: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
+        output_hidden_states = (
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
+        )
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+
+        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+        outputs = self.model(
+            input_ids=input_ids,
+            text_input_ids=text_input_ids,
+            whisper_input_feature=whisper_input_feature,
+            is_continuous_mask=is_continuous_mask,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        if return_dict:
+            hidden_states, mimo_hidden_states = (
+                outputs.last_hidden_state[0],
+                outputs.last_hidden_state[1],
+            )
+        else:
+            hidden_states, mimo_hidden_states = outputs[0], outputs[1]
+
+        text_logits = self.lm_head(hidden_states)
+        audio_logits = self.mimo_output(mimo_hidden_states)
+
+        if not return_dict:
+            output = (audio_logits, text_logits) + outputs[2:]
+            return output
+        return CausalLMOutputWithPast(
+            loss=None,
+            logits=(audio_logits, text_logits),
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )

special_tokens_map.json

@@ -0,0 +1,425 @@
+{
+  "additional_special_tokens": [
+    "<|im_msg_end|>",
+    "<|im_user_msg_start|>",
+    "<|im_assistant_msg_start|>",
+    "<|reserved_token_0|>",
+    "<|reserved_token_1|>",
+    "<|reserved_token_2|>",
+    "<|reserved_token_3|>",
+    "[EOT]",
+    "<|reserved_token_4|>",
+    "<|reserved_token_5|>",
+    "<|reserved_token_6|>",
+    "<|reserved_token_7|>",
+    "<|reserved_token_8|>",
+    "<|reserved_token_9|>",
+    "<|reserved_token_10|>",
+    "<|reserved_token_11|>",
+    "<|im_media_begin|>",
+    "<|reserved_token_12|>",
+    "<|im_media_end|>",
+    "<|reserved_token_13|>",
+    "<|reserved_token_14|>",
+    "<|im_kimia_text_blank|>",
+    "<|im_kimia_text_eos|>",
+    "<|reserved_token_15|>",
+    "<|reserved_token_16|>",
+    "<|im_kimia_user_msg_start|>",
+    "<|im_kimia_assistant_msg_start|>",
+    "<|reserved_token_17|>",
+    "<|reserved_token_18|>",
+    "<|reserved_token_19|>",
+    "<|im_kimia_speech_ct_id|>",
+    "<|im_kimia_speech_ctd_id|>",
+    "<|reserved_token_20|>",
+    "<|reserved_token_21|>",
+    "<|reserved_token_22|>",
+    "<|reserved_token_23|>",
+    "<|reserved_token_24|>",
+    "<|reserved_token_25|>",
+    "<|reserved_token_26|>",
+    "<|reserved_token_27|>",
+    "<|reserved_token_28|>",
+    "<|reserved_token_29|>",
+    "<|reserved_token_30|>",
+    "<|reserved_token_31|>",
+    "<|reserved_token_32|>",
+    "<|reserved_token_33|>",
+    "<|reserved_token_34|>",
+    "<|reserved_token_35|>",
+    "<|reserved_token_36|>",
+    "<|reserved_token_37|>",
+    "<|reserved_token_38|>",
+    "<|reserved_token_39|>",
+    "<|reserved_token_40|>",
+    "<|reserved_token_41|>",
+    "<|reserved_token_42|>",
+    "<|reserved_token_43|>",
+    "<|reserved_token_44|>",
+    "<|reserved_token_45|>",
+    "<|reserved_token_46|>",
+    "<|reserved_token_47|>",
+    "<|reserved_token_48|>",
+    "<|reserved_token_49|>",
+    "<|reserved_token_50|>",
+    "<|reserved_token_51|>",
+    "<|reserved_token_52|>",
+    "<|reserved_token_53|>",
+    "<|reserved_token_54|>",
+    "<|reserved_token_55|>",
+    "<|reserved_token_56|>",
+    "<|reserved_token_57|>",
+    "<|reserved_token_58|>",
+    "<|reserved_token_59|>",
+    "<|reserved_token_60|>",
+    "<|reserved_token_61|>",
+    "<|reserved_token_62|>",
+    "<|reserved_token_63|>",
+    "<|reserved_token_64|>",
+    "<|reserved_token_65|>",
+    "<|reserved_token_66|>",
+    "<|reserved_token_67|>",
+    "<|reserved_token_68|>",
+    "<|reserved_token_69|>",
+    "<|reserved_token_70|>",
+    "<|reserved_token_71|>",
+    "<|reserved_token_72|>",
+    "<|reserved_token_73|>",
+    "<|reserved_token_74|>",
+    "<|reserved_token_75|>",
+    "<|reserved_token_76|>",
+    "<|reserved_token_77|>",
+    "<|reserved_token_78|>",
+    "<|reserved_token_79|>",
+    "<|reserved_token_80|>",
+    "<|reserved_token_81|>",
+    "<|reserved_token_82|>",
+    "<|reserved_token_83|>",
+    "<|reserved_token_84|>",
+    "<|reserved_token_85|>",
+    "<|reserved_token_86|>",
+    "<|reserved_token_87|>",
+    "<|reserved_token_88|>",
+    "<|reserved_token_89|>",
+    "<|reserved_token_90|>",
+    "<|reserved_token_91|>",
+    "<|reserved_token_92|>",
+    "<|reserved_token_93|>",
+    "<|reserved_token_94|>",
+    "<|reserved_token_95|>",
+    "<|reserved_token_96|>",
+    "<|reserved_token_97|>",
+    "<|reserved_token_98|>",
+    "<|reserved_token_99|>",
+    "<|reserved_token_100|>",
+    "<|reserved_token_101|>",
+    "<|reserved_token_102|>",
+    "<|reserved_token_103|>",
+    "<|reserved_token_104|>",
+    "<|reserved_token_105|>",
+    "<|reserved_token_106|>",
+    "<|reserved_token_107|>",
+    "<|reserved_token_108|>",
+    "<|reserved_token_109|>",
+    "<|reserved_token_110|>",
+    "<|reserved_token_111|>",
+    "<|reserved_token_112|>",
+    "<|reserved_token_113|>",
+    "<|reserved_token_114|>",
+    "<|reserved_token_115|>",
+    "<|reserved_token_116|>",
+    "<|reserved_token_117|>",
+    "<|reserved_token_118|>",
+    "<|reserved_token_119|>",
+    "<|reserved_token_120|>",
+    "<|reserved_token_121|>",
+    "<|reserved_token_122|>",
+    "<|reserved_token_123|>",
+    "<|reserved_token_124|>",
+    "<|reserved_token_125|>",
+    "<|reserved_token_126|>",
+    "<|reserved_token_127|>",
+    "<|reserved_token_128|>",
+    "<|reserved_token_129|>",
+    "<|reserved_token_130|>",
+    "<|reserved_token_131|>",
+    "<|reserved_token_132|>",
+    "<|reserved_token_133|>",
+    "<|reserved_token_134|>",
+    "<|reserved_token_135|>",
+    "<|reserved_token_136|>",
+    "<|reserved_token_137|>",
+    "<|reserved_token_138|>",
+    "<|reserved_token_139|>",
+    "<|reserved_token_140|>",
+    "<|reserved_token_141|>",
+    "<|reserved_token_142|>",
+    "<|reserved_token_143|>",
+    "<|reserved_token_144|>",
+    "<|reserved_token_145|>",
+    "<|reserved_token_146|>",
+    "<|reserved_token_147|>",
+    "<|reserved_token_148|>",
+    "<|reserved_token_149|>",
+    "<|reserved_token_150|>",
+    "<|reserved_token_151|>",
+    "<|reserved_token_152|>",
+    "<|reserved_token_153|>",
+    "<|reserved_token_154|>",
+    "<|reserved_token_155|>",
+    "<|reserved_token_156|>",
+    "<|reserved_token_157|>",
+    "<|reserved_token_158|>",
+    "<|reserved_token_159|>",
+    "<|reserved_token_160|>",
+    "<|reserved_token_161|>",
+    "<|reserved_token_162|>",
+    "<|reserved_token_163|>",
+    "<|reserved_token_164|>",
+    "<|reserved_token_165|>",
+    "<|reserved_token_166|>",
+    "<|reserved_token_167|>",
+    "<|reserved_token_168|>",
+    "<|reserved_token_169|>",
+    "<|reserved_token_170|>",
+    "<|reserved_token_171|>",
+    "<|reserved_token_172|>",
+    "<|reserved_token_173|>",
+    "<|reserved_token_174|>",
+    "<|reserved_token_175|>",
+    "<|reserved_token_176|>",
+    "<|reserved_token_177|>",
+    "<|reserved_token_178|>",
+    "<|reserved_token_179|>",
+    "<|reserved_token_180|>",
+    "<|reserved_token_181|>",
+    "<|reserved_token_182|>",
+    "<|reserved_token_183|>",
+    "<|reserved_token_184|>",
+    "<|reserved_token_185|>",
+    "<|reserved_token_186|>",
+    "<|reserved_token_187|>",
+    "<|reserved_token_188|>",
+    "<|reserved_token_189|>",
+    "<|reserved_token_190|>",
+    "<|reserved_token_191|>",
+    "<|reserved_token_192|>",
+    "<|reserved_token_193|>",
+    "<|reserved_token_194|>",
+    "<|reserved_token_195|>",
+    "<|reserved_token_196|>",
+    "<|reserved_token_197|>",
+    "<|reserved_token_198|>",
+    "<|reserved_token_199|>",
+    "<|reserved_token_200|>",
+    "<|reserved_token_201|>",
+    "<|reserved_token_202|>",
+    "<|reserved_token_203|>",
+    "<|reserved_token_204|>",
+    "<|reserved_token_205|>",
+    "<|reserved_token_206|>",
+    "<|reserved_token_207|>",
+    "<|reserved_token_208|>",
+    "<|reserved_token_209|>",
+    "<|reserved_token_210|>",
+    "<|reserved_token_211|>",
+    "<|reserved_token_212|>",
+    "<|reserved_token_213|>",
+    "<|reserved_token_214|>",
+    "<|reserved_token_215|>",
+    "<|reserved_token_216|>",
+    "<|reserved_token_217|>",
+    "<|reserved_token_218|>",
+    "<|reserved_token_219|>",
+    "<|reserved_token_220|>",
+    "<|reserved_token_221|>",
+    "<|reserved_token_222|>",
+    "<|reserved_token_223|>",
+    "<|reserved_token_224|>",
+    "<|reserved_token_225|>",
+    "<|reserved_token_226|>",
+    "<|reserved_token_227|>",
+    "<|reserved_token_228|>",
+    "<|reserved_token_229|>",
+    "<|reserved_token_230|>",
+    "<|reserved_token_231|>",
+    "<|reserved_token_232|>",
+    "<|reserved_token_233|>",
+    "<|reserved_token_234|>",
+    "<|reserved_token_235|>",
+    "<|reserved_token_236|>",
+    "<|reserved_token_237|>",
+    "<|reserved_token_238|>",
+    "<|reserved_token_239|>",
+    "<|reserved_token_240|>",
+    "<|reserved_token_241|>",
+    "<|reserved_token_242|>",
+    "<|reserved_token_243|>",
+    "<|reserved_token_244|>",
+    "<|reserved_token_245|>",
+    "<|reserved_token_246|>",
+    "<|reserved_token_247|>",
+    "<|reserved_token_248|>",
+    "<|reserved_token_249|>",
+    "<|reserved_token_250|>",
+    "<|reserved_token_251|>",
+    "<|reserved_token_252|>",
+    "<|reserved_token_253|>",
+    "<|reserved_token_254|>",
+    "<|reserved_token_255|>",
+    "<|reserved_token_256|>",
+    "<|reserved_token_257|>",
+    "<|reserved_token_258|>",
+    "<|reserved_token_259|>",
+    "<|reserved_token_260|>",
+    "<|reserved_token_261|>",
+    "<|reserved_token_262|>",
+    "<|reserved_token_263|>",
+    "<|reserved_token_264|>",
+    "<|reserved_token_265|>",
+    "<|reserved_token_266|>",
+    "<|reserved_token_267|>",
+    "<|reserved_token_268|>",
+    "<|reserved_token_269|>",
+    "<|reserved_token_270|>",
+    "<|reserved_token_271|>",
+    "<|reserved_token_272|>",
+    "<|reserved_token_273|>",
+    "<|reserved_token_274|>",
+    "<|reserved_token_275|>",
+    "<|reserved_token_276|>",
+    "<|reserved_token_277|>",
+    "<|reserved_token_278|>",
+    "<|reserved_token_279|>",
+    "<|reserved_token_280|>",
+    "<|reserved_token_281|>",
+    "<|reserved_token_282|>",
+    "<|reserved_token_283|>",
+    "<|reserved_token_284|>",
+    "<|reserved_token_285|>",
+    "<|reserved_token_286|>",
+    "<|reserved_token_287|>",
+    "<|reserved_token_288|>",
+    "<|reserved_token_289|>",
+    "<|reserved_token_290|>",
+    "<|reserved_token_291|>",
+    "<|reserved_token_292|>",
+    "<|reserved_token_293|>",
+    "<|reserved_token_294|>",
+    "<|reserved_token_295|>",
+    "<|reserved_token_296|>",
+    "<|reserved_token_297|>",
+    "<|reserved_token_298|>",
+    "<|reserved_token_299|>",
+    "<|reserved_token_300|>",
+    "<|reserved_token_301|>",
+    "<|reserved_token_302|>",
+    "<|reserved_token_303|>",
+    "<|reserved_token_304|>",
+    "<|reserved_token_305|>",
+    "<|reserved_token_306|>",
+    "<|reserved_token_307|>",
+    "<|reserved_token_308|>",
+    "<|reserved_token_309|>",
+    "<|reserved_token_310|>",
+    "<|reserved_token_311|>",
+    "<|reserved_token_312|>",
+    "<|reserved_token_313|>",
+    "<|reserved_token_314|>",
+    "<|reserved_token_315|>",
+    "<|reserved_token_316|>",
+    "<|reserved_token_317|>",
+    "<|reserved_token_318|>",
+    "<|reserved_token_319|>",
+    "<|reserved_token_320|>",
+    "<|reserved_token_321|>",
+    "<|reserved_token_322|>",
+    "<|reserved_token_323|>",
+    "<|reserved_token_324|>",
+    "<|reserved_token_325|>",
+    "<|reserved_token_326|>",
+    "<|reserved_token_327|>",
+    "<|reserved_token_328|>",
+    "<|reserved_token_329|>",
+    "<|reserved_token_330|>",
+    "<|reserved_token_331|>",
+    "<|reserved_token_332|>",
+    "<|reserved_token_333|>",
+    "<|reserved_token_334|>",
+    "<|reserved_token_335|>",
+    "<|reserved_token_336|>",
+    "<|reserved_token_337|>",
+    "<|reserved_token_338|>",
+    "<|reserved_token_339|>",
+    "<|reserved_token_340|>",
+    "<|reserved_token_341|>",
+    "<|reserved_token_342|>",
+    "<|reserved_token_343|>",
+    "<|reserved_token_344|>",
+    "<|reserved_token_345|>",
+    "<|reserved_token_346|>",
+    "<|reserved_token_347|>",
+    "<|reserved_token_348|>",
+    "<|reserved_token_349|>",
+    "<|reserved_token_350|>",
+    "<|reserved_token_351|>",
+    "<|reserved_token_352|>",
+    "<|reserved_token_353|>",
+    "<|reserved_token_354|>",
+    "<|reserved_token_355|>",
+    "<|reserved_token_356|>",
+    "<|reserved_token_357|>",
+    "<|reserved_token_358|>",
+    "<|reserved_token_359|>",
+    "<|reserved_token_360|>",
+    "<|reserved_token_361|>",
+    "<|reserved_token_362|>",
+    "<|reserved_token_363|>",
+    "<|reserved_token_364|>",
+    "<|reserved_token_365|>",
+    "<|reserved_token_366|>",
+    "<|reserved_token_367|>",
+    "<|reserved_token_368|>",
+    "<|reserved_token_369|>",
+    "<|reserved_token_370|>",
+    "<|reserved_token_371|>",
+    "<|reserved_token_372|>",
+    "<|reserved_token_373|>",
+    "<|reserved_token_374|>",
+    "<|reserved_token_375|>",
+    "<|reserved_token_376|>",
+    "<|reserved_token_377|>",
+    "<|reserved_token_378|>",
+    "<|reserved_token_379|>",
+    "<|reserved_token_380|>",
+    "<|reserved_token_381|>",
+    "<|reserved_token_382|>",
+    "<|reserved_token_383|>",
+    "<|reserved_token_384|>",
+    "<|reserved_token_385|>",
+    "<|reserved_token_386|>",
+    "<|reserved_token_387|>",
+    "<|reserved_token_388|>",
+    "<|reserved_token_389|>",
+    "<|reserved_token_390|>",
+    "<|reserved_token_391|>",
+    "<|reserved_token_392|>",
+    "<|reserved_token_393|>",
+    "<|reserved_token_394|>",
+    "<|reserved_token_395|>",
+    "<|reserved_token_396|>",
+    "<|reserved_token_397|>",
+    "<|reserved_token_398|>",
+    "<|reserved_token_399|>",
+    "<|reserved_token_400|>",
+    "<|reserved_token_401|>",
+    "<|reserved_token_402|>",
+    "<|reserved_token_403|>",
+    "<|reserved_token_404|>"
+  ],
+  "bos_token": "[BOS]",
+  "eos_token": "[EOS]",
+  "pad_token": "<|reserved_token_406|>",
+  "unk_token": "<|reserved_token_405|>"
+}

tiktoken.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2b1b8dfb5cc5f024bafc373121c6aba3f66f9a5a0269e243470a1de16a33186
+size 2561218

tokenization_echochat.py

@@ -0,0 +1,336 @@
+# Modified for EchoChat distribution: renamed public entry points; computation unchanged.
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+
+"""Megatron tokenizers."""
+from transformers.tokenization_utils import PreTrainedTokenizer
+from typing import Union
+from typing import (
+    AbstractSet,
+    cast,
+    Collection,
+    Dict,
+    Iterator,
+    List,
+    Literal,
+    Sequence,
+    Union,
+    Optional,
+)
+from tiktoken.load import load_tiktoken_bpe
+import tiktoken
+from pathlib import Path
+import os
+import logging
+from tokenizers import AddedToken
+
+logger = logging.getLogger(__name__)
+VOCAB_FILES_NAMES = {"vocab_file": "tiktoken.model"}
+
+
+class TikTokenTokenizer(PreTrainedTokenizer):
+    """
+    Tokenizing and encoding/decoding text using the Tiktoken tokenizer.
+    """
+
+    special_tokens: Dict[str, int]
+
+    num_reserved_special_tokens = 293 + 128
+
+    pat_str = "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
+
+    vocab_files_names = VOCAB_FILES_NAMES
+
+    def __init__(
+        self,
+        vocab_file,
+        bos_token: Union[str, AddedToken] = "[BOS]",
+        eos_token: Union[str, AddedToken] = "[EOS]",
+        unk_token: Union[str, AddedToken] = "[UNK]",
+        pad_token: Union[str, AddedToken] = "[PAD]",
+        additional_special_tokens: Optional[List[str]] = None,
+        added_tokens_decoder: Optional[dict] = None,
+        **kwargs,
+    ):
+        """
+        Initializes the Tokenizer with a Tiktoken model.
+
+        Args:
+            model_path (str): The path to the Tiktoken model file.
+        """
+        assert os.path.isfile(vocab_file), vocab_file
+
+        mergeable_ranks = load_tiktoken_bpe(vocab_file)
+        num_base_tokens = len(mergeable_ranks)
+
+        used_special_tokens = [
+            "[BOS]",
+            "[EOS]",
+            "<|im_msg_end|>",  # 0
+            "<|im_user_msg_start|>",  # 1
+            "<|im_assistant_msg_start|>",  # 2
+            "<|reserved_token_0|>",  # 3
+            "<|reserved_token_1|>",
+            "<|reserved_token_2|>",
+            "<|reserved_token_3|>",  # 4
+            "[EOT]",
+            "<|reserved_token_4|>",  # 5
+            "<|reserved_token_5|>",  # 6
+            "<|reserved_token_6|>",  # 7
+            "<|reserved_token_7|>",  # 8
+            "<|reserved_token_8|>",  # 9
+            "<|reserved_token_9|>",  # 10
+            "<|reserved_token_10|>",  # 11
+            "<|reserved_token_11|>",  # 12
+            "<|im_media_begin|>",  # 13
+            "<|reserved_token_12|>",  # 14
+            "<|im_media_end|>",  # 15
+            "<|reserved_token_13|>",  # 16
+            "<|reserved_token_14|>",  # 17
+            "<|im_kimia_text_blank|>",  # 18
+            "<|im_kimia_text_eos|>",  # 19
+            "<|reserved_token_15|>",  # 20
+            "<|reserved_token_16|>",  # 21
+            "<|im_kimia_user_msg_start|>",  # 22
+            "<|im_kimia_assistant_msg_start|>",  # 23
+            "<|reserved_token_17|>",  # 24
+            "<|reserved_token_18|>",  # 25
+            "<|reserved_token_19|>",  # 26
+            "<|im_kimia_speech_ct_id|>",  # 27
+            "<|im_kimia_speech_ctd_id|>",  # 28
+        ]
+        autoset_special_tokens = [
+            f"<|reserved_token_{i}|>"
+            for i in range(
+                20, self.num_reserved_special_tokens - len(used_special_tokens) + 20
+            )
+        ]
+        special_tokens = used_special_tokens + autoset_special_tokens
+        self.special_tokens = {
+            token: num_base_tokens + i for i, token in enumerate(special_tokens)
+        }
+        self.model = tiktoken.Encoding(
+            name=Path(vocab_file).name,
+            pat_str=self.pat_str,
+            mergeable_ranks=mergeable_ranks,
+            special_tokens=self.special_tokens,
+        )
+        logger.info(f"Reloaded tiktoken model from {vocab_file}")
+
+        self.n_words: int = self.model.n_vocab
+        # BOS / EOS token IDs
+        self.bos_token = "[BOS]"
+        self.bos_id: int = self.special_tokens["[BOS]"]
+        self.eos_token = "[EOS]"
+        self.eos_id: int = self.special_tokens["[EOS]"]
+
+        # use last speical token as pad token, the last - 1 is unk_token
+        self.pad_token: str = special_tokens[-1]
+        self.pad_id: int = self.special_tokens[self.pad_token]
+
+        self.unk_token: str = special_tokens[-2]
+        self.unk_id: int = self.special_tokens[self.pad_token]
+
+        self.stop_tokens = {
+            self.special_tokens["[EOS]"],
+            self.special_tokens["[EOT]"],
+        }
+
+        logger.info(
+            f"#words: {self.n_words} - BOS ID: {self.bos_id} - EOS ID: {self.eos_id}"
+        )
+
+    def encode(
+        self,
+        s: str,
+        *,
+        bos: bool,
+        eos: bool,
+        allowed_special: Union[Literal["all"], AbstractSet[str]] = set(),
+        disallowed_special: Union[Literal["all"], Collection[str]] = (),
+    ) -> List[int]:
+        """
+        Encodes a string into a list of token IDs.
+
+        Args:
+            s (str): The input string to be encoded.
+            bos (bool): Whether to prepend the beginning-of-sequence token.
+            eos (bool): Whether to append the end-of-sequence token.
+            allowed_tokens ("all"|set[str]): allowed special tokens in string
+            disallowed_tokens ("all"|set[str]): special tokens that raise an error when in string
+
+        Returns:
+            list[int]: A list of token IDs.
+
+        By default, setting disallowed_special=() encodes a string by ignoring
+        special tokens. Specifically:
+        - Setting `disallowed_special` to () will cause all text corresponding
+          to special tokens to be encoded as natural text (insteading of raising
+          an error).
+        - Setting `allowed_special` to "all" will treat all text corresponding
+          to special tokens to be encoded as special tokens.
+        """
+        assert type(s) is str
+
+        # The tiktoken tokenizer can handle <=400k chars without
+        # pyo3_runtime.PanicException.
+        TIKTOKEN_MAX_ENCODE_CHARS = 400_000
+
+        # https://github.com/openai/tiktoken/issues/195
+        # Here we iterate over subsequences and split if we exceed the limit
+        # of max consecutive non-whitespace or whitespace characters.
+        MAX_NO_WHITESPACES_CHARS = 25_000
+
+        substrs = (
+            substr
+            for i in range(0, len(s), TIKTOKEN_MAX_ENCODE_CHARS)
+            for substr in self._split_whitespaces_or_nonwhitespaces(
+                s[i : i + TIKTOKEN_MAX_ENCODE_CHARS], MAX_NO_WHITESPACES_CHARS
+            )
+        )
+        t: List[int] = []
+        for substr in substrs:
+            t.extend(
+                self.model.encode(
+                    substr,
+                    allowed_special=allowed_special,
+                    disallowed_special=disallowed_special,
+                )
+            )
+        if bos:
+            t.insert(0, self.bos_id)
+        if eos:
+            t.append(self.eos_id)
+        return t
+
+    def decode(self, t: Sequence[int]) -> str:
+        """
+        Decodes a list of token IDs into a string.
+
+        Args:
+            t (List[int]): The list of token IDs to be decoded.
+
+        Returns:
+            str: The decoded string.
+        """
+        # Typecast is safe here. Tiktoken doesn't do anything list-related with the sequence.
+        return self.model.decode(cast(List[int], t))
+
+    @staticmethod
+    def _split_whitespaces_or_nonwhitespaces(
+        s: str, max_consecutive_slice_len: int
+    ) -> Iterator[str]:
+        """
+        Splits the string `s` so that each substring contains no more than `max_consecutive_slice_len`
+        consecutive whitespaces or consecutive non-whitespaces.
+        """
+        current_slice_len = 0
+        current_slice_is_space = s[0].isspace() if len(s) > 0 else False
+        slice_start = 0
+
+        for i in range(len(s)):
+            is_now_space = s[i].isspace()
+
+            if current_slice_is_space ^ is_now_space:
+                current_slice_len = 1
+                current_slice_is_space = is_now_space
+            else:
+                current_slice_len += 1
+                if current_slice_len > max_consecutive_slice_len:
+                    yield s[slice_start:i]
+                    slice_start = i
+                    current_slice_len = 1
+        yield s[slice_start:]
+
+    """ ----- Below are the abstract methods required by megatron ----- """
+
+    @property
+    def vocab_size(self):
+        return self.n_words
+
+    @property
+    def vocab(self):
+        if hasattr(self, "str_vocab"):
+            return self.str_vocab
+        self.str_vocab = {}
+
+        # convert mergeable_ranks from bytes to string
+        utf8_num, unicode_num = 0, 0
+        for byte_key, index in self.model._mergeable_ranks.items():
+            try:
+                str_key = byte_key.decode("utf-8")
+                utf8_num += 1
+            except UnicodeDecodeError:
+                # use backslashreplace so we can get num vocab different tokens
+                # see: https://docs.python.org/3/howto/unicode.html
+                # this vocab is only used for offline processing, so this is fine
+                str_key = byte_key.decode("utf-8", "backslashreplace") + "_unicode_"
+                unicode_num += 1
+
+            self.str_vocab[str_key] = index
+        logger.info(f"num utf8: {utf8_num}, num unicode: {unicode_num}")
+
+        # add all special tokens to the dictionary
+        self.str_vocab.update(self.model._special_tokens)
+
+        assert len(self.str_vocab) == self.vocab_size
+        return self.str_vocab
+
+    @property
+    def inv_vocab(self):
+        return {v: k for k, v in self.vocab.items()}
+
+    def tokenize(self, text, eos=True):
+        # BOS: always add bos token
+        # EOS:
+        #    Most cases should be true when we are tokenizing a full sequence
+        #    Only setting to false when we are running a inference
+        return self.encode(text, bos=True, eos=eos)
+
+    def detokenize(self, tokens):
+        # convert tensor to list if needed...
+        if not isinstance(tokens, list):
+            tokens = tokens.tolist()
+        return self.decode(tokens)
+
+    @property
+    def eod(self):
+        return self.eos_id
+
+    def bod(self):
+        return self.bos_id
+
+    @property
+    def msk_start_id(self):
+        return self.msk_start
+
+    @property
+    def msk_end_id(self):
+        return self.msk_end
+
+    def _get_index_2_bytes(self):
+        if hasattr(self, "index_2_bytes"):
+            return self.index_2_bytes
+
+        # use array rather than dict for faster access
+        self.index_2_bytes = [0] * self.model.n_vocab
+        for byte_key, index in self.model._mergeable_ranks.items():
+            self.index_2_bytes[index] = len(byte_key)
+
+        for _, index in self.model._special_tokens.items():
+            # in total we have 256 special tokens, 2^8 = 256
+            # so the num of bytes of each token is only 1
+            self.index_2_bytes[index] = 1
+
+        return self.index_2_bytes
+
+    def get_array_bytes(self, array):
+        index_2_bytes = self._get_index_2_bytes()
+        return sum(index_2_bytes[i] for i in array)
+
+    @property
+    def eos_token_id(self):
+        return self.eos_id
+
+    @property
+    def pad_token_id(self):
+        return self.pad_id

tokenizer_config.json

@@ -0,0 +1,3805 @@
+{
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "[BOS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "[EOS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_msg_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|im_user_msg_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|im_assistant_msg_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|reserved_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|reserved_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|reserved_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|reserved_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "[EOT]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|reserved_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|reserved_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|reserved_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|reserved_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<|reserved_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151658": {
+      "content": "<|reserved_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151659": {
+      "content": "<|reserved_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151660": {
+      "content": "<|reserved_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151661": {
+      "content": "<|im_media_begin|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151662": {
+      "content": "<|reserved_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151663": {
+      "content": "<|im_media_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151664": {
+      "content": "<|reserved_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151665": {
+      "content": "<|reserved_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "<|im_kimia_text_blank|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "<|im_kimia_text_eos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "<|reserved_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151669": {
+      "content": "<|reserved_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<|im_kimia_user_msg_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "<|im_kimia_assistant_msg_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<|reserved_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "<|reserved_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "<|reserved_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151675": {
+      "content": "<|im_kimia_speech_ct_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151676": {
+      "content": "<|im_kimia_speech_ctd_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151677": {
+      "content": "<|reserved_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151678": {
+      "content": "<|reserved_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151679": {
+      "content": "<|reserved_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151680": {
+      "content": "<|reserved_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151681": {
+      "content": "<|reserved_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151682": {
+      "content": "<|reserved_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151683": {
+      "content": "<|reserved_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151684": {
+      "content": "<|reserved_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151685": {
+      "content": "<|reserved_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151686": {
+      "content": "<|reserved_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151687": {
+      "content": "<|reserved_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151688": {
+      "content": "<|reserved_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151689": {
+      "content": "<|reserved_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151690": {
+      "content": "<|reserved_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151691": {
+      "content": "<|reserved_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151692": {
+      "content": "<|reserved_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151693": {
+      "content": "<|reserved_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151694": {
+      "content": "<|reserved_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151695": {
+      "content": "<|reserved_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151696": {
+      "content": "<|reserved_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151697": {
+      "content": "<|reserved_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151698": {
+      "content": "<|reserved_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151699": {
+      "content": "<|reserved_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151700": {
+      "content": "<|reserved_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151701": {
+      "content": "<|reserved_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151702": {
+      "content": "<|reserved_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151703": {
+      "content": "<|reserved_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151704": {
+      "content": "<|reserved_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151705": {
+      "content": "<|reserved_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151706": {
+      "content": "<|reserved_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151707": {
+      "content": "<|reserved_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151708": {
+      "content": "<|reserved_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151709": {
+      "content": "<|reserved_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151710": {
+      "content": "<|reserved_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151711": {
+      "content": "<|reserved_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151712": {
+      "content": "<|reserved_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151713": {
+      "content": "<|reserved_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151714": {
+      "content": "<|reserved_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151715": {
+      "content": "<|reserved_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151716": {
+      "content": "<|reserved_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151717": {
+      "content": "<|reserved_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151718": {
+      "content": "<|reserved_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151719": {
+      "content": "<|reserved_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151720": {
+      "content": "<|reserved_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151721": {
+      "content": "<|reserved_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151722": {
+      "content": "<|reserved_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151723": {
+      "content": "<|reserved_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151724": {
+      "content": "<|reserved_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151725": {
+      "content": "<|reserved_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151726": {
+      "content": "<|reserved_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151727": {
+      "content": "<|reserved_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151728": {
+      "content": "<|reserved_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151729": {
+      "content": "<|reserved_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151730": {
+      "content": "<|reserved_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151731": {
+      "content": "<|reserved_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151732": {
+      "content": "<|reserved_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151733": {
+      "content": "<|reserved_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151734": {
+      "content": "<|reserved_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151735": {
+      "content": "<|reserved_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151736": {
+      "content": "<|reserved_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151737": {
+      "content": "<|reserved_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151738": {
+      "content": "<|reserved_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151739": {
+      "content": "<|reserved_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151740": {
+      "content": "<|reserved_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151741": {
+      "content": "<|reserved_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151742": {
+      "content": "<|reserved_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151743": {
+      "content": "<|reserved_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151744": {
+      "content": "<|reserved_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151745": {
+      "content": "<|reserved_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151746": {
+      "content": "<|reserved_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151747": {
+      "content": "<|reserved_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151748": {
+      "content": "<|reserved_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151749": {
+      "content": "<|reserved_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151750": {
+      "content": "<|reserved_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151751": {
+      "content": "<|reserved_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151752": {
+      "content": "<|reserved_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151753": {
+      "content": "<|reserved_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151754": {
+      "content": "<|reserved_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151755": {
+      "content": "<|reserved_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151756": {
+      "content": "<|reserved_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151757": {
+      "content": "<|reserved_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151758": {
+      "content": "<|reserved_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151759": {
+      "content": "<|reserved_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151760": {
+      "content": "<|reserved_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151761": {
+      "content": "<|reserved_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151762": {
+      "content": "<|reserved_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151763": {
+      "content": "<|reserved_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151764": {
+      "content": "<|reserved_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151765": {
+      "content": "<|reserved_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151766": {
+      "content": "<|reserved_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151767": {
+      "content": "<|reserved_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151768": {
+      "content": "<|reserved_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151769": {
+      "content": "<|reserved_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151770": {
+      "content": "<|reserved_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151771": {
+      "content": "<|reserved_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151772": {
+      "content": "<|reserved_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151773": {
+      "content": "<|reserved_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151774": {
+      "content": "<|reserved_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151775": {
+      "content": "<|reserved_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151776": {
+      "content": "<|reserved_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151777": {
+      "content": "<|reserved_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151778": {
+      "content": "<|reserved_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151779": {
+      "content": "<|reserved_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151780": {
+      "content": "<|reserved_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151781": {
+      "content": "<|reserved_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151782": {
+      "content": "<|reserved_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151783": {
+      "content": "<|reserved_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151784": {
+      "content": "<|reserved_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151785": {
+      "content": "<|reserved_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151786": {
+      "content": "<|reserved_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151787": {
+      "content": "<|reserved_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151788": {
+      "content": "<|reserved_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151789": {
+      "content": "<|reserved_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151790": {
+      "content": "<|reserved_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151791": {
+      "content": "<|reserved_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151792": {
+      "content": "<|reserved_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151793": {
+      "content": "<|reserved_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151794": {
+      "content": "<|reserved_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151795": {
+      "content": "<|reserved_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151796": {
+      "content": "<|reserved_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151797": {
+      "content": "<|reserved_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151798": {
+      "content": "<|reserved_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151799": {
+      "content": "<|reserved_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151800": {
+      "content": "<|reserved_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151801": {
+      "content": "<|reserved_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151802": {
+      "content": "<|reserved_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151803": {
+      "content": "<|reserved_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151804": {
+      "content": "<|reserved_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151805": {
+      "content": "<|reserved_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151806": {
+      "content": "<|reserved_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151807": {
+      "content": "<|reserved_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151808": {
+      "content": "<|reserved_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151809": {
+      "content": "<|reserved_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151810": {
+      "content": "<|reserved_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151811": {
+      "content": "<|reserved_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151812": {
+      "content": "<|reserved_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151813": {
+      "content": "<|reserved_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151814": {
+      "content": "<|reserved_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151815": {
+      "content": "<|reserved_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151816": {
+      "content": "<|reserved_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151817": {
+      "content": "<|reserved_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151818": {
+      "content": "<|reserved_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151819": {
+      "content": "<|reserved_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151820": {
+      "content": "<|reserved_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151821": {
+      "content": "<|reserved_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151822": {
+      "content": "<|reserved_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151823": {
+      "content": "<|reserved_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151824": {
+      "content": "<|reserved_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151825": {
+      "content": "<|reserved_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151826": {
+      "content": "<|reserved_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151827": {
+      "content": "<|reserved_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151828": {
+      "content": "<|reserved_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151829": {
+      "content": "<|reserved_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151830": {
+      "content": "<|reserved_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151831": {
+      "content": "<|reserved_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151832": {
+      "content": "<|reserved_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151833": {
+      "content": "<|reserved_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151834": {
+      "content": "<|reserved_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151835": {
+      "content": "<|reserved_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151836": {
+      "content": "<|reserved_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151837": {
+      "content": "<|reserved_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151838": {
+      "content": "<|reserved_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151839": {
+      "content": "<|reserved_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151840": {
+      "content": "<|reserved_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151841": {
+      "content": "<|reserved_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151842": {
+      "content": "<|reserved_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151843": {
+      "content": "<|reserved_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151844": {
+      "content": "<|reserved_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151845": {
+      "content": "<|reserved_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151846": {
+      "content": "<|reserved_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151847": {
+      "content": "<|reserved_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151848": {
+      "content": "<|reserved_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151849": {
+      "content": "<|reserved_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151850": {
+      "content": "<|reserved_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151851": {
+      "content": "<|reserved_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151852": {
+      "content": "<|reserved_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151853": {
+      "content": "<|reserved_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151854": {
+      "content": "<|reserved_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151855": {
+      "content": "<|reserved_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151856": {
+      "content": "<|reserved_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151857": {
+      "content": "<|reserved_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151858": {
+      "content": "<|reserved_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151859": {
+      "content": "<|reserved_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151860": {
+      "content": "<|reserved_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151861": {
+      "content": "<|reserved_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151862": {
+      "content": "<|reserved_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151863": {
+      "content": "<|reserved_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151864": {
+      "content": "<|reserved_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151865": {
+      "content": "<|reserved_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151866": {
+      "content": "<|reserved_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151867": {
+      "content": "<|reserved_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151868": {
+      "content": "<|reserved_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151869": {
+      "content": "<|reserved_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151870": {
+      "content": "<|reserved_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151871": {
+      "content": "<|reserved_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151872": {
+      "content": "<|reserved_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151873": {
+      "content": "<|reserved_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151874": {
+      "content": "<|reserved_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151875": {
+      "content": "<|reserved_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151876": {
+      "content": "<|reserved_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151877": {
+      "content": "<|reserved_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151878": {
+      "content": "<|reserved_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151879": {
+      "content": "<|reserved_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151880": {
+      "content": "<|reserved_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151881": {
+      "content": "<|reserved_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151882": {
+      "content": "<|reserved_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151883": {
+      "content": "<|reserved_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151884": {
+      "content": "<|reserved_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151885": {
+      "content": "<|reserved_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151886": {
+      "content": "<|reserved_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151887": {
+      "content": "<|reserved_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151888": {
+      "content": "<|reserved_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151889": {
+      "content": "<|reserved_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151890": {
+      "content": "<|reserved_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151891": {
+      "content": "<|reserved_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151892": {
+      "content": "<|reserved_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151893": {
+      "content": "<|reserved_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151894": {
+      "content": "<|reserved_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151895": {
+      "content": "<|reserved_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151896": {
+      "content": "<|reserved_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151897": {
+      "content": "<|reserved_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151898": {
+      "content": "<|reserved_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151899": {
+      "content": "<|reserved_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151900": {
+      "content": "<|reserved_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151901": {
+      "content": "<|reserved_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151902": {
+      "content": "<|reserved_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151903": {
+      "content": "<|reserved_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151904": {
+      "content": "<|reserved_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151905": {
+      "content": "<|reserved_token_248|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151906": {
+      "content": "<|reserved_token_249|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151907": {
+      "content": "<|reserved_token_250|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151908": {
+      "content": "<|reserved_token_251|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151909": {
+      "content": "<|reserved_token_252|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151910": {
+      "content": "<|reserved_token_253|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151911": {
+      "content": "<|reserved_token_254|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151912": {
+      "content": "<|reserved_token_255|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151913": {
+      "content": "<|reserved_token_256|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151914": {
+      "content": "<|reserved_token_257|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151915": {
+      "content": "<|reserved_token_258|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151916": {
+      "content": "<|reserved_token_259|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151917": {
+      "content": "<|reserved_token_260|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151918": {
+      "content": "<|reserved_token_261|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151919": {
+      "content": "<|reserved_token_262|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151920": {
+      "content": "<|reserved_token_263|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151921": {
+      "content": "<|reserved_token_264|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151922": {
+      "content": "<|reserved_token_265|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151923": {
+      "content": "<|reserved_token_266|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151924": {
+      "content": "<|reserved_token_267|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151925": {
+      "content": "<|reserved_token_268|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151926": {
+      "content": "<|reserved_token_269|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151927": {
+      "content": "<|reserved_token_270|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151928": {
+      "content": "<|reserved_token_271|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151929": {
+      "content": "<|reserved_token_272|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151930": {
+      "content": "<|reserved_token_273|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151931": {
+      "content": "<|reserved_token_274|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151932": {
+      "content": "<|reserved_token_275|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151933": {
+      "content": "<|reserved_token_276|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151934": {
+      "content": "<|reserved_token_277|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151935": {
+      "content": "<|reserved_token_278|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151936": {
+      "content": "<|reserved_token_279|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151937": {
+      "content": "<|reserved_token_280|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151938": {
+      "content": "<|reserved_token_281|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151939": {
+      "content": "<|reserved_token_282|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151940": {
+      "content": "<|reserved_token_283|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151941": {
+      "content": "<|reserved_token_284|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151942": {
+      "content": "<|reserved_token_285|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151943": {
+      "content": "<|reserved_token_286|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151944": {
+      "content": "<|reserved_token_287|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151945": {
+      "content": "<|reserved_token_288|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151946": {
+      "content": "<|reserved_token_289|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151947": {
+      "content": "<|reserved_token_290|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151948": {
+      "content": "<|reserved_token_291|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151949": {
+      "content": "<|reserved_token_292|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151950": {
+      "content": "<|reserved_token_293|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151951": {
+      "content": "<|reserved_token_294|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151952": {
+      "content": "<|reserved_token_295|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151953": {
+      "content": "<|reserved_token_296|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151954": {
+      "content": "<|reserved_token_297|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151955": {
+      "content": "<|reserved_token_298|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151956": {
+      "content": "<|reserved_token_299|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151957": {
+      "content": "<|reserved_token_300|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151958": {
+      "content": "<|reserved_token_301|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151959": {
+      "content": "<|reserved_token_302|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151960": {
+      "content": "<|reserved_token_303|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151961": {
+      "content": "<|reserved_token_304|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151962": {
+      "content": "<|reserved_token_305|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151963": {
+      "content": "<|reserved_token_306|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151964": {
+      "content": "<|reserved_token_307|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151965": {
+      "content": "<|reserved_token_308|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151966": {
+      "content": "<|reserved_token_309|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151967": {
+      "content": "<|reserved_token_310|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151968": {
+      "content": "<|reserved_token_311|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151969": {
+      "content": "<|reserved_token_312|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151970": {
+      "content": "<|reserved_token_313|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151971": {
+      "content": "<|reserved_token_314|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151972": {
+      "content": "<|reserved_token_315|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151973": {
+      "content": "<|reserved_token_316|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151974": {
+      "content": "<|reserved_token_317|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151975": {
+      "content": "<|reserved_token_318|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151976": {
+      "content": "<|reserved_token_319|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151977": {
+      "content": "<|reserved_token_320|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151978": {
+      "content": "<|reserved_token_321|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151979": {
+      "content": "<|reserved_token_322|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151980": {
+      "content": "<|reserved_token_323|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151981": {
+      "content": "<|reserved_token_324|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151982": {
+      "content": "<|reserved_token_325|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151983": {
+      "content": "<|reserved_token_326|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151984": {
+      "content": "<|reserved_token_327|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151985": {
+      "content": "<|reserved_token_328|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151986": {
+      "content": "<|reserved_token_329|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151987": {
+      "content": "<|reserved_token_330|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151988": {
+      "content": "<|reserved_token_331|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151989": {
+      "content": "<|reserved_token_332|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151990": {
+      "content": "<|reserved_token_333|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151991": {
+      "content": "<|reserved_token_334|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151992": {
+      "content": "<|reserved_token_335|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151993": {
+      "content": "<|reserved_token_336|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151994": {
+      "content": "<|reserved_token_337|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151995": {
+      "content": "<|reserved_token_338|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151996": {
+      "content": "<|reserved_token_339|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151997": {
+      "content": "<|reserved_token_340|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151998": {
+      "content": "<|reserved_token_341|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151999": {
+      "content": "<|reserved_token_342|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152000": {
+      "content": "<|reserved_token_343|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152001": {
+      "content": "<|reserved_token_344|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152002": {
+      "content": "<|reserved_token_345|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152003": {
+      "content": "<|reserved_token_346|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152004": {
+      "content": "<|reserved_token_347|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152005": {
+      "content": "<|reserved_token_348|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152006": {
+      "content": "<|reserved_token_349|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152007": {
+      "content": "<|reserved_token_350|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152008": {
+      "content": "<|reserved_token_351|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152009": {
+      "content": "<|reserved_token_352|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152010": {
+      "content": "<|reserved_token_353|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152011": {
+      "content": "<|reserved_token_354|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152012": {
+      "content": "<|reserved_token_355|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152013": {
+      "content": "<|reserved_token_356|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152014": {
+      "content": "<|reserved_token_357|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152015": {
+      "content": "<|reserved_token_358|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152016": {
+      "content": "<|reserved_token_359|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152017": {
+      "content": "<|reserved_token_360|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152018": {
+      "content": "<|reserved_token_361|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152019": {
+      "content": "<|reserved_token_362|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152020": {
+      "content": "<|reserved_token_363|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152021": {
+      "content": "<|reserved_token_364|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152022": {
+      "content": "<|reserved_token_365|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152023": {
+      "content": "<|reserved_token_366|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152024": {
+      "content": "<|reserved_token_367|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152025": {
+      "content": "<|reserved_token_368|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152026": {
+      "content": "<|reserved_token_369|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152027": {
+      "content": "<|reserved_token_370|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152028": {
+      "content": "<|reserved_token_371|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152029": {
+      "content": "<|reserved_token_372|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152030": {
+      "content": "<|reserved_token_373|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152031": {
+      "content": "<|reserved_token_374|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152032": {
+      "content": "<|reserved_token_375|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152033": {
+      "content": "<|reserved_token_376|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152034": {
+      "content": "<|reserved_token_377|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152035": {
+      "content": "<|reserved_token_378|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152036": {
+      "content": "<|reserved_token_379|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152037": {
+      "content": "<|reserved_token_380|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152038": {
+      "content": "<|reserved_token_381|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152039": {
+      "content": "<|reserved_token_382|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152040": {
+      "content": "<|reserved_token_383|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152041": {
+      "content": "<|reserved_token_384|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152042": {
+      "content": "<|reserved_token_385|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152043": {
+      "content": "<|reserved_token_386|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152044": {
+      "content": "<|reserved_token_387|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152045": {
+      "content": "<|reserved_token_388|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152046": {
+      "content": "<|reserved_token_389|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152047": {
+      "content": "<|reserved_token_390|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152048": {
+      "content": "<|reserved_token_391|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152049": {
+      "content": "<|reserved_token_392|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152050": {
+      "content": "<|reserved_token_393|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152051": {
+      "content": "<|reserved_token_394|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152052": {
+      "content": "<|reserved_token_395|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152053": {
+      "content": "<|reserved_token_396|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152054": {
+      "content": "<|reserved_token_397|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152055": {
+      "content": "<|reserved_token_398|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152056": {
+      "content": "<|reserved_token_399|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152057": {
+      "content": "<|reserved_token_400|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152058": {
+      "content": "<|reserved_token_401|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152059": {
+      "content": "<|reserved_token_402|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152060": {
+      "content": "<|reserved_token_403|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152061": {
+      "content": "<|reserved_token_404|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152062": {
+      "content": "<|reserved_token_405|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152063": {
+      "content": "<|reserved_token_406|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_msg_end|>",
+    "<|im_user_msg_start|>",
+    "<|im_assistant_msg_start|>",
+    "<|reserved_token_0|>",
+    "<|reserved_token_1|>",
+    "<|reserved_token_2|>",
+    "<|reserved_token_3|>",
+    "[EOT]",
+    "<|reserved_token_4|>",
+    "<|reserved_token_5|>",
+    "<|reserved_token_6|>",
+    "<|reserved_token_7|>",
+    "<|reserved_token_8|>",
+    "<|reserved_token_9|>",
+    "<|reserved_token_10|>",
+    "<|reserved_token_11|>",
+    "<|im_media_begin|>",
+    "<|reserved_token_12|>",
+    "<|im_media_end|>",
+    "<|reserved_token_13|>",
+    "<|reserved_token_14|>",
+    "<|im_kimia_text_blank|>",
+    "<|im_kimia_text_eos|>",
+    "<|reserved_token_15|>",
+    "<|reserved_token_16|>",
+    "<|im_kimia_user_msg_start|>",
+    "<|im_kimia_assistant_msg_start|>",
+    "<|reserved_token_17|>",
+    "<|reserved_token_18|>",
+    "<|reserved_token_19|>",
+    "<|im_kimia_speech_ct_id|>",
+    "<|im_kimia_speech_ctd_id|>",
+    "<|reserved_token_20|>",
+    "<|reserved_token_21|>",
+    "<|reserved_token_22|>",
+    "<|reserved_token_23|>",
+    "<|reserved_token_24|>",
+    "<|reserved_token_25|>",
+    "<|reserved_token_26|>",
+    "<|reserved_token_27|>",
+    "<|reserved_token_28|>",
+    "<|reserved_token_29|>",
+    "<|reserved_token_30|>",
+    "<|reserved_token_31|>",
+    "<|reserved_token_32|>",
+    "<|reserved_token_33|>",
+    "<|reserved_token_34|>",
+    "<|reserved_token_35|>",
+    "<|reserved_token_36|>",
+    "<|reserved_token_37|>",
+    "<|reserved_token_38|>",
+    "<|reserved_token_39|>",
+    "<|reserved_token_40|>",
+    "<|reserved_token_41|>",
+    "<|reserved_token_42|>",
+    "<|reserved_token_43|>",
+    "<|reserved_token_44|>",
+    "<|reserved_token_45|>",
+    "<|reserved_token_46|>",
+    "<|reserved_token_47|>",
+    "<|reserved_token_48|>",
+    "<|reserved_token_49|>",
+    "<|reserved_token_50|>",
+    "<|reserved_token_51|>",
+    "<|reserved_token_52|>",
+    "<|reserved_token_53|>",
+    "<|reserved_token_54|>",
+    "<|reserved_token_55|>",
+    "<|reserved_token_56|>",
+    "<|reserved_token_57|>",
+    "<|reserved_token_58|>",
+    "<|reserved_token_59|>",
+    "<|reserved_token_60|>",
+    "<|reserved_token_61|>",
+    "<|reserved_token_62|>",
+    "<|reserved_token_63|>",
+    "<|reserved_token_64|>",
+    "<|reserved_token_65|>",
+    "<|reserved_token_66|>",
+    "<|reserved_token_67|>",
+    "<|reserved_token_68|>",
+    "<|reserved_token_69|>",
+    "<|reserved_token_70|>",
+    "<|reserved_token_71|>",
+    "<|reserved_token_72|>",
+    "<|reserved_token_73|>",
+    "<|reserved_token_74|>",
+    "<|reserved_token_75|>",
+    "<|reserved_token_76|>",
+    "<|reserved_token_77|>",
+    "<|reserved_token_78|>",
+    "<|reserved_token_79|>",
+    "<|reserved_token_80|>",
+    "<|reserved_token_81|>",
+    "<|reserved_token_82|>",
+    "<|reserved_token_83|>",
+    "<|reserved_token_84|>",
+    "<|reserved_token_85|>",
+    "<|reserved_token_86|>",
+    "<|reserved_token_87|>",
+    "<|reserved_token_88|>",
+    "<|reserved_token_89|>",
+    "<|reserved_token_90|>",
+    "<|reserved_token_91|>",
+    "<|reserved_token_92|>",
+    "<|reserved_token_93|>",
+    "<|reserved_token_94|>",
+    "<|reserved_token_95|>",
+    "<|reserved_token_96|>",
+    "<|reserved_token_97|>",
+    "<|reserved_token_98|>",
+    "<|reserved_token_99|>",
+    "<|reserved_token_100|>",
+    "<|reserved_token_101|>",
+    "<|reserved_token_102|>",
+    "<|reserved_token_103|>",
+    "<|reserved_token_104|>",
+    "<|reserved_token_105|>",
+    "<|reserved_token_106|>",
+    "<|reserved_token_107|>",
+    "<|reserved_token_108|>",
+    "<|reserved_token_109|>",
+    "<|reserved_token_110|>",
+    "<|reserved_token_111|>",
+    "<|reserved_token_112|>",
+    "<|reserved_token_113|>",
+    "<|reserved_token_114|>",
+    "<|reserved_token_115|>",
+    "<|reserved_token_116|>",
+    "<|reserved_token_117|>",
+    "<|reserved_token_118|>",
+    "<|reserved_token_119|>",
+    "<|reserved_token_120|>",
+    "<|reserved_token_121|>",
+    "<|reserved_token_122|>",
+    "<|reserved_token_123|>",
+    "<|reserved_token_124|>",
+    "<|reserved_token_125|>",
+    "<|reserved_token_126|>",
+    "<|reserved_token_127|>",
+    "<|reserved_token_128|>",
+    "<|reserved_token_129|>",
+    "<|reserved_token_130|>",
+    "<|reserved_token_131|>",
+    "<|reserved_token_132|>",
+    "<|reserved_token_133|>",
+    "<|reserved_token_134|>",
+    "<|reserved_token_135|>",
+    "<|reserved_token_136|>",
+    "<|reserved_token_137|>",
+    "<|reserved_token_138|>",
+    "<|reserved_token_139|>",
+    "<|reserved_token_140|>",
+    "<|reserved_token_141|>",
+    "<|reserved_token_142|>",
+    "<|reserved_token_143|>",
+    "<|reserved_token_144|>",
+    "<|reserved_token_145|>",
+    "<|reserved_token_146|>",
+    "<|reserved_token_147|>",
+    "<|reserved_token_148|>",
+    "<|reserved_token_149|>",
+    "<|reserved_token_150|>",
+    "<|reserved_token_151|>",
+    "<|reserved_token_152|>",
+    "<|reserved_token_153|>",
+    "<|reserved_token_154|>",
+    "<|reserved_token_155|>",
+    "<|reserved_token_156|>",
+    "<|reserved_token_157|>",
+    "<|reserved_token_158|>",
+    "<|reserved_token_159|>",
+    "<|reserved_token_160|>",
+    "<|reserved_token_161|>",
+    "<|reserved_token_162|>",
+    "<|reserved_token_163|>",
+    "<|reserved_token_164|>",
+    "<|reserved_token_165|>",
+    "<|reserved_token_166|>",
+    "<|reserved_token_167|>",
+    "<|reserved_token_168|>",
+    "<|reserved_token_169|>",
+    "<|reserved_token_170|>",
+    "<|reserved_token_171|>",
+    "<|reserved_token_172|>",
+    "<|reserved_token_173|>",
+    "<|reserved_token_174|>",
+    "<|reserved_token_175|>",
+    "<|reserved_token_176|>",
+    "<|reserved_token_177|>",
+    "<|reserved_token_178|>",
+    "<|reserved_token_179|>",
+    "<|reserved_token_180|>",
+    "<|reserved_token_181|>",
+    "<|reserved_token_182|>",
+    "<|reserved_token_183|>",
+    "<|reserved_token_184|>",
+    "<|reserved_token_185|>",
+    "<|reserved_token_186|>",
+    "<|reserved_token_187|>",
+    "<|reserved_token_188|>",
+    "<|reserved_token_189|>",
+    "<|reserved_token_190|>",
+    "<|reserved_token_191|>",
+    "<|reserved_token_192|>",
+    "<|reserved_token_193|>",
+    "<|reserved_token_194|>",
+    "<|reserved_token_195|>",
+    "<|reserved_token_196|>",
+    "<|reserved_token_197|>",
+    "<|reserved_token_198|>",
+    "<|reserved_token_199|>",
+    "<|reserved_token_200|>",
+    "<|reserved_token_201|>",
+    "<|reserved_token_202|>",
+    "<|reserved_token_203|>",
+    "<|reserved_token_204|>",
+    "<|reserved_token_205|>",
+    "<|reserved_token_206|>",
+    "<|reserved_token_207|>",
+    "<|reserved_token_208|>",
+    "<|reserved_token_209|>",
+    "<|reserved_token_210|>",
+    "<|reserved_token_211|>",
+    "<|reserved_token_212|>",
+    "<|reserved_token_213|>",
+    "<|reserved_token_214|>",
+    "<|reserved_token_215|>",
+    "<|reserved_token_216|>",
+    "<|reserved_token_217|>",
+    "<|reserved_token_218|>",
+    "<|reserved_token_219|>",
+    "<|reserved_token_220|>",
+    "<|reserved_token_221|>",
+    "<|reserved_token_222|>",
+    "<|reserved_token_223|>",
+    "<|reserved_token_224|>",
+    "<|reserved_token_225|>",
+    "<|reserved_token_226|>",
+    "<|reserved_token_227|>",
+    "<|reserved_token_228|>",
+    "<|reserved_token_229|>",
+    "<|reserved_token_230|>",
+    "<|reserved_token_231|>",
+    "<|reserved_token_232|>",
+    "<|reserved_token_233|>",
+    "<|reserved_token_234|>",
+    "<|reserved_token_235|>",
+    "<|reserved_token_236|>",
+    "<|reserved_token_237|>",
+    "<|reserved_token_238|>",
+    "<|reserved_token_239|>",
+    "<|reserved_token_240|>",
+    "<|reserved_token_241|>",
+    "<|reserved_token_242|>",
+    "<|reserved_token_243|>",
+    "<|reserved_token_244|>",
+    "<|reserved_token_245|>",
+    "<|reserved_token_246|>",
+    "<|reserved_token_247|>",
+    "<|reserved_token_248|>",
+    "<|reserved_token_249|>",
+    "<|reserved_token_250|>",
+    "<|reserved_token_251|>",
+    "<|reserved_token_252|>",
+    "<|reserved_token_253|>",
+    "<|reserved_token_254|>",
+    "<|reserved_token_255|>",
+    "<|reserved_token_256|>",
+    "<|reserved_token_257|>",
+    "<|reserved_token_258|>",
+    "<|reserved_token_259|>",
+    "<|reserved_token_260|>",
+    "<|reserved_token_261|>",
+    "<|reserved_token_262|>",
+    "<|reserved_token_263|>",
+    "<|reserved_token_264|>",
+    "<|reserved_token_265|>",
+    "<|reserved_token_266|>",
+    "<|reserved_token_267|>",
+    "<|reserved_token_268|>",
+    "<|reserved_token_269|>",
+    "<|reserved_token_270|>",
+    "<|reserved_token_271|>",
+    "<|reserved_token_272|>",
+    "<|reserved_token_273|>",
+    "<|reserved_token_274|>",
+    "<|reserved_token_275|>",
+    "<|reserved_token_276|>",
+    "<|reserved_token_277|>",
+    "<|reserved_token_278|>",
+    "<|reserved_token_279|>",
+    "<|reserved_token_280|>",
+    "<|reserved_token_281|>",
+    "<|reserved_token_282|>",
+    "<|reserved_token_283|>",
+    "<|reserved_token_284|>",
+    "<|reserved_token_285|>",
+    "<|reserved_token_286|>",
+    "<|reserved_token_287|>",
+    "<|reserved_token_288|>",
+    "<|reserved_token_289|>",
+    "<|reserved_token_290|>",
+    "<|reserved_token_291|>",
+    "<|reserved_token_292|>",
+    "<|reserved_token_293|>",
+    "<|reserved_token_294|>",
+    "<|reserved_token_295|>",
+    "<|reserved_token_296|>",
+    "<|reserved_token_297|>",
+    "<|reserved_token_298|>",
+    "<|reserved_token_299|>",
+    "<|reserved_token_300|>",
+    "<|reserved_token_301|>",
+    "<|reserved_token_302|>",
+    "<|reserved_token_303|>",
+    "<|reserved_token_304|>",
+    "<|reserved_token_305|>",
+    "<|reserved_token_306|>",
+    "<|reserved_token_307|>",
+    "<|reserved_token_308|>",
+    "<|reserved_token_309|>",
+    "<|reserved_token_310|>",
+    "<|reserved_token_311|>",
+    "<|reserved_token_312|>",
+    "<|reserved_token_313|>",
+    "<|reserved_token_314|>",
+    "<|reserved_token_315|>",
+    "<|reserved_token_316|>",
+    "<|reserved_token_317|>",
+    "<|reserved_token_318|>",
+    "<|reserved_token_319|>",
+    "<|reserved_token_320|>",
+    "<|reserved_token_321|>",
+    "<|reserved_token_322|>",
+    "<|reserved_token_323|>",
+    "<|reserved_token_324|>",
+    "<|reserved_token_325|>",
+    "<|reserved_token_326|>",
+    "<|reserved_token_327|>",
+    "<|reserved_token_328|>",
+    "<|reserved_token_329|>",
+    "<|reserved_token_330|>",
+    "<|reserved_token_331|>",
+    "<|reserved_token_332|>",
+    "<|reserved_token_333|>",
+    "<|reserved_token_334|>",
+    "<|reserved_token_335|>",
+    "<|reserved_token_336|>",
+    "<|reserved_token_337|>",
+    "<|reserved_token_338|>",
+    "<|reserved_token_339|>",
+    "<|reserved_token_340|>",
+    "<|reserved_token_341|>",
+    "<|reserved_token_342|>",
+    "<|reserved_token_343|>",
+    "<|reserved_token_344|>",
+    "<|reserved_token_345|>",
+    "<|reserved_token_346|>",
+    "<|reserved_token_347|>",
+    "<|reserved_token_348|>",
+    "<|reserved_token_349|>",
+    "<|reserved_token_350|>",
+    "<|reserved_token_351|>",
+    "<|reserved_token_352|>",
+    "<|reserved_token_353|>",
+    "<|reserved_token_354|>",
+    "<|reserved_token_355|>",
+    "<|reserved_token_356|>",
+    "<|reserved_token_357|>",
+    "<|reserved_token_358|>",
+    "<|reserved_token_359|>",
+    "<|reserved_token_360|>",
+    "<|reserved_token_361|>",
+    "<|reserved_token_362|>",
+    "<|reserved_token_363|>",
+    "<|reserved_token_364|>",
+    "<|reserved_token_365|>",
+    "<|reserved_token_366|>",
+    "<|reserved_token_367|>",
+    "<|reserved_token_368|>",
+    "<|reserved_token_369|>",
+    "<|reserved_token_370|>",
+    "<|reserved_token_371|>",
+    "<|reserved_token_372|>",
+    "<|reserved_token_373|>",
+    "<|reserved_token_374|>",
+    "<|reserved_token_375|>",
+    "<|reserved_token_376|>",
+    "<|reserved_token_377|>",
+    "<|reserved_token_378|>",
+    "<|reserved_token_379|>",
+    "<|reserved_token_380|>",
+    "<|reserved_token_381|>",
+    "<|reserved_token_382|>",
+    "<|reserved_token_383|>",
+    "<|reserved_token_384|>",
+    "<|reserved_token_385|>",
+    "<|reserved_token_386|>",
+    "<|reserved_token_387|>",
+    "<|reserved_token_388|>",
+    "<|reserved_token_389|>",
+    "<|reserved_token_390|>",
+    "<|reserved_token_391|>",
+    "<|reserved_token_392|>",
+    "<|reserved_token_393|>",
+    "<|reserved_token_394|>",
+    "<|reserved_token_395|>",
+    "<|reserved_token_396|>",
+    "<|reserved_token_397|>",
+    "<|reserved_token_398|>",
+    "<|reserved_token_399|>",
+    "<|reserved_token_400|>",
+    "<|reserved_token_401|>",
+    "<|reserved_token_402|>",
+    "<|reserved_token_403|>",
+    "<|reserved_token_404|>"
+  ],
+  "bos_token": "[BOS]",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "[EOS]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|reserved_token_406|>",
+  "pat_str": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "tokenizer_class": "TikTokenTokenizer",
+  "unk_token": "<|reserved_token_405|>",
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_echochat.TikTokenTokenizer",
+      null
+    ]
+  }
+}