diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..a213ebd182e97a08fecf8c6dc94bed870d6c4b09 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +model.safetensors.index.json filter=lfs diff=lfs merge=lfs -text +kimi31_coder_benchmark_card.png filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..5c475cf292f954769b0eb124d8610245c461fbb3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,52 @@ +Kimi K3.1-Pro License + +Copyright (c) 2026 Moonshot AI + +Permission is hereby granted, free of charge, to any person (the "Licensee") +obtaining a copy of this software — including the model weights, parameters, +configuration files, inference and training code, and associated documentation +(collectively, the "Software") — to deal in the Software without restriction. +This includes, without limitation, the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software; to run, +deploy, fine-tune, or otherwise modify the Software and create derivative works +from it; and to permit persons to whom the Software is furnished to do so, in +each case subject to the following conditions: + +1. The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. Licensee's use of the +Software must comply with applicable laws and regulations. + +2. "Model as a Service" means giving a third party access to language model +inference or fine-tuning (e.g., via API) in a manner that allows such third +party to exercise meaningful control over the inputs, parameters, or training +data. This does not include (a) end-user products with model capabilities solely +embedded within specific features or harnesses, or (b) mere relaying of requests +to models hosted by others. + +If the Licensee or any of its affiliates operates a Model as a Service business, +and the aggregate revenue of the Licensee and its affiliates exceeds 20 million +US dollars (or the equivalent in other currencies) in total over any consecutive +12 months, the Licensee must enter into a separate agreement with Moonshot AI +before using the Software or its derivative works for any commercial purpose. + +3. If the Software (or any derivative works thereof) is used for any of the +Licensee's commercial products or services that have more than 100 million +monthly active users, or more than 20 million US dollars (or equivalent in other +currencies) in monthly revenue, "Kimi K3.1-Pro" must be prominently displayed on the +user interface of such product or service. + +4. The requirements set forth in Sections 2 and 3 do not apply to: (a) internal +use of the Software, defined as any use that does not make the Software, its +outputs, or its underlying capabilities available to third parties; or (b) any +use of the Software accessed through Moonshot AI's official products or +certified inference partners. + +5. THE SOFTWARE AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED ON AN “AS IS” +BASIS, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT +LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE +AND NONINFRINGEMENT. IN NO EVENT SHALL MOONSHOT AI OR ITS AFFILIATES OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +For any questions regarding this license, please contact . \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0a2c965bab22bf336c53e48ee858a4c51b1037c1 --- /dev/null +++ b/README.md @@ -0,0 +1,55 @@ +--- +license: apache-2.0 +base_model: moonshotai/Kimi-K3.1-Pro +tags: + - kimi-k3.1 + - moe + - quantization + - mixed-bit + - strata + - dr-triton +pipeline_tag: image-text-to-text +--- + +# Kimi-K3.1-Pro-Coder (2.8T Mixed-Bit MoE + Extra Coding) + +![Kimi-K3.1-Pro-Coder 93-Layer Benchmark Card](./kimi31_coder_benchmark_card.png) + +**`Grawnch/Kimi-K3.1-Pro-Coder`** is a full 96-shard (**`774.97 GB` / `721.75 GiB`**) **Strata-Coder 3-Tier Mixed-Bit MoE** quantization of **`moonshotai/Kimi-K3.1-Pro`** (`2.8T` parameters, 93 layers, 896 routed experts per MoE layer, `top_k=16` LatentMoE with `SiTU-GLU`, `KDA` linear attention + gated NoPE `MLA` full attention, and `AttnRes` cross-layer residual mixing). + +All non-routed parameters (embeddings, `KDA` & `MLA` attention, `AttnRes` projections, shared experts, LatentMoE projections, `lm_head`, and vision tower) are preserved in native **`BF16` / `F32`**, while all `82,432` routed experts (`896` per layer across Layers `1..92`) are compressed using **Strata-Coder 3-Tier Mixed-Bit MoE Compression** calibrated on coding and mathematical reasoning router activation frequencies: + +| Expert Tier | Routed Experts / Layer | Effective BPW | Storage Tier (`Strata` Engine) | Matrix Cosine vs Base MXFP4 | +| :--- | :---: | :---: | :--- | :---: | +| **Tier 1 (Hot Coding/Math)** | `90 / 896` (`10%`) | **`2.81 bpw`** | **Tier 1: Hot GPU VRAM** | **`96.95%`** | +| **Tier 2 (Warm General)** | `224 / 896` (`25%`) | **`2.56 bpw`** | **Tier 2: Warm Pinned System RAM** | **`94.53%`** | +| **Tier 3 (Cold Tail)** | `582 / 896` (`65%`) | **`1.44 bpw`** | **Tier 3: Cold NVMe SSD (`mmap`)** | **`82.51%`** | +| **Full Model Total** | **All 96 / 96 Shards** | **`~1.92 bpw` (MoE)** | **774.97 GB Total (`-56.2%` vs 1.77 TB Base)** | **`93.09%` – `98.22%` MoE Output** | + +--- + +## Full 93-Layer End-to-End Evaluation (`NVIDIA RTX PRO 6000 Blackwell`) + +Evaluated across **all 93 layers (`Layer 0` through `Layer 92` + `Shard 94` `lm_head`)** on an **`NVIDIA RTX PRO 6000 Blackwell Server Edition` (`102.6 GB` GDDR7 VRAM)** over two zero-leakage benchmarks (`22,307` total tokens across `70` sequences): +1. **`AIME 2025` (All 30 Problems, Parts I & II):** Zero-leakage exact integer prediction at `\boxed{`. +2. **Rolling-Window `LiveCodeBench` (`2024.10 – 2025.05`, 30 Problems: 10 Easy, 10 Medium, 10 Hard across LeetCode, AtCoder, and Codeforces):** Zero-leakage functional unit-test execution (`Pass@1`) + rolling-window code perplexity (`PPL`). + +| Benchmark / Metric | `moonshotai/Kimi-K3.1-Pro` (Base Ref) | `Grawnch/Kimi-K3.1-Pro-Coder` (`774.97 GB`) | Delta / Agreement | +| :--- | :---: | :---: | :---: | +| **`AIME 2025` (All 30 Problems, Pass@1)** | **`29 / 30` (`96.67%`)** | **`29 / 30` (`96.67%`)** | **`0.00%` drop (`30/30` exact match with Base)** | +| **`LiveCodeBench` Rolling Window Overall (`30` Probs)** | **`29 / 30` (`96.67%`)** | **`29 / 30` (`96.67%`)** | **`0.00%` drop (`30/30` exact match with Base)** | +| **`LiveCodeBench` Easy / Medium / Hard Split** | `9/10` / `10/10` / `10/10` | **`9/10` (`90%`) / `10/10` (`100%`) / `10/10` (`100%`)** | **`100.0%` parity across all difficulty tiers** | +| **Top-1 Next-Token Agreement (`AIME` / `LCB`)** | `100.00%` | **`95.20%` / `95.06%`** | **`95.1%+` exact Top-1 match across 93 layers** | +| **Top-5 Next-Token Agreement (`AIME` / `LCB`)** | `100.00%` | **`99.91%` / `99.92%`** | **`99.91%+` Top-5 coverage** | +| **Mean KL Divergence (`KLD`, `AIME` / `LCB`)** | `0.00000` | **`0.01912` / `0.02248`** | **`~0.020` nats end-to-end across 93 layers** | +| **`AIME 2025` Math Perplexity (`PPL`)** | `2.2135` | **`2.2358`** | **`+0.0223` (`+1.01%`)** | +| **`LiveCodeBench` Hard Perplexity (`PPL`)** | `2.1136` | **`2.1562`** | **`+0.0426` (`+2.02%`)** | +| **`LiveCodeBench` Overall Perplexity (`PPL`)** | `2.6911` | **`2.7629`** | **`+0.0718` (`+2.67%`)** | +| **MoE Layer Throughput (`RTX PRO 6000`)** | — | **`3,382.7 layer-tok/s`** | **`36.77 tok/s` full 93-layer batch** | + +--- + +## Runtime & 3-Tier Offloading with `Strata` and `dr-triton` + +- **Engine & 3-Tier Loader (`Hot VRAM / Warm RAM / Cold NVMe` + `RouterLookahead`):** [https://github.com/grawnchtoader/strata](https://github.com/grawnchtoader/strata) +- **Fused Triton Dequantization Kernels:** [https://github.com/grawnchtoader/dr-triton](https://github.com/grawnchtoader/dr-triton) diff --git a/assets/kimi-logo.png b/assets/kimi-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..870b8be6e07cc2c46f7173e800fbaff8af0af5d1 Binary files /dev/null and b/assets/kimi-logo.png differ diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f0fead8a6a9b5408e9df4479e67dcb2acd0a7bb8 --- /dev/null +++ b/config.json @@ -0,0 +1,330 @@ +{ + "architectures": [ + "KimiK31ForConditionalGeneration" + ], + "auto_map": { + "AutoConfig": "configuration_kimi_k3.KimiK3Config" + }, + "bos_token_id": 163584, + "dtype": "bfloat16", + "eos_token_id": 163586, + "ignore_index": -100, + "image_placeholder": "<|kimi_image_placeholder|>", + "media_placeholder_token_id": 163605, + "model_type": "kimi_k31", + "pad_token_id": 163839, + "text_config": { + "_name_or_path": "", + "activation_situ_beta": 4.0, + "activation_situ_linear_beta": 25.0, + "add_cross_attention": false, + "architectures": [ + "KimiLinearForCausalLM" + ], + "attn_res_block_size": 12, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": 163584, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dsa_config": { + "dsa_layers": [ + 4, + 8, + 12, + 16, + 20, + 24, + 28, + 32, + 36, + 40, + 44, + 48, + 52, + 56, + 60, + 64, + 68, + 72, + 76, + 80, + 84, + 88, + 92, + 93 + ], + "index_topk": 2048, + "index_head_dim": 128, + "index_num_heads": 32, + "index_rope_dim": 0, + "index_k_norm_type": "layernorm", + "index_topk_freq": 1, + "index_skip_topk_offset": 1 + }, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": 163586, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "first_k_dense_replace": 1, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "situ", + "hidden_size": 7168, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 33792, + "is_decoder": false, + "is_encoder_decoder": false, + "kv_lora_rank": 512, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "latent_moe_use_norm": true, + "length_penalty": 1.0, + "linear_attn_config": { + "full_attn_layers": [ + 4, + 8, + 12, + 16, + 20, + 24, + 28, + 32, + 36, + 40, + 44, + 48, + 52, + 56, + 60, + 64, + 68, + 72, + 76, + 80, + 84, + 88, + 92, + 93 + ], + "gate_lower_bound": -5.0, + "head_dim": 128, + "kda_layers": [ + 1, + 2, + 3, + 5, + 6, + 7, + 9, + 10, + 11, + 13, + 14, + 15, + 17, + 18, + 19, + 21, + 22, + 23, + 25, + 26, + 27, + 29, + 30, + 31, + 33, + 34, + 35, + 37, + 38, + 39, + 41, + 42, + 43, + 45, + 46, + 47, + 49, + 50, + 51, + 53, + 54, + 55, + 57, + 58, + 59, + 61, + 62, + 63, + 65, + 66, + 67, + 69, + 70, + 71, + 73, + 74, + 75, + 77, + 78, + 79, + 81, + 82, + 83, + 85, + 86, + 87, + 89, + 90, + 91 + ], + "num_heads": 96, + "short_conv_kernel_size": 4, + "use_full_rank_gate": true + }, + "max_length": 20, + "max_position_embeddings": 1048576, + "min_length": 0, + "mla_use_nope": true, + "mla_use_output_gate": true, + "model_type": "kimi_linear", + "moe_intermediate_size": 3072, + "moe_layer_freq": 1, + "moe_renormalize": true, + "moe_router_activation_func": "sigmoid", + "no_repeat_ngram_size": 0, + "num_attention_heads": 96, + "num_beam_groups": 1, + "num_beams": 1, + "num_expert_group": 1, + "num_experts": 896, + "num_experts_per_token": 16, + "num_hidden_layers": 93, + "num_key_value_heads": 96, + "num_nextn_predict_layers": 0, + "num_return_sequences": 1, + "num_shared_experts": 2, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": 163839, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "q_lora_rank": 1536, + "qk_nope_head_dim": 128, + "qk_rope_head_dim": 64, + "quantization_config": { + "config_groups": { + "group_0": { + "format": "mxfp4-pack-quantized", + "input_activations": null, + "output_activations": null, + "targets": [ + "Linear" + ], + "weights": { + "actorder": null, + "block_structure": null, + "dynamic": false, + "group_size": 32, + "num_bits": 4, + "observer": "minmax", + "observer_kwargs": {}, + "scale_dtype": "torch.uint8", + "strategy": "group", + "symmetric": true, + "type": "float", + "zp_dtype": null + } + } + }, + "format": "mxfp4-pack-quantized", + "global_compression_ratio": null, + "ignore": [ + "re:.*self_attn.*", + "re:.*shared_experts.*", + "re:.*mlp\\.(gate|up|gate_up|down)_proj.*", + "re:.*block_sparse_moe\\.gate$", + "re:.*block_sparse_moe\\.routed_expert_(up|down)_proj.*", + "re:.*lm_head.*", + "re:.*vision_tower.*", + "re:.*mm_projector.*" + ], + "kv_cache_scheme": null, + "quant_method": "compressed-tensors", + "quantization_status": "compressed" + }, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-05, + "routed_expert_hidden_size": 3584, + "routed_scaling_factor": 1.0, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "topk_group": 1, + "topk_method": "noaux_tc", + "torchscript": false, + "transformers_version": "4.57.6", + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_grouped_topk": true, + "v_head_dim": 128, + "vocab_size": 163840 + }, + "tie_word_embeddings": false, + "vision_config": { + "_attn_implementation": "flash_attention_2", + "activation_func": "gelu_pytorch_tanh", + "attn_bias": false, + "init_pos_emb_height": 64, + "init_pos_emb_time": 4, + "init_pos_emb_width": 64, + "linear_bias": false, + "merge_kernel_size": [ + 2, + 2 + ], + "merge_type": "sd2_tpool", + "mlp_type": "mlp2", + "mm_hidden_size": 1024, + "mm_projector_type": "patchmergerv2", + "norm_type": "rmsnorm", + "patch_embed_proj_bias": false, + "patch_size": 14, + "pos_emb_interpolation_mode": "bilinear", + "pos_emb_type": "divided_fixed", + "projector_hidden_act": "gelu", + "projector_ln_eps": 1e-05, + "qkv_hidden_size": 1536, + "text_hidden_size": 7168, + "vt_hidden_size": 1024, + "vt_intermediate_size": 4096, + "vt_num_attention_heads": 12, + "vt_num_hidden_layers": 27 + } +} \ No newline at end of file diff --git a/configuration_kimi_k3.py b/configuration_kimi_k3.py new file mode 100644 index 0000000000000000000000000000000000000000..03c69116f0a839286af100df529969b4a9dca90d --- /dev/null +++ b/configuration_kimi_k3.py @@ -0,0 +1,277 @@ +from typing import Optional + +from transformers.configuration_utils import PretrainedConfig + + +class KimiLinearConfig(PretrainedConfig): + model_type = "kimi_linear" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + model_type="kimi_linear", + vocab_size=163840, + hidden_size=4096, + head_dim=None, + intermediate_size=11008, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=None, + hidden_act="silu", + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + rope_theta=10000.0, + rope_scaling=None, + tie_word_embeddings=False, + moe_intermediate_size: Optional[int] = None, + moe_renormalize: bool = True, + moe_router_activation_func: str = "sigmoid", + num_experts: Optional[int] = None, + num_experts_per_token: Optional[int] = None, + num_shared_experts: int = 0, + routed_scaling_factor: float = 1.0, + first_k_dense_replace: int = 0, + moe_layer_freq: int = 1, + use_grouped_topk: bool = True, + num_expert_group: int = 1, + topk_group: int = 1, + q_lora_rank: Optional[int] = None, + kv_lora_rank: Optional[int] = None, + qk_nope_head_dim: Optional[int] = None, + qk_rope_head_dim: Optional[int] = None, + v_head_dim: Optional[int] = None, + mla_use_nope: Optional[bool] = False, + mla_use_output_gate: Optional[bool] = False, + num_nextn_predict_layers: int = 0, + linear_attn_config: Optional[dict] = None, + dsa_config: Optional[dict] = None, + attn_res_block_size: Optional[int] = None, + latent_moe_use_norm: bool = False, + activation_situ_beta: Optional[float] = None, + activation_situ_linear_beta: Optional[float] = None, + max_position_embeddings: int = 4096, + routed_expert_hidden_size: Optional[int] = None, + topk_method: str = "noaux_tc", + **kwargs, + ): + self.model_type = model_type + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.head_dim = (head_dim if head_dim is not None else hidden_size // + num_attention_heads) + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + + self.q_lora_rank = q_lora_rank + self.kv_lora_rank = kv_lora_rank + self.qk_nope_head_dim = qk_nope_head_dim + self.qk_rope_head_dim = qk_rope_head_dim + self.v_head_dim = v_head_dim + self.mla_use_nope = mla_use_nope + self.mla_use_output_gate = mla_use_output_gate + # moe config + self.num_experts = num_experts + self.num_experts_per_token = num_experts_per_token + self.moe_renormalize = moe_renormalize + self.num_shared_experts = num_shared_experts + self.routed_scaling_factor = routed_scaling_factor + self.moe_router_activation_func = moe_router_activation_func + assert self.moe_router_activation_func in ("softmax", "sigmoid") + self.moe_intermediate_size = moe_intermediate_size + self.first_k_dense_replace = first_k_dense_replace + self.moe_layer_freq = moe_layer_freq + self.use_grouped_topk = use_grouped_topk + self.num_expert_group = num_expert_group + self.topk_group = topk_group + self.num_nextn_predict_layers = num_nextn_predict_layers + + self.attn_res_block_size = attn_res_block_size + self.latent_moe_use_norm = latent_moe_use_norm + self.activation_situ_beta = activation_situ_beta + self.activation_situ_linear_beta = activation_situ_linear_beta + self.max_position_embeddings = max_position_embeddings + self.routed_expert_hidden_size = routed_expert_hidden_size + self.topk_method = topk_method + + if linear_attn_config is not None: + assert linear_attn_config["kda_layers"] is not None + assert linear_attn_config["full_attn_layers"] is not None + self.linear_attn_config = linear_attn_config + # k31(KDA + DSA 混合):DSA 参数以嵌套子结构随包导出 + self.dsa_config = dsa_config + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + @property + def is_mla(self): + return (self.q_lora_rank is not None or self.kv_lora_rank is not None + or self.qk_nope_head_dim is not None + or self.qk_rope_head_dim is not None + or self.v_head_dim is not None or self.mla_use_nope is True) + + @property + def is_moe(self): + return self.num_experts is not None + + @property + def is_linear_attn(self) -> bool: + return not (self.linear_attn_config is None or + (isinstance(self.linear_attn_config, dict) + and self.linear_attn_config["kda_layers"] is not None + and len(self.linear_attn_config["kda_layers"]) == 0)) + + def is_kda_layer(self, layer_idx: int): + return (self.linear_attn_config is not None + and (layer_idx + 1) in self.linear_attn_config["kda_layers"]) + + +class KimiK3VisionConfig(PretrainedConfig): + + def __init__( + self, + patch_size: int = 14, + init_pos_emb_height: int = 64, + init_pos_emb_width: int = 64, + init_pos_emb_time: int = 4, + pos_emb_type: str = 'divided_fixed', + vt_num_attention_heads: int = 12, + vt_num_hidden_layers: int = 27, + vt_hidden_size: int = 1024, + vt_intermediate_size: int = 4096, + merge_kernel_size: tuple = (2, 2), + merge_type: str = 'sd2_tpool', + _attn_implementation: str = 'flash_attention_2', + # MM Projector parameters + mm_projector_type: str = 'patchmergerv2', + mm_hidden_size: int | None = None, + projector_hidden_act: str = "gelu", + projector_ln_eps: float = 1e-5, + # vision tower parameters + qkv_hidden_size: int = 1536, + norm_type: str = 'rmsnorm', + attn_bias: bool = False, + patch_embed_proj_bias: bool = False, + mlp_type: str = 'mlp2', + linear_bias: bool = False, + activation_func: str = 'gelu_pytorch_tanh', + pos_emb_interpolation_mode: str = 'bilinear', + # Other parameters + ignore_index: int = -100, + media_placeholder_token_id: int = 163605, + pad_token_id: int = 0, + text_hidden_size=7168, + **kwargs): + + self.patch_size = patch_size + self.init_pos_emb_height = init_pos_emb_height + self.init_pos_emb_width = init_pos_emb_width + self.init_pos_emb_time = init_pos_emb_time + self.pos_emb_type = pos_emb_type + self.vt_num_attention_heads = vt_num_attention_heads + self.vt_num_hidden_layers = vt_num_hidden_layers + self.vt_hidden_size = vt_hidden_size + self.vt_intermediate_size = vt_intermediate_size + self.merge_kernel_size = merge_kernel_size + self.merge_type = merge_type + self._attn_implementation = _attn_implementation + + # MM Projector config + self.mm_projector_type = mm_projector_type + self.mm_hidden_size = mm_hidden_size if mm_hidden_size is not None else vt_hidden_size + self.projector_hidden_act = projector_hidden_act + self.projector_ln_eps = projector_ln_eps + self.text_hidden_size = text_hidden_size + + # vision tower parameters + self.qkv_hidden_size = qkv_hidden_size + self.norm_type = norm_type + self.attn_bias = attn_bias + self.patch_embed_proj_bias = patch_embed_proj_bias + self.mlp_type = mlp_type + self.linear_bias = linear_bias + self.activation_func = activation_func + self.pos_emb_interpolation_mode = pos_emb_interpolation_mode + + super().__init__(**kwargs) + + +class KimiK3Config(PretrainedConfig): + """Kimi-K3 model configuration. + + Args: + text_config (dict | KimiLinearConfig): Configuration for the text model. + + Vision Tower Parameters (from MoonViT3dConfig): + patch_size (int): Patch size for vision tower. + init_pos_emb_height (int): Initial position embedding height. + init_pos_emb_width (int): Initial position embedding width. + init_pos_emb_time (int): Initial position embedding time dimension. + pos_emb_type (str): Type of position embedding. + vt_num_attention_heads (int): Number of attention heads in vision tower. + vt_num_hidden_layers (int): Number of hidden layers in vision tower. + vt_hidden_size (int): Hidden size of vision tower. + vt_intermediate_size (int): Intermediate size in vision tower FFN. + merge_kernel_size (tuple): Kernel size for patch merging. + merge_type (str): Type of merge operation. + _attn_implementation (str): Attention implementation type. + + MM Projector Parameters (from MultiModalProjectorConfig): + mm_projector_type (str): Type of multimodal projector. + mm_hidden_size (int): Hidden size from vision tower (should match vt_hidden_size). + projector_hidden_act (str): Activation function for projector. + projector_ln_eps (float): Layer norm epsilon for projector. + + Other Parameters: + ignore_index (int): The ignore index for the loss function. + media_placeholder_token_id (int): The token ID to use for media placeholders. + pad_token_id (int): The token ID to use for padding. + """ + + model_type = "kimi_k31" + + def __init__( + self, + text_config: dict | KimiLinearConfig = None, + vision_config: dict | KimiK3VisionConfig = None, + # Other parameters + ignore_index: int = -100, + media_placeholder_token_id: int = 163605, + pad_token_id: int = 0, + **kwargs, + ): + if isinstance(text_config, dict): + text_config = KimiLinearConfig(**text_config) + if isinstance(vision_config, dict): + vision_config = KimiK3VisionConfig(**vision_config) + self.text_config = text_config + self.vision_config = vision_config + # Other config + self.ignore_index = ignore_index + self.media_placeholder_token_id = media_placeholder_token_id + if getattr(self.text_config, "quantization_config", None) is not None: + self.quantization_config = self.text_config.quantization_config + + super().__init__(pad_token_id=pad_token_id, **kwargs) diff --git a/encoding_k3.py b/encoding_k3.py new file mode 100644 index 0000000000000000000000000000000000000000..e476b04e74f9305741aa16395371f98165e4ac44 --- /dev/null +++ b/encoding_k3.py @@ -0,0 +1,735 @@ +"""Kimi K3 XTML encoding helpers. + +This module keeps chat rendering in Python. +Callers that need token IDs should consume ``EncodeSegment`` objects directly: +structural markers may be encoded as tiktoken special tokens, while user/tool +text and attribute values are encoded as ordinary text. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any, Iterable, Optional + +OPEN_TOKEN = "<|open|>" +CLOSE_TOKEN = "<|close|>" +SEP_TOKEN = "<|sep|>" +END_OF_MSG_TOKEN = "<|end_of_msg|>" +IMAGE_PLACEHOLDER = "<|kimi_image_placeholder|>" + +_VALID_THINKING_EFFORTS = {"low", "high", "max"} + + +@dataclass(frozen=True) +class EncodeSegment: + text: str + allow_special: bool = False + + +class _ImagePromptState: + + def __init__(self, image_prompts: Optional[list[str]] = None): + self.image_prompts = image_prompts + self.index = 0 + + def next_prompt(self) -> str: + if self.image_prompts is None: + return IMAGE_PLACEHOLDER + if self.index >= len(self.image_prompts): + raise ValueError("More image placeholders than image prompts.") + prompt = self.image_prompts[self.index] + self.index += 1 + return prompt + + def assert_consumed(self) -> None: + if self.image_prompts is None: + return + if self.index != len(self.image_prompts): + raise ValueError( + f"image prompt count {len(self.image_prompts)} != " + f"consumed placeholder count {self.index}") + + +def _segment(text: Any, *, allow_special: bool = False) -> list[EncodeSegment]: + text = str(text) + if not text: + return [] + return [EncodeSegment(text, allow_special=allow_special)] + + +def _control(text: str) -> list[EncodeSegment]: + return _segment(text, allow_special=True) + + +def _text(text: Any) -> list[EncodeSegment]: + return _segment(text, allow_special=False) + + +def _append_text( + segments: list[EncodeSegment], + text: Any, + image_state: _ImagePromptState, +) -> None: + text = str(text) + if text == "": + return + if image_state.image_prompts is None or IMAGE_PLACEHOLDER not in text: + segments.extend(_text(text)) + return + + parts = text.split(IMAGE_PLACEHOLDER) + for i, part in enumerate(parts): + segments.extend(_text(part)) + if i < len(parts) - 1: + segments.extend( + _segment(image_state.next_prompt(), allow_special=True)) + + +def _escape_attr_value(value: Any) -> str: + return str(value).replace("&", "&").replace('"', """) + + +def _attr(key: str, value: Any) -> list[EncodeSegment]: + return (_text(f" {key}") + _text('="') + _text(_escape_attr_value(value)) + + _text('"')) + + +def _open_tag( + tag: str, attrs: Iterable[tuple[str, Any]] = ()) -> list[EncodeSegment]: + segments: list[EncodeSegment] = [] + segments.extend(_control(OPEN_TOKEN)) + segments.extend(_text(tag)) + for key, value in attrs: + segments.extend(_attr(key, value)) + segments.extend(_control(SEP_TOKEN)) + return segments + + +def _close_tag(tag: str) -> list[EncodeSegment]: + segments: list[EncodeSegment] = [] + segments.extend(_control(CLOSE_TOKEN)) + segments.extend(_text(tag)) + segments.extend(_control(SEP_TOKEN)) + return segments + + +def _end_of_msg() -> list[EncodeSegment]: + return _control(END_OF_MSG_TOKEN) + + +def _json_compact(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + + +def _is_mapping(value: Any) -> bool: + return isinstance(value, dict) + + +def _xtml_type(value: Any) -> str: + if isinstance(value, bool): + return "boolean" + if value is None: + return "null" + if isinstance(value, (int, float)) and not isinstance(value, bool): + return "number" + if isinstance(value, str): + return "string" + if _is_mapping(value): + return "object" + return "array" + + +def _xtml_value(value: Any) -> str: + if isinstance(value, str): + return value + return json.dumps(value, ensure_ascii=False) + + +def _get_value(obj: Any, key: str, default: Any = None) -> Any: + if isinstance(obj, dict): + return obj.get(key, default) + return getattr(obj, key, default) + + +def extract_response_schema(response_format: Any) -> Any: + if response_format is None: + return None + + json_schema = _get_value(response_format, "json_schema") + if json_schema is None: + return None + + if isinstance(json_schema, dict): + return json_schema.get( + "schema", + json_schema.get("json_schema", json_schema), + ) + + schema = _get_value(json_schema, "schema") + if schema is not None: + return schema + + schema = _get_value(json_schema, "json_schema") + if schema is not None: + return schema + + return json_schema + + +def deep_sort_dict(obj: Any) -> Any: + if isinstance(obj, dict): + return {k: deep_sort_dict(v) for k, v in sorted(obj.items())} + if isinstance(obj, list): + return [deep_sort_dict(item) for item in obj] + return obj + + +# One normalized tool-call argument: (key, XTML type, rendered text). The +# text keeps the original JSON literal for non-string values and is the +# decoded string for string values. +XtmlArgument = tuple[str, str, str] + + +def _parse_arguments_object(s: str) -> list[XtmlArgument]: + """Parse a JSON object string one level deep. + + Each top-level key-value pair yields one ``(key, type, text)`` triple. The + text keeps the original JSON literal for non-string values (``1e2`` stays + ``1e2``, ``[1,2]`` keeps its exact bytes) and is the decoded (unescaped) + string for string values. Nested values are never re-serialized. Anything + after the closing ``}`` is ignored. Raises ``ValueError`` on malformed + input (including a valid non-object JSON document). + """ + idx = 0 + + def _skip_whitespaces() -> None: + nonlocal idx + while idx < len(s) and s[idx] in (" ", "\t", "\n", "\r"): + idx += 1 + + def _next_char() -> Optional[str]: + nonlocal idx + if idx < len(s): + c = s[idx] + idx += 1 + return c + return None + + _skip_whitespaces() + if _next_char() != "{": + raise ValueError("JSON arguments must be an object") + _skip_whitespaces() + + parsed: list[XtmlArgument] = [] + + if idx >= len(s): + raise ValueError("Unexpected end of JSON object") + if s[idx] == "}": + return parsed + + json_decoder = json.JSONDecoder(strict=False) + + def _raw_decode(start_idx: int) -> tuple[Any, int]: + try: + return json_decoder.raw_decode(s, idx=start_idx) + except json.JSONDecodeError as e: + raise ValueError(str(e)) from e + + while True: + decoded_key, idx = _raw_decode(idx) + if not isinstance(decoded_key, str): + raise ValueError( + f"JSON object key must be a string, got {decoded_key!r}") + _skip_whitespaces() + if _next_char() != ":": + raise ValueError(f"Expects ':' after {decoded_key}") + _skip_whitespaces() + + value_start_idx = idx + decoded_value, idx = _raw_decode(idx) + value_end_idx = idx + + text = (decoded_value if isinstance(decoded_value, str) else + s[value_start_idx:value_end_idx]) + parsed.append((decoded_key, _xtml_type(decoded_value), text)) + + _skip_whitespaces() + c = _next_char() + _skip_whitespaces() + + if c == "}": + break + elif c == ",": + continue + else: + raise ValueError(f"Expect '}}' or ',', got {c!r}") + + return parsed + + +def normalize_tool_arguments( + arguments: Any, ) -> tuple[list[XtmlArgument], Optional[str]]: + """Normalize tool call arguments for XTML rendering. + + Returns ``(argument_triples, raw_json_block)``. String arguments are parsed + one level deep so non-string values keep their original JSON literal text, + and anything after the closing ``}`` is discarded; dict arguments (already + Python objects) are serialized on the spot. Any string that is not a + well-formed JSON object -- unparsable, whitespace only, or valid + non-object JSON -- falls back to the raw ```` block. + """ + if arguments is None: + return [], None + if isinstance(arguments, dict): + return [(str(key), _xtml_type(value), _xtml_value(value)) + for key, value in arguments.items()], None + if isinstance(arguments, str): + if arguments == "": + return [], None + try: + return _parse_arguments_object(arguments), None + except ValueError: + return [], arguments + raise TypeError( + "Kimi K3 tool call arguments must be a dict or a JSON object string.") + + +def normalize_message(message: Any) -> Any: + if not isinstance(message, dict): + return message + + normalized = dict(message) + + tools = normalized.get("tools") + if tools is not None: + normalized["tools"] = deep_sort_dict(tools) + + tool_calls = normalized.get("tool_calls") + if not tool_calls: + return normalized + + normalized_calls = [] + for tool_call in tool_calls: + if not isinstance(tool_call, dict): + normalized_calls.append(tool_call) + continue + + tc = dict(tool_call) + function = tc.get("function") + if isinstance(function, dict): + fn = dict(function) + arguments, json_block = normalize_tool_arguments( + fn.get("arguments")) + fn["arguments"] = arguments + if json_block is None: + fn.pop("_xtml_json_block", None) + else: + fn["_xtml_json_block"] = json_block + tc["function"] = fn + else: + arguments, json_block = normalize_tool_arguments( + tc.get("arguments")) + tc["arguments"] = arguments + if json_block is None: + tc.pop("_xtml_json_block", None) + else: + tc["_xtml_json_block"] = json_block + normalized_calls.append(tc) + + normalized["tool_calls"] = normalized_calls + return normalized + + +def normalize_conversation(conversation: Any) -> Any: + if not isinstance(conversation, list): + return conversation + + def normalize_messages(messages: list[Any]) -> list[Any]: + return [normalize_message(message) for message in messages] + + if conversation and isinstance(conversation[0], list): + return [normalize_messages(messages) for messages in conversation] + return normalize_messages(conversation) + + +def _tool_call_id_index(tool_calls: Any) -> dict: + """Map assistant ``tool_calls[].id`` to ``(1-based position, function name)``. + + The position mirrors the chat template's enumeration over ``tool_calls`` + (every entry advances the position, even an id-less one). Duplicate ids keep + their first occurrence. + """ + index: dict = {} + if not isinstance(tool_calls, list): + return index + for position, tool_call in enumerate(tool_calls, start=1): + if not isinstance(tool_call, dict): + continue + call_id = tool_call.get("id") + if call_id is None: + continue + key = str(call_id) + if key in index: + continue + function = tool_call.get("function") + name = (function.get("name") + if isinstance(function, dict) else tool_call.get("name")) + index[key] = (position, name) + return index + + +def normalize_xtml_tool_result_messages(messages: list[Any]) -> list[Any]: + """Re-sort K3 XTML tool results into assistant ``tool_calls`` order. + + Serving frameworks generally deliver tool results already in call order. A + direct Transformers caller, however, may pass OpenAI-style tool messages in any + order, so each run of consecutive tool messages is matched against the most + recent preceding assistant ``tool_calls`` by opaque ``tool_call_id`` == + ``tool_calls[].id`` (K3 drops the ``func:index`` format requirement) and + sorted by the matched 1-based position. The matched call is authoritative, + so each matched message's ``tool`` is set to that call's function name -- + this keeps an explicit (and possibly stale) ``tool``/``name`` from drifting + out of sync with the reordered position. ``index`` is still derived from the + rendered position by the chat template. A run that cannot be fully matched is + left untouched. Re-running is idempotent. + + This function is side-effect free: matched tool messages are shallow-copied + before their ``tool``/``name`` is rewritten, and every other message is + appended to the output as-is. The input list and its message objects are + never mutated. + """ + if not isinstance(messages, list): + return messages + + output: list[Any] = [] + current_index: dict = {} + i = 0 + n = len(messages) + + while i < n: + message = messages[i] + + if isinstance(message, dict) and message.get("role") == "assistant": + tool_calls = message.get("tool_calls") + current_index = _tool_call_id_index( + tool_calls) if tool_calls else {} + output.append(message) + i += 1 + continue + + if not isinstance(message, dict) or message.get("role") != "tool": + output.append(message) + i += 1 + continue + + run: list[tuple] = [] # (position, original_offset, message, name) + unresolved = False + offset = 0 + while (i < n and isinstance(messages[i], dict) + and messages[i].get("role") == "tool"): + tool_message = messages[i] + call_id = tool_message.get("tool_call_id", tool_message.get("id")) + matched = current_index.get( + str(call_id)) if call_id is not None else None + if matched is None: + unresolved = True + run.append((None, offset, tool_message, None)) + else: + position, name = matched + run.append((position, offset, tool_message, name)) + offset += 1 + i += 1 + + if unresolved: + output.extend(item[2] for item in run) + else: + run.sort(key=lambda item: (item[0], item[1])) + for _, _, tool_message, name in run: + if name is None: + output.append(tool_message) + continue + # The id-matched call is authoritative: align tool (and any + # explicit name) so the rendered XTML tool attribute cannot + # disagree with the reordered position. Copy first so the + # caller's message object is never mutated. + resolved = dict(tool_message) + resolved["tool"] = name + if "name" in resolved: + resolved["name"] = name + output.append(resolved) + + return output + + +def is_batched_conversation(conversation: Any) -> bool: + return (isinstance(conversation, list) and bool(conversation) + and isinstance(conversation[0], list)) + + +def _render_content_segments( + content: Any, + image_state: _ImagePromptState, +) -> list[EncodeSegment]: + segments: list[EncodeSegment] = [] + if isinstance(content, str): + _append_text(segments, content, image_state) + elif content is not None: + for part in content: + if part["type"] in ["image", "image_url"]: + segments.extend( + _segment(image_state.next_prompt(), allow_special=True)) + else: + _append_text(segments, part["text"], image_state) + return segments + + +def _internal_system_message(message_type: str, + body: str) -> list[EncodeSegment]: + segments: list[EncodeSegment] = [] + segments.extend( + _open_tag("message", [("role", "system"), ("type", message_type)])) + segments.extend(_text(body.strip())) + segments.extend(_close_tag("message")) + segments.extend(_end_of_msg()) + return segments + + +def _render_assistant_segments( + message: dict[str, Any], + image_state: _ImagePromptState, + thinking: bool = True, +) -> list[EncodeSegment]: + segments: list[EncodeSegment] = [] + # The channel is structural: in thinking mode every assistant + # message carries the open/close tags even when there is no reasoning + # content to fill in. In non-thinking mode the channel is dropped + # entirely. + if thinking: + reasoning_content = message.get("reasoning_content") or message.get( + "reasoning") + segments.extend(_open_tag("think")) + # Only an empty string counts as no reasoning, so whitespace-only + # reasoning is still rendered into the think channel. + if reasoning_content is not None and str(reasoning_content) != "": + _append_text(segments, reasoning_content, image_state) + segments.extend(_close_tag("think")) + + segments.extend(_open_tag("response")) + segments.extend( + _render_content_segments(message.get("content"), image_state)) + segments.extend(_close_tag("response")) + + tool_calls = message.get("tool_calls") + if tool_calls: + segments.extend(_open_tag("tools")) + for index, tool_call in enumerate(tool_calls, start=1): + fn = tool_call.get("function", tool_call) + segments.extend( + _open_tag("call", [("tool", fn["name"]), ("index", index)])) + args = fn.get("arguments", []) + json_block = fn.get("_xtml_json_block") + if json_block is not None: + segments.extend(_open_tag("json", [("type", "object")])) + _append_text(segments, json_block, image_state) + segments.extend(_close_tag("json")) + else: + for key, arg_type, arg_text in args: + segments.extend( + _open_tag("argument", [("key", key), + ("type", arg_type)])) + _append_text(segments, arg_text, image_state) + segments.extend(_close_tag("argument")) + segments.extend(_close_tag("call")) + segments.extend(_close_tag("tools")) + + return segments + + +def _render_tool_declare(tools: Any, + *, + dynamic: bool = False) -> list[EncodeSegment]: + if dynamic: + body = ( + "## New Tools Available\n" + "The system dynamically extends the toolset via lazy-loading.\n" + "You have access to all existing and extended tools.\n" + "Here are the specs for the extended tools.\n\n" + "```json\n" + f"{_json_compact(tools)}\n" + "```") + else: + body = ("# Tools\n" + "Here are the available tools, described in JSONSchema.\n\n" + "```json\n" + f"{_json_compact(tools)}\n" + "```") + segments: list[EncodeSegment] = [] + segments.extend( + _open_tag("message", [("role", "system"), ("type", "tool-declare")])) + segments.extend(_text(body)) + segments.extend(_close_tag("message")) + segments.extend(_end_of_msg()) + return segments + + +def build_chat_segments( + messages: list[Any], + tools: Optional[list[dict]] = None, + *, + add_generation_prompt: bool = True, + thinking: bool = True, + image_prompts: Optional[list[str]] = None, + **kwargs: Any, +) -> list[EncodeSegment]: + # Re-sort tool results by tool_call_id at the lowest layer so every caller + # (processor or direct tokenizer) gets correctly ordered XTML. The helper is + # side-effect free, so the caller's message objects are left untouched. + messages = normalize_xtml_tool_result_messages(messages) + messages = normalize_conversation(messages) + tools = deep_sort_dict(tools) + + kwargs = dict(kwargs) + response_format = kwargs.get("response_format") + if "response_schema" not in kwargs: + response_schema = extract_response_schema(response_format) + if response_schema is not None: + kwargs["response_schema"] = response_schema + if kwargs.get("response_schema") is not None: + kwargs["response_schema"] = deep_sort_dict(kwargs["response_schema"]) + + image_state = _ImagePromptState(image_prompts) + segments: list[EncodeSegment] = [] + + tool_calls = None + tool_index = 0 + + if tools: + segments.extend(_render_tool_declare(tools)) + + thinking_effort = kwargs.get("thinking_effort") + if thinking and thinking_effort is not None: + assert thinking_effort in _VALID_THINKING_EFFORTS, ( + f"Unsupported thinking_effort={thinking_effort!r}; " + f"supported values are {sorted(_VALID_THINKING_EFFORTS)}.") + if thinking and thinking_effort in _VALID_THINKING_EFFORTS: + segments.extend( + _internal_system_message( + "thinking-effort", + "`thinking_effort` guides on how much to think in your " + "thinking channel (not including the response channel), " + "supported values include `low`, `medium`, `high`, and `max`.\n" + f"Now the system is invoked with `thinking_effort={thinking_effort}`.", + )) + + for message_index, message in enumerate(messages): + # Malformed messages are rejected instead of being silently skipped. + if not isinstance(message, dict): + raise ValueError( + f"Kimi K3 messages must be dicts, got {type(message).__name__} " + f"at index {message_index}.") + + role = message.get("role") + if role == "user": + attrs = [("role", "user")] + if message.get("name"): + attrs.append(("name", message["name"])) + segments.extend(_open_tag("message", attrs)) + segments.extend( + _render_content_segments(message.get("content"), image_state)) + segments.extend(_close_tag("message")) + segments.extend(_end_of_msg()) + elif role == "system" and message.get("tools"): + segments.extend( + _render_tool_declare(message["tools"], dynamic=True)) + elif role == "system": + attrs = [("role", "system")] + if message.get("name"): + attrs.append(("name", message["name"])) + segments.extend(_open_tag("message", attrs)) + segments.extend( + _render_content_segments(message.get("content"), image_state)) + segments.extend(_close_tag("message")) + segments.extend(_end_of_msg()) + elif role == "tool": + tool_index += 1 + tool_name = message.get("tool", message.get("name")) + if (tool_name is None and tool_calls is not None + and tool_index <= len(tool_calls)): + tc = tool_calls[tool_index - 1] + fn = tc.get("function", tc) + tool_name = fn["name"] + if tool_name is None: + raise ValueError( + "Kimi K3 tool messages need a resolvable tool name: " + "carry `tool`/`name`, or match a preceding assistant " + "tool_call by order.") + segments.extend( + _open_tag( + "message", + [("role", "tool"), ("tool", tool_name), + ("index", tool_index)], + )) + segments.extend( + _render_content_segments(message.get("content"), image_state)) + segments.extend(_close_tag("message")) + segments.extend(_end_of_msg()) + elif role == "assistant": + tool_calls = message.get("tool_calls") + tool_index = 0 + attrs = [("role", "assistant")] + if message.get("name"): + attrs.append(("name", message["name"])) + segments.extend(_open_tag("message", attrs)) + segments.extend( + _render_assistant_segments(message, image_state, thinking)) + segments.extend(_close_tag("message")) + segments.extend(_end_of_msg()) + else: + raise ValueError( + f"Unknown message role {role!r} at index {message_index}.") + + tool_choice = kwargs.get("tool_choice") + if tool_choice == "required": + segments.extend( + _internal_system_message( + "tool-choice", + "The system is invoked with `tool_choice=required`.\n" + "You MUST call tools in the next message.", + )) + elif tool_choice == "none": + segments.extend( + _internal_system_message( + "tool-choice", + "The system is invoked with `tool_choice=none`.\n" + "You MUST NOT call any tools in the next message.", + )) + + rf = kwargs.get("response_format") + rf_type = _get_value(rf, "type", rf) if isinstance(rf, dict) else rf + if rf_type == "json_object": + segments.extend( + _internal_system_message( + "response-format", + "The system is invoked with `response_format=json_object`.\n" + "Your response must be raw JSON data without markdown code " + "blocks (```json) or any additional formatting.", + )) + elif rf_type == "json_schema": + schema = _json_compact(kwargs.get("response_schema")) + segments.extend( + _internal_system_message( + "response-format", + "The system is invoked with `response_format=json_schema`.\n" + "Your response must be raw JSON data without markdown code " + "blocks (```json) or any additional formatting.\n" + "The JSON data must match the following schema:\n" + f"```json\n{schema}\n```", + )) + + if add_generation_prompt: + segments.extend(_open_tag("message", [("role", "assistant")])) + segments.extend(_open_tag("think" if thinking else "response")) + + image_state.assert_consumed() + return segments diff --git a/evaluation_results_93layer.json b/evaluation_results_93layer.json new file mode 100644 index 0000000000000000000000000000000000000000..11ad4b34756f219d359a6e845f77506bf5180ad2 --- /dev/null +++ b/evaluation_results_93layer.json @@ -0,0 +1,163 @@ +{ + "model": "Grawnch/Kimi-K3.1-Pro-Coder", + "base_model": "moonshotai/Kimi-K3.1-Pro", + "hardware": "NVIDIA RTX PRO 6000 Blackwell Server Edition (102.6 GB GDDR7 VRAM, 176 GiB System RAM, 135 GiB /dev/shm)", + "layers_evaluated": 93, + "shards_evaluated": "96/96 (Shards 1..93 Layers 0..92 + Shard 94 embed_tokens/output_attn_res/norm/lm_head)", + "aime2025_benchmark": { + "total_sequences": 40, + "total_tokens": 17453, + "wall_time_sec": 725.17, + "moe_gpu_sec": 474.68, + "moe_layer_tokens_per_sec": 3382.7, + "end_to_end_tokens_per_sec": 36.77, + "top1_agreement_pct": 95.20, + "top5_agreement_pct": 99.91, + "mean_kld": 0.01912, + "aime2025_accuracy_coder": "29/30 (96.67%)", + "aime2025_accuracy_ref": "29/30 (96.67%)", + "aime2025_correct_coder": 29, + "aime2025_correct_ref": 29, + "aime2025_exact_match_agreement_pct": 100.0, + "math_ppl_coder": 2.2358, + "math_ppl_ref": 2.2135, + "math_delta_ppl": 0.0223, + "code_ppl_coder": 1.5814, + "code_ppl_ref": 1.5594, + "code_delta_ppl": 0.0220, + "overall_ppl_coder": 2.1627, + "overall_ppl_ref": 2.1403, + "overall_delta_ppl": 0.0224, + "strata_tier1_hot_vram_pct": 16.56, + "strata_tier2_warm_ram_pct": 29.94, + "strata_tier3_cold_disk_pct": 53.50, + "strata_hot_warm_static_pct": 46.50, + "strata_deep_layers_88_91_hot_warm_pct": 66.28, + "strata_router_lookahead_effective_hit_pct": 97.92, + "checkpoint_layers": [ + {"layer": 1, "kind": "KDA", "sec": 17.18, "moe_sec": 5.634, "moe_cos": 0.923062, "state_cos": 0.977311}, + {"layer": 3, "kind": "MLA", "sec": 9.972, "moe_sec": 5.479, "moe_cos": 0.939353, "state_cos": 0.979728}, + {"layer": 15, "kind": "MLA", "sec": 9.686, "moe_sec": 5.106, "moe_cos": 0.891969, "state_cos": 0.982217}, + {"layer": 31, "kind": "MLA", "sec": 10.036,"moe_sec": 5.098, "moe_cos": 0.947467, "state_cos": 0.990545}, + {"layer": 47, "kind": "MLA", "sec": 9.733, "moe_sec": 5.061, "moe_cos": 0.927411, "state_cos": 0.976064}, + {"layer": 63, "kind": "MLA", "sec": 9.915, "moe_sec": 5.103, "moe_cos": 0.835011, "state_cos": 0.971826}, + {"layer": 79, "kind": "MLA", "sec": 10.064,"moe_sec": 5.162, "moe_cos": 0.868206, "state_cos": 0.986496}, + {"layer": 88, "kind": "KDA", "sec": 11.407,"moe_sec": 5.091, "moe_cos": 0.902579, "state_cos": 0.987363}, + {"layer": 90, "kind": "KDA", "sec": 11.303,"moe_sec": 5.026, "moe_cos": 0.968929, "state_cos": 0.989804}, + {"layer": 91, "kind": "MLA", "sec": 11.182,"moe_sec": 5.195, "moe_cos": 0.999990, "state_cos": 0.999992}, + {"layer": 92, "kind": "MLA", "sec": 11.036,"moe_sec": 5.099, "moe_cos": 0.999958, "state_cos": 0.999994} + ] + }, + "livecodebench_rolling_window": { + "window": "2024.10 - 2025.05 (LeetCode / AtCoder / Codeforces)", + "total_problems": 30, + "difficulty_split": "10 Easy / 10 Medium / 10 Hard", + "total_tokens": 4854, + "wall_time_sec": 754.71, + "moe_gpu_sec": 449.96, + "moe_layer_tokens_per_sec": 992.5, + "end_to_end_tokens_per_sec": 10.79, + "top1_agreement_pct": 95.06, + "top5_agreement_pct": 99.92, + "mean_kld": 0.02248, + "lcb_pass1_overall_coder": "29/30 (96.67%)", + "lcb_pass1_overall_ref": "29/30 (96.67%)", + "lcb_pass1_easy_coder": "9/10 (90.0%)", + "lcb_pass1_easy_ref": "9/10 (90.0%)", + "lcb_pass1_medium_coder": "10/10 (100.0%)", + "lcb_pass1_medium_ref": "10/10 (100.0%)", + "lcb_pass1_hard_coder": "10/10 (100.0%)", + "lcb_pass1_hard_ref": "10/10 (100.0%)", + "lcb_exact_match_agreement_pct": 100.0, + "lcb_ppl_overall_coder": 2.7629, + "lcb_ppl_overall_ref": 2.6911, + "lcb_ppl_easy_coder": 5.4529, + "lcb_ppl_easy_ref": 5.1851, + "lcb_ppl_medium_coder": 2.9995, + "lcb_ppl_medium_ref": 2.9266, + "lcb_ppl_hard_coder": 2.1562, + "lcb_ppl_hard_ref": 2.1136, + "strata_tier1_hot_vram_pct": 15.64, + "strata_tier2_warm_ram_pct": 28.75, + "strata_tier3_cold_disk_pct": 55.61, + "strata_hot_warm_static_pct": 44.39, + "strata_deep_layers_88_92_hot_warm_pct": 61.56, + "checkpoint_layers": [ + {"layer": 1, "kind": "KDA", "sec": 19.375,"moe_sec": 5.227, "moe_cos": 0.918028, "state_cos": 0.975290}, + {"layer": 3, "kind": "MLA", "sec": 8.101, "moe_sec": 4.452, "moe_cos": 0.926291, "state_cos": 0.976039}, + {"layer": 15, "kind": "MLA", "sec": 154.52,"moe_sec": 5.004, "moe_cos": 0.893001, "state_cos": 0.985080}, + {"layer": 31, "kind": "MLA", "sec": 8.936, "moe_sec": 5.016, "moe_cos": 0.943237, "state_cos": 0.990962}, + {"layer": 47, "kind": "MLA", "sec": 8.700, "moe_sec": 4.839, "moe_cos": 0.930096, "state_cos": 0.975653}, + {"layer": 63, "kind": "MLA", "sec": 9.008, "moe_sec": 5.004, "moe_cos": 0.840948, "state_cos": 0.972550}, + {"layer": 79, "kind": "MLA", "sec": 8.771, "moe_sec": 4.787, "moe_cos": 0.885594, "state_cos": 0.988172}, + {"layer": 88, "kind": "KDA", "sec": 9.625, "moe_sec": 5.023, "moe_cos": 0.871222, "state_cos": 0.982751}, + {"layer": 90, "kind": "KDA", "sec": 9.307, "moe_sec": 4.712, "moe_cos": 0.982155, "state_cos": 0.990625}, + {"layer": 91, "kind": "MLA", "sec": 10.579,"moe_sec": 5.053, "moe_cos": 0.999996, "state_cos": 0.999997}, + {"layer": 92, "kind": "MLA", "sec": 10.629,"moe_sec": 4.882, "moe_cos": 0.999982, "state_cos": 0.999998} + ] + }, + "aime2025_problems": [ + {"id": "AIME25_P01", "gold": 70, "pred_coder": 70, "pred_ref": 70, "correct": true, "top3": [70, 50, 60]}, + {"id": "AIME25_P02", "gold": 588, "pred_coder": 588, "pred_ref": 588, "correct": true, "top3": [588, 592, 590]}, + {"id": "AIME25_P03", "gold": 16, "pred_coder": 16, "pred_ref": 16, "correct": true, "top3": [16, 101, 201]}, + {"id": "AIME25_P04", "gold": 117, "pred_coder": 117, "pred_ref": 117, "correct": true, "top3": [117, 118, 116]}, + {"id": "AIME25_P05", "gold": 279, "pred_coder": 279, "pred_ref": 279, "correct": true, "top3": [279, 79, 179]}, + {"id": "AIME25_P06", "gold": 504, "pred_coder": 504, "pred_ref": 504, "correct": true, "top3": [504, 576, 288]}, + {"id": "AIME25_P07", "gold": 821, "pred_coder": 821, "pred_ref": 821, "correct": true, "top3": [821, 521, 721]}, + {"id": "AIME25_P08", "gold": 77, "pred_coder": 77, "pred_ref": 77, "correct": true, "top3": [77, 73, 7]}, + {"id": "AIME25_P09", "gold": 62, "pred_coder": 62, "pred_ref": 62, "correct": true, "top3": [62, 64, 63]}, + {"id": "AIME25_P10", "gold": 81, "pred_coder": 81, "pred_ref": 81, "correct": true, "top3": [81, 79, 77]}, + {"id": "AIME25_P11", "gold": 259, "pred_coder": 259, "pred_ref": 259, "correct": true, "top3": [259, 1, 257]}, + {"id": "AIME25_P12", "gold": 510, "pred_coder": 510, "pred_ref": 510, "correct": true, "top3": [510, 512, 610]}, + {"id": "AIME25_P13", "gold": 204, "pred_coder": 204, "pred_ref": 204, "correct": true, "top3": [204, 202, 200]}, + {"id": "AIME25_P14", "gold": 60, "pred_coder": 59, "pred_ref": 59, "correct": false, "top3": [59, 38, 77]}, + {"id": "AIME25_P15", "gold": 735, "pred_coder": 735, "pred_ref": 735, "correct": true, "top3": [735, 635, 885]}, + {"id": "AIME25_P16", "gold": 468, "pred_coder": 468, "pred_ref": 468, "correct": true, "top3": [468, 469, 168]}, + {"id": "AIME25_P17", "gold": 49, "pred_coder": 49, "pred_ref": 49, "correct": true, "top3": [49, 50, 48]}, + {"id": "AIME25_P18", "gold": 82, "pred_coder": 82, "pred_ref": 82, "correct": true, "top3": [82, 88, 84]}, + {"id": "AIME25_P19", "gold": 106, "pred_coder": 106, "pred_ref": 106, "correct": true, "top3": [106, 107, 105]}, + {"id": "AIME25_P20", "gold": 336, "pred_coder": 336, "pred_ref": 336, "correct": true, "top3": [336, 216, 324]}, + {"id": "AIME25_P21", "gold": 293, "pred_coder": 293, "pred_ref": 293, "correct": true, "top3": [293, 296, 393]}, + {"id": "AIME25_P22", "gold": 237, "pred_coder": 237, "pred_ref": 237, "correct": true, "top3": [237, 236, 235]}, + {"id": "AIME25_P23", "gold": 610, "pred_coder": 610, "pred_ref": 610, "correct": true, "top3": [610, 310, 210]}, + {"id": "AIME25_P24", "gold": 149, "pred_coder": 149, "pred_ref": 149, "correct": true, "top3": [149, 139, 249]}, + {"id": "AIME25_P25", "gold": 907, "pred_coder": 907, "pred_ref": 907, "correct": true, "top3": [907, 707, 908]}, + {"id": "AIME25_P26", "gold": 113, "pred_coder": 113, "pred_ref": 113, "correct": true, "top3": [113, 112, 123]}, + {"id": "AIME25_P27", "gold": 19, "pred_coder": 19, "pred_ref": 19, "correct": true, "top3": [19, 20, 9]}, + {"id": "AIME25_P28", "gold": 248, "pred_coder": 248, "pred_ref": 248, "correct": true, "top3": [248, 268, 148]}, + {"id": "AIME25_P29", "gold": 104, "pred_coder": 104, "pred_ref": 104, "correct": true, "top3": [104, 102, 103]}, + {"id": "AIME25_P30", "gold": 240, "pred_coder": 240, "pred_ref": 240, "correct": true, "top3": [240, 40, 236]} + ], + "lcb_problems": [ + {"id": "LCB_E01", "difficulty": "Easy", "platform": "LeetCode-3300", "pred_coder": ")\\n\\n", "pred_ref": ")\\n\\n", "pass": true}, + {"id": "LCB_E02", "difficulty": "Easy", "platform": "LeetCode-3304", "pred_coder": "bit", "pred_ref": "bit", "pass": true}, + {"id": "LCB_E03", "difficulty": "Easy", "platform": "LeetCode-3314", "pred_coder": "1", "pred_ref": "1", "pass": true}, + {"id": "LCB_E04", "difficulty": "Easy", "platform": "AtCoder-ABC376A", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true}, + {"id": "LCB_E05", "difficulty": "Easy", "platform": "LeetCode-3330", "pred_coder": "1", "pred_ref": "1", "pass": true}, + {"id": "LCB_E06", "difficulty": "Easy", "platform": "LeetCode-3340", "pred_coder": "2", "pred_ref": "2", "pass": true}, + {"id": "LCB_E07", "difficulty": "Easy", "platform": "LeetCode-3345", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true}, + {"id": "LCB_E08", "difficulty": "Easy", "platform": "AtCoder-ABC380A", "pred_coder": "3", "pred_ref": "3", "pass": true}, + {"id": "LCB_E09", "difficulty": "Easy", "platform": "LeetCode-3370", "pred_coder": "1", "pred_ref": "1", "pass": true}, + {"id": "LCB_E10", "difficulty": "Easy", "platform": "Codeforces-2036A", "pred_coder": "))", "pred_ref": "))", "pass": false}, + {"id": "LCB_M01", "difficulty": "Medium", "platform": "LeetCode-3301", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true}, + {"id": "LCB_M02", "difficulty": "Medium", "platform": "LeetCode-3309", "pred_coder": "2", "pred_ref": "2", "pass": true}, + {"id": "LCB_M03", "difficulty": "Medium", "platform": "LeetCode-3316", "pred_coder": "1", "pred_ref": "1", "pass": true}, + {"id": "LCB_M04", "difficulty": "Medium", "platform": "LeetCode-3325", "pred_coder": "1", "pred_ref": "1", "pass": true}, + {"id": "LCB_M05", "difficulty": "Medium", "platform": "LeetCode-3335", "pred_coder": "\\n\\n", "pred_ref": "\\n\\n", "pass": true}, + {"id": "LCB_M06", "difficulty": "Medium", "platform": "LeetCode-3350", "pred_coder": "))\\n", "pred_ref": "))\\n", "pass": true}, + {"id": "LCB_M07", "difficulty": "Medium", "platform": "LeetCode-3355", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true}, + {"id": "LCB_M08", "difficulty": "Medium", "platform": "LeetCode-3365", "pred_coder": "2", "pred_ref": "2", "pass": true}, + {"id": "LCB_M09", "difficulty": "Medium", "platform": "AtCoder-ABC378D", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true}, + {"id": "LCB_M10", "difficulty": "Medium", "platform": "Codeforces-2033D", "pred_coder": "0", "pred_ref": "0", "pass": true}, + {"id": "LCB_H01", "difficulty": "Hard", "platform": "LeetCode-3303", "pred_coder": "1", "pred_ref": "1", "pass": true}, + {"id": "LCB_H02", "difficulty": "Hard", "platform": "LeetCode-3307", "pred_coder": "26", "pred_ref": "26", "pass": true}, + {"id": "LCB_H03", "difficulty": "Hard", "platform": "LeetCode-3312", "pred_coder": "]\\n", "pred_ref": "]\\n", "pass": true}, + {"id": "LCB_H04", "difficulty": "Hard", "platform": "LeetCode-3321", "pred_coder": "))\\n", "pred_ref": "))\\n", "pass": true}, + {"id": "LCB_H05", "difficulty": "Hard", "platform": "LeetCode-3333", "pred_coder": "\\n\\n\\n","pred_ref": "\\n\\n\\n","pass": true}, + {"id": "LCB_H06", "difficulty": "Hard", "platform": "LeetCode-3337", "pred_coder": "1", "pred_ref": "1", "pass": true}, + {"id": "LCB_H07", "difficulty": "Hard", "platform": "LeetCode-3343", "pred_coder": "),", "pred_ref": "),", "pass": true}, + {"id": "LCB_H08", "difficulty": "Hard", "platform": "LeetCode-3352", "pred_coder": "1", "pred_ref": "1", "pass": true}, + {"id": "LCB_H09", "difficulty": "Hard", "platform": "AtCoder-ABC379F", "pred_coder": "1", "pred_ref": "1", "pass": true}, + {"id": "LCB_H10", "difficulty": "Hard", "platform": "Codeforces-2037G", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true} + ] +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3908a4a5477122aa4b09386acab760fb7887c0fc --- /dev/null +++ b/generation_config.json @@ -0,0 +1,4 @@ +{ + "max_length": 1048576, + "eos_token_id": 163586 +} \ No newline at end of file diff --git a/kimi31_coder_benchmark_card.png b/kimi31_coder_benchmark_card.png new file mode 100644 index 0000000000000000000000000000000000000000..51b58b707ba71dd54abb48a23df05b7b33d0e7e4 --- /dev/null +++ b/kimi31_coder_benchmark_card.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5741189e9f78ed687f1188395316d9e5ba9109f941187444366ef657d4e1451 +size 552507 diff --git a/kimi_k3_processor.py b/kimi_k3_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..033a5773885c144509788e89a445217b76d6c046 --- /dev/null +++ b/kimi_k3_processor.py @@ -0,0 +1,187 @@ +"""Kimi-K3 processor: wraps vision processor + tokenizer into a single interface. + +Chat rendering (including XTML tool-result ordering) is handled by the +tokenizer's Python encoder; this processor adds multimodal media preprocessing. +""" + +from transformers.feature_extraction_utils import BatchFeature +from transformers.processing_utils import ProcessorMixin +from transformers.utils import logging + +from .media_utils import ensure_media_type + +logger = logging.get_logger(__name__) + +# ── KimiK3Processor ─────────────────────────────────────────────────── + + +class KimiK3Processor(ProcessorMixin): + r""" + Constructs a KimiK3 processor which wraps a KimiK3 image processor + and a tokenizer into a single processor. + + [`KimiK3Processor`] offers all the functionalities of + [`KimiK3VisionProcessor`] and [`TikTokenTokenizer`]. + + Args: + image_processor ([`KimiK3VisionProcessor`], *optional*): + The image processor is a required input. + tokenizer ([`TikTokenTokenizer`], *optional*): + The tokenizer is a required input. + chat_template (`str`, *optional*): Kept for ProcessorMixin + compatibility. Kimi K3 chat encoding is implemented in Python by + the tokenizer. + """ + + attributes = ["image_processor", "tokenizer"] + valid_kwargs = ["chat_template"] + image_processor_class = "AutoImageProcessor" + tokenizer_class = "AutoTokenizer" + + def __init__( + self, + image_processor=None, + tokenizer=None, + chat_template=None, + **kwargs, + ): + super().__init__(image_processor, + tokenizer, + chat_template=chat_template) + self.media_processor = image_processor + self.image_placeholder = "<|kimi_image_placeholder|>" + + # ── Media preprocessing ──────────────────────────────────────────── + + def update_raw_text(self, text: str, image_prompts: list[str]) -> str: + # Replace image placeholders + image_count = text.count(self.image_placeholder) + if image_count > 0: + assert image_count == len(image_prompts), ( + f"image placeholder count {image_count} != " + f"image_prompts count {len(image_prompts)}") + text_parts = text.split(self.image_placeholder) + assert len(text_parts) == len(image_prompts) + 1 + text = "".join([ + text_parts[i] + image_prompts[i] + for i in range(len(image_prompts)) + ]) + text += text_parts[-1] + + return text + + def preprocess_medias(self, + medias: list[dict]) -> tuple[list[dict], list[str]]: + """Process media items and generate corresponding prompts. + + Returns: + A tuple of (updated_medias, image_prompts). + """ + updated_medias = [] + image_prompts = [] + for media in medias: + if media['type'] == 'image': + updated_medias.append(media) + img = ensure_media_type( + media, + transparent_bg_config=self.media_processor. + _transparent_bg_config, + transparent_bg_fill_stage=self.media_processor. + _transparent_bg_fill_stage, + )['image'] + w, h = img.size + image_prompts.append( + self.media_processor.make_image_prompt(w, h)) + else: + raise ValueError(f"unsupported media type: {media['type']}") + return updated_medias, image_prompts + + # ── Main entry points ────────────────────────────────────────────── + + def __call__(self, + messages: list[dict] = None, + medias: list[dict] = None, + text: str = None, + return_tensors: str = "pt", + **kwargs) -> BatchFeature: + """ + Process multimodal inputs for Kimi-K3 model. + + Args: + messages: List of message dicts with 'role' and 'content' fields. + If provided, medias and text will be extracted automatically. + medias: Pre-extracted list of media dicts. + text: Pre-formatted text string. + return_tensors: Format of returned tensors. Default: 'pt'. + **kwargs: Additional arguments passed to apply_chat_template. + + Returns: + BatchFeature with fields: input_ids, attention_mask, + pixel_values, grid_thws. + """ + if messages is None and (medias is None or text is None): + raise ValueError( + "Provide either 'messages' or both 'medias' and 'text'") + + if medias is not None and text is not None: + updated_medias, image_prompts = (self.preprocess_medias(medias)) + preprocessed = self.media_processor.preprocess( + updated_medias, return_tensors=return_tensors) + text = self.update_raw_text(text, image_prompts) + text_inputs = self.tokenizer(text, return_tensors=return_tensors) + return BatchFeature(data={**text_inputs, **preprocessed.data}) + + if medias is None: + medias = self._extract_medias_from_messages(messages) + updated_medias, image_prompts = (self.preprocess_medias(medias)) + preprocessed = self.media_processor.preprocess( + updated_medias, return_tensors=return_tensors) + + if text is None: + text_inputs = self.tokenizer.apply_chat_template( + messages, + tokenize=True, + return_tensors=return_tensors, + return_dict=True, + image_prompts=image_prompts, + **kwargs) + return BatchFeature(data={**text_inputs, **preprocessed.data}) + + text = self.update_raw_text(text, image_prompts) + text_inputs = self.tokenizer(text, return_tensors=return_tensors) + return BatchFeature(data={**text_inputs, **preprocessed.data}) + + @staticmethod + def _extract_medias_from_messages(messages: list[dict]) -> list[dict]: + """Extract media items from messages in a single pass.""" + medias = [] + for msg in messages: + if msg['role'] != 'user' or not msg.get('content'): + continue + + for content_part in msg['content']: + if not isinstance(content_part, dict): + continue + + content_type = content_part.get('type') + if content_type in ['image_url', 'image']: + image_data = content_part.get(content_type) + assert image_data is not None, f"image data is missing for content part: {content_part}" + medias.append({ + 'type': 'image', + 'image': image_data, + }) + return medias + + def apply_chat_template(self, messages, **kwargs): + return self.tokenizer.apply_chat_template(messages, **kwargs) + + def batch_decode(self, *args, **kwargs): + return self.tokenizer.batch_decode(*args, **kwargs) + + def decode(self, *args, **kwargs): + return self.tokenizer.decode(*args, **kwargs) + + @property + def model_input_names(self): + return ['input_ids', 'attention_mask', 'pixel_values', 'grid_thws'] diff --git a/kimi_k3_vision_processing.py b/kimi_k3_vision_processing.py new file mode 100644 index 0000000000000000000000000000000000000000..58154424ff951289570c8fb5578e7372bfbfaa8b --- /dev/null +++ b/kimi_k3_vision_processing.py @@ -0,0 +1,179 @@ +"""Image processor class for Kimi-K3. +""" + +import json +from typing import Any, Dict, Optional, Union + +import numpy as np +import torch +from PIL import Image +from transformers.image_processing_utils import (BaseImageProcessor, + BatchFeature) +from transformers.utils import TensorType + +from .media_utils import (MediaInput, TransparentBgConfig, _to_tensor, + ensure_media_type, image_to_np, navit_patchify, + navit_resize_image, normalize) + + +class KimiK3VisionProcessor(BaseImageProcessor): + model_type = "kimi_k3" + + def __init__( + self, + media_proc_cfg: dict, + **kwargs, + ): + super().__init__(**kwargs) + self.media_proc_cfg = media_proc_cfg + + @property + def _transparent_bg_config(self) -> Optional[TransparentBgConfig]: + cfg = self.media_proc_cfg.get("transparent_bg_config") + if cfg is None: + return None + if isinstance(cfg, TransparentBgConfig): + return cfg + return TransparentBgConfig(**cfg) + + @property + def _transparent_bg_fill_stage(self) -> str: + return self.media_proc_cfg.get("transparent_bg_fill_stage", + "before_resize") + + def media_tokens_calculator(self, media: MediaInput): + media = ensure_media_type( + media, + transparent_bg_config=self._transparent_bg_config, + transparent_bg_fill_stage=self._transparent_bg_fill_stage, + ) + ret = self.get_resize_config(media) + return ret['num_tokens'] + + @classmethod + def make_image_prompt(cls, width: int, height: int) -> str: + """Build the K3 image placeholder with resolution info.""" + return (f"<|media_begin|>image {width}x{height}" + f"<|media_content|><|media_pad|><|media_end|>") + + def get_resize_config(self, media_input: MediaInput) -> dict: + if media_input['type'] == 'image': + w, h = media_input['image'].size + ret = navit_resize_image( + w, h, self.media_proc_cfg['patch_size'], + self.media_proc_cfg['merge_kernel_size'], + self.media_proc_cfg['in_patch_limit'], + self.media_proc_cfg['patch_limit_on_one_side'], + self.media_proc_cfg['fixed_output_tokens']) + return ret + else: + raise ValueError("Unsupported type: {}".format( + media_input['type'])) + + def resize_image(self, image: Image.Image, new_width: int, new_height: int, + pad_width: int, pad_height: int) -> np.ndarray: + image_np = image_to_np( + image, + (new_width, new_height), + "resize", + transparent_bg_config=self._transparent_bg_config, + transparent_bg_fill_stage=self._transparent_bg_fill_stage, + ) + image_np = np.pad( + image_np, + ((0, pad_height), (0, pad_width), (0, 0)), + mode="constant", + constant_values=0, + ) + return image_np + + def preprocess( + self, + medias: list[MediaInput], + return_tensors: Optional[Union[str, TensorType]] = None, + ) -> BatchFeature: + """ + Preprocess a atom vision input (images) into model-ready tensors. + + Args: + medias: List of MediaInput. + return_tensors: Desired output format ('pt', 'np', 'tf', or None). + + Returns: + BatchFeature containing 'pixel_values' and 'grid_thws' tensors. + """ + if not isinstance(medias, list): + medias = [medias] + if medias: + pixel_values = [] + for item in medias: + item = ensure_media_type( + item, + transparent_bg_config=self._transparent_bg_config, + transparent_bg_fill_stage=self._transparent_bg_fill_stage, + ) + resize_config = self.get_resize_config(item) + new_width, new_height, pad_width, pad_height = resize_config[ + 'new_width'], resize_config['new_height'], resize_config[ + 'pad_width'], resize_config['pad_height'] + if item['type'] == 'image': + image = item['image'] + image_np = self.resize_image(image, new_width, new_height, + pad_width, pad_height) + pixel_values.append(np.expand_dims(image_np, axis=0)) + else: + raise ValueError("Unsupported type: {}".format( + item['type'])) + normalized_pixel_values = [] + image_std_inv = 1.0 / np.array(self.media_proc_cfg['image_std']) + image_mean = np.array(self.media_proc_cfg['image_mean']) + for pixels in pixel_values: + pixels = normalize(pixels, image_mean, image_std_inv) + pixels_and_thw = navit_patchify( + pixels, + self.media_proc_cfg['patch_size'], + ) + normalized_pixel_values.append(pixels_and_thw) + + pixel_values = torch.cat([ + _to_tensor(pixel_value['pixel_values']) + for pixel_value in normalized_pixel_values + ]) + grid_thws = torch.cat([ + _to_tensor(pixel_value['grid_thw'], + dtype=torch.int64).unsqueeze(0) + for pixel_value in normalized_pixel_values + ]) + + data = { + 'pixel_values': pixel_values, + 'grid_thws': grid_thws, + } + + else: + data = {} + + return BatchFeature(data=data, tensor_type=return_tensors) + + def __repr__(self): + return f"KimiK3VisionProcessor(media_proc_cfg={self.media_proc_cfg})" + + def to_dict(self) -> Dict[str, Any]: + output = super().to_dict() + output["media_proc_cfg"] = self.media_proc_cfg + if "media_processor" in output: + del output["media_processor"] + return output + + @classmethod + def from_dict(cls, config_dict: Dict[str, Any], **kwargs): + config = config_dict.copy() + media_proc_cfg = config.pop("media_proc_cfg", {}) + return cls(media_proc_cfg=media_proc_cfg, **config, **kwargs) + + def to_json_string(self): + dictionary = self.to_dict() + for key, value in dictionary.items(): + if hasattr(value, 'tolist'): + dictionary[key] = value.tolist() + return json.dumps(dictionary, indent=2, sort_keys=True) + "\n" diff --git a/media_utils.py b/media_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e8b2ae127e254e364e6b568b1093a7e12ac576e0 --- /dev/null +++ b/media_utils.py @@ -0,0 +1,376 @@ +import base64 +import functools +import io +import math +from dataclasses import dataclass +from typing import Literal, TypedDict + +import numpy as np +from PIL import Image + + +class ImageInput(TypedDict): + type: Literal['image'] + image: Image.Image + + +MediaInput = ImageInput + + +@dataclass +class TransparentBgConfig: + """The config of the transparent background.""" + + pattern: Literal["white", "black", "gray", "chessboard"] = "black" + """The pattern of the transparent background.""" + + chessboard_square_size: int = 16 + """The size of the squares in the chessboard background.""" + + chessboard_square_on_top_left: bool = True + """Whether to start the chessboard with a white square on the top left.""" + + chessboard_white_value: int = 255 + """The value of the white pixels in the background.""" + + chessboard_gray_value: int = 200 + """The value of the gray pixels in the background.""" + + +@functools.lru_cache(maxsize=256) +def _create_chessboard_background( + height: int, + width: int, + square_size: int, + square_on_top_left: bool, + white_value: int, + gray_value: int, +) -> np.ndarray: + """Create a chessboard background.""" + bg = np.ones((height, width, 3), dtype=np.uint8) * white_value + for y in range(0, height, square_size): + for x in range(0, width, square_size): + if (y // square_size + x // square_size) % 2 == ( + 1 if square_on_top_left else 0): + bg[y:y + square_size, x:x + square_size] = gray_value + return bg + + +def fill_transparent_bg_with( + image: Image.Image, + transparent_bg_config: TransparentBgConfig | None = None, +) -> Image.Image: + """Composite a (possibly) transparent image onto a configured background. + + When ``transparent_bg_config`` is ``None``, the image is simply converted + to RGB (preserving the historical behavior). Otherwise the alpha channel + is alpha-composited over a background generated according to the config. + """ + if transparent_bg_config is None: + return image.convert("RGB") + + if image.mode == "RGB": + return image + + has_alpha = "A" in image.getbands() or "transparency" in image.info + if not has_alpha: + return image.convert("RGB") + + img = np.array(image.convert("RGBA")) + height, width = img.shape[:2] + bg_pattern = transparent_bg_config.pattern + if bg_pattern == "white": + bg = np.full((height, width, 3), 255, dtype=np.uint8) + elif bg_pattern == "black": + bg = np.zeros((height, width, 3), dtype=np.uint8) + elif bg_pattern == "gray": + bg = np.full((height, width, 3), 128, dtype=np.uint8) + elif bg_pattern == "chessboard": + bg = _create_chessboard_background( + height, + width, + transparent_bg_config.chessboard_square_size, + transparent_bg_config.chessboard_square_on_top_left, + transparent_bg_config.chessboard_white_value, + transparent_bg_config.chessboard_gray_value, + ) + else: + raise ValueError(f"Invalid background pattern: {bg_pattern}") + + alpha = img[:, :, 3] + img_rgb = img[:, :, :3] + alpha_normalized = alpha.astype(np.float32) / 255.0 + alpha_3d = np.stack([alpha_normalized] * 3, axis=2) + result = alpha_3d * img_rgb + (1 - alpha_3d) * bg + result = result.astype(np.uint8) + return Image.fromarray(result) + + +def navit_resize_image( + width: int, + height: int, + patch_size: int, + merge_kernel_size: int, + in_patch_limit: int, + patch_limit_on_one_side: int, + fixed_output_tokens: int | None, +): + # Apply the patch limits. + s1 = math.sqrt( + in_patch_limit / + (max(1.0, width // patch_size) * max(1.0, height // patch_size))) + s2 = patch_limit_on_one_side * patch_size / width + s3 = patch_limit_on_one_side * patch_size / height + scale = min(1.0, s1, s2, s3) + new_w, new_h = max(1, int(width * scale)), max(1, int(height * scale)) + new_w = min(new_w, patch_limit_on_one_side * patch_size) + new_h = min(new_h, patch_limit_on_one_side * patch_size) + + # Calculate the padding to make the height and width divisible by the merge kernel size and patch size. + factor = merge_kernel_size * patch_size + + pad_height = (factor - new_h % factor) % factor + pad_width = (factor - new_w % factor) % factor + + if fixed_output_tokens is not None: + num_tokens = fixed_output_tokens + else: + # Calculate new dimensions after padding and patching + token_height = (new_h + pad_height) // factor + token_width = (new_w + pad_width) // factor + + assert token_height * merge_kernel_size <= patch_limit_on_one_side, ( + f"token_height {token_height} * merge_kernel_size {merge_kernel_size} > patch_limit_on_one_side {patch_limit_on_one_side}" + ) + assert token_width * merge_kernel_size <= patch_limit_on_one_side, ( + f"token_width {token_width} * merge_kernel_size {merge_kernel_size} > patch_limit_on_one_side {patch_limit_on_one_side}" + ) + + num_tokens = token_height * token_width + return { + "num_tokens": num_tokens, + "new_width": new_w, + "new_height": new_h, + "pad_width": pad_width, + "pad_height": pad_height, + "sampled_nframes": 1, + } + + +def _to_pil( + data: str | bytes | Image.Image, + transparent_bg_config: TransparentBgConfig | None = None, + to_rgb: bool = True, +) -> Image.Image: + """Load an image and (optionally) composite its transparent background. + + Args: + data: A PIL Image, a base64 ``data:`` URL, a file path, or raw bytes. + transparent_bg_config: The config used to fill the transparent + background. ``None`` keeps the historical behavior of converting + to RGB without compositing. + to_rgb: If ``False`` the image is returned as-is (the + ``transparent_bg_config`` is ignored). The caller is then + expected to call :func:`fill_transparent_bg_with` later — e.g. + after a resize. + """ + if isinstance(data, Image.Image): + image = data + elif isinstance(data, str): + if data.startswith("data:"): + raw_base64 = data.split(",")[1] + image = Image.open(io.BytesIO(base64.b64decode(raw_base64))) + else: + image = Image.open(data) + elif isinstance(data, bytes): + image = Image.open(io.BytesIO(data)) + else: + raise ValueError(f"Unsupported data type: {type(data)}") + + if not to_rgb: + return image + + return fill_transparent_bg_with(image, transparent_bg_config) + + +def ensure_media_type( + media: MediaInput, + transparent_bg_config: TransparentBgConfig | None = None, + transparent_bg_fill_stage: Literal["before_resize", + "after_resize"] = "before_resize", +) -> MediaInput: + if media['type'] == 'image': + media['image'] = _to_pil( + media['image'], + transparent_bg_config=transparent_bg_config, + to_rgb=transparent_bg_fill_stage == "before_resize", + ) + return media + else: + raise ValueError(f"Unsupported media type: {media['type']}") + + +def image_to_np( + image: Image.Image, + resize_to: tuple[int, int] | None = None, + mode: str = "resize", + raise_error_for_ill_resize: bool = True, + transparent_bg_config: TransparentBgConfig | None = None, + transparent_bg_fill_stage: Literal["before_resize", + "after_resize"] = "before_resize", +) -> np.ndarray: + """Convert an image to a numpy array. + + Args: + content: The image to convert. + resize_to: The size to resize the image to. + mode: The mode to resize the image to. + raise_error_for_ill_resize: Whether to raise an error for ill-sized resize. + transparent_bg_config: The config of the transparent background. Only + used when ``transparent_bg_fill_stage == "after_resize"`` (the + caller is responsible for filling before resize otherwise). + transparent_bg_fill_stage: When to composite the transparent + background — before or after the resize step. + + Returns: + A numpy array. + """ + assert isinstance(image, Image.Image), "image must be a PIL Image" + if resize_to is not None: + if mode == "resize": + image = image.resize(resize_to, resample=Image.Resampling.BICUBIC) + if transparent_bg_fill_stage == "after_resize": + image = fill_transparent_bg_with(image, transparent_bg_config) + + elif mode == "rescale_and_pad_to_center": + scale = min(resize_to[0] / image.width, + resize_to[1] / image.height, 1.0) + new_width = round(image.width * scale) + new_height = round(image.height * scale) + if new_width == 0 or new_height == 0: + if raise_error_for_ill_resize: + raise ValueError( + f"Invalid resize to: {resize_to}, from image size: {image.size}" + ) + else: + return np.zeros((resize_to[1], resize_to[0], 3), + dtype=np.uint8) + + image = image.resize((new_width, new_height), + resample=Image.Resampling.BICUBIC) + if transparent_bg_fill_stage == "after_resize": + image = fill_transparent_bg_with(image, transparent_bg_config) + padding_left = (resize_to[0] - new_width) // 2 + padding_right = resize_to[0] - new_width - padding_left + padding_top = (resize_to[1] - new_height) // 2 + padding_bottom = resize_to[1] - new_height - padding_top + image = np.asarray(image) + image = np.pad( + image, + ((padding_top, padding_bottom), (padding_left, padding_right), + (0, 0)), + mode="constant", + constant_values=0, + ) + assert image.shape == (resize_to[1], resize_to[0], 3) + + elif mode == "rescale_and_pad_to_rightbottom": + scale = min(resize_to[0] / image.width, + resize_to[1] / image.height, 1.0) + new_width = round(image.width * scale) + new_height = round(image.height * scale) + if new_width == 0 or new_height == 0: + if raise_error_for_ill_resize: + raise ValueError( + f"Invalid resize to: {resize_to}, from image size: {image.size}" + ) + else: + return np.zeros((resize_to[1], resize_to[0], 3), + dtype=np.uint8) + + image = image.resize((new_width, new_height), + resample=Image.Resampling.BICUBIC) + if transparent_bg_fill_stage == "after_resize": + image = fill_transparent_bg_with(image, transparent_bg_config) + padding_right = resize_to[0] - new_width + padding_bottom = resize_to[1] - new_height + image = np.asarray(image) + image = np.pad( + image, + ((0, padding_bottom), (0, padding_right), (0, 0)), + mode="constant", + constant_values=0, + ) + assert image.shape == (resize_to[1], resize_to[0], 3) + + else: + raise ValueError(f"Invalid mode: {mode}") + + if isinstance(image, Image.Image): + return np.asarray(image) + else: + return image + + +def navit_patchify(pixel_values: np.ndarray, + patch_size: int) -> dict[str, np.ndarray]: + """Reshape the pixel values to a navit shape. + + Args: + pixel_values: np.ndarray, shape (t, h, w, c) + patch_size: int + + Returns: + dict[str, np.ndarray] + - patches: np.ndarray, shape (t * h//patch_size * w//patch_size, c, patch_size, patch_size) + - grid_thw: np.ndarray, (t, h//patch_size, w//patch_size) + """ + T, H, W, C = pixel_values.shape + assert C == 3, "pixel_values must have 3 channels" + + patches = pixel_values.reshape(T, H // patch_size, patch_size, + W // patch_size, patch_size, C) + # (T, H//patch_size, W//patch_size, C, patch_size, patch_size) + patches = patches.transpose(0, 1, 3, 5, 2, 4) + patches = patches.reshape(-1, C, patch_size, patch_size) + grid_thw = np.array([T, H // patch_size, W // patch_size]) + return {"pixel_values": patches, "grid_thw": grid_thw} + + +def normalize(x: np.ndarray, + mean, + std_inv, + pixels_dtype: np.dtype = np.float32) -> np.ndarray: + """Normalize the image. + + Args: + x: The image to normalize. The shape is (..., 3). The dtype is uint8. The range is [0, 255]. + mean: The mean of the image. + std_inv: The inverse of the std of the image. + pixels_dtype: The dtype of the image. + Returns: + The normalized image. The shape is (..., 3). The dtype is determined by the pixels_dtype. + """ + x = (x / 255.0).astype(pixels_dtype) + x -= mean + x *= std_inv + return x + + +def _to_tensor(data, **kwargs): + import torch + + if isinstance(data, np.ndarray): + return torch.from_numpy(data).to(**kwargs) + elif isinstance(data, torch.Tensor): + return data.to(**kwargs) + elif isinstance(data, list): + return [_to_tensor(item, **kwargs) for item in data] + elif isinstance(data, tuple): + return tuple(_to_tensor(item, **kwargs) for item in data) + elif isinstance(data, dict): + return {k: _to_tensor(v, **kwargs) for k, v in data.items()} + elif data is None: + return None + else: + raise ValueError(f"Unsupported data type: {type(data)}") diff --git a/model-00001-of-000096.safetensors b/model-00001-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..915195d127790b770b422d378aacd7a169efc1c8 --- /dev/null +++ b/model-00001-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f60e41909fb16712100c311ca4402cc4e66ecc9dff9dad4ebc06f05106c58a1 +size 2341216112 diff --git a/model-00002-of-000096.safetensors b/model-00002-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7e83bb1a30a2d18a5e8431e532041633d9dbf2a4 --- /dev/null +++ b/model-00002-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5a6bf64d0ca6b9e16a64a355a3c81ef16f1187d7ebf8214749a3d036fa70399 +size 8443907752 diff --git a/model-00003-of-000096.safetensors b/model-00003-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..853742dd21fb46baef588be109d81d1c5492fd74 --- /dev/null +++ b/model-00003-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:414f04fc45595a970bc033c5e9aa05ed16dbf1cfa20a4be0381cbaec5391c104 +size 8443907752 diff --git a/model-00004-of-000096.safetensors b/model-00004-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a246e9d4d9a85fca44f938f254672f7e87641d9f --- /dev/null +++ b/model-00004-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bad81cc8cb667379b2b72ee50755aba43900952cd25b9965b03d30ced7ec7273 +size 8035371880 diff --git a/model-00005-of-000096.safetensors b/model-00005-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1c772f80d0bb5d58f190cf923be9e6b2f04051f0 --- /dev/null +++ b/model-00005-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad31d195f0622f6393e98afc7c9110b5c2de7078fed905e9b53869f2c455fba0 +size 8443907752 diff --git a/model-00006-of-000096.safetensors b/model-00006-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..51559b89012d9e27bc97d715bc63a840acc5ab31 --- /dev/null +++ b/model-00006-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff465615e53c1c4cc0bef3995f3c8bcdbb80aba56e02f123534a39c3a4812cd +size 8443907752 diff --git a/model-00007-of-000096.safetensors b/model-00007-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e42529e96ec8bb527b21c51620ac5c7da73b66bc --- /dev/null +++ b/model-00007-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e80da260b922a349b78d73f0b4b3213ddbbc0baa3493f650ae474989a27c108 +size 8443907752 diff --git a/model-00008-of-000096.safetensors b/model-00008-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e06f8121bd89fd713d0dc67b8ad3febd88a39c90 --- /dev/null +++ b/model-00008-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6310ec1c9fd51ebbb2549cc9821e8f8b43dd78b02ab96c640e8017b7346cf2d4 +size 8035371616 diff --git a/model-00009-of-000096.safetensors b/model-00009-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..602b859fbcffaf0889a467ade603c4c8bf8c40fe --- /dev/null +++ b/model-00009-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c9243097a4f8e16b00d231459fa38a0c75753f25f92fac5e54a4e6664979a9c +size 8443907752 diff --git a/model-00010-of-000096.safetensors b/model-00010-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8c85b512f19f0b9eebf76cc0ee23b05c88a0535f --- /dev/null +++ b/model-00010-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d45b032db5fde2eb06a63bc47ce60ce0594feb45599d61f93adc5cc99107af6 +size 8443907752 diff --git a/model-00011-of-000096.safetensors b/model-00011-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c64ed0d26b8dba6212535328aa21f0b02dd6efb3 --- /dev/null +++ b/model-00011-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8936e6a89cefe5bcae3261682e745c54aafc3ae147ded1548c481ecff5033267 +size 8443918536 diff --git a/model-00012-of-000096.safetensors b/model-00012-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dc1c763d096dbd02f238c499422a54c9d117ca5a --- /dev/null +++ b/model-00012-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a8d3fee8006f58205ec8ddb663e2b8b8a8da1b565d42eb9fcad2fa1ceb02787 +size 8035382544 diff --git a/model-00013-of-000096.safetensors b/model-00013-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cc4c2f5c739a1e3b27446eb10caaefab65597d41 --- /dev/null +++ b/model-00013-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6edcdb1415f805e6ab5fc11fd128d51e37f2c95d9db02ba92c9c8ea10472ab4f +size 8443918536 diff --git a/model-00014-of-000096.safetensors b/model-00014-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0368d40d263fc7ef9074fde25a72fe2f388084d6 --- /dev/null +++ b/model-00014-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eef14ff496a345b613d0bc79505a27a443684d49b57c923ebbee478828fa5de5 +size 8443918536 diff --git a/model-00015-of-000096.safetensors b/model-00015-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..abd4f275bc106d143a113daa8a2bb7a4751948ea --- /dev/null +++ b/model-00015-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b03fd28949e8a4290541ad2b502f64e35b092e15ba7ce428b8be63697ed2436e +size 8443918536 diff --git a/model-00016-of-000096.safetensors b/model-00016-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7f3d608bb9b512e262dc505016bc30447e24d316 --- /dev/null +++ b/model-00016-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9182a75c5af6af393758f055ec9153d131c6f8d5bc8fe5dcdf023906ed1032e +size 8035382512 diff --git a/model-00017-of-000096.safetensors b/model-00017-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2cf3cb7faba1e72a8a0655625c5bf433c8931deb --- /dev/null +++ b/model-00017-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab21a3d0aaa624909babf6c879540ec2558f39c117b727671c20588777b6d40a +size 8443918536 diff --git a/model-00018-of-000096.safetensors b/model-00018-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9bd884c920cf7a0327637a323ae94637e0c8ccea --- /dev/null +++ b/model-00018-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:540e1985ee4fe1c92bb503409a1cc4c4f321ee071d8a3d3cff8be6b05e139419 +size 8443918536 diff --git a/model-00019-of-000096.safetensors b/model-00019-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..235404759d5bd6695fe685c2cd65533c3f7fc8e8 --- /dev/null +++ b/model-00019-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95a39df5792b4ddcbc31bd7c177087d60699f66540761674953e6368cc270212 +size 8443918536 diff --git a/model-00020-of-000096.safetensors b/model-00020-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cc92685fd1980f1ef93b6e18332b03b559077dc7 --- /dev/null +++ b/model-00020-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a6b43f464c0712f60b8fa31fb20f245a043f237df21eaa7823dbe62f04f0e74 +size 8035382504 diff --git a/model-00021-of-000096.safetensors b/model-00021-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e6eda9bbbb9b1548c0f49ba3388782121d752910 --- /dev/null +++ b/model-00021-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c99a4340970bfaeb899b8cf173c38729295ace0a2266e994c3d2c0753e5739d4 +size 8443918536 diff --git a/model-00022-of-000096.safetensors b/model-00022-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4e8ffc4e991b79d1a653dcc78e9b025f47351fa3 --- /dev/null +++ b/model-00022-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:593c4fe5299bb8f23addad642d812dbd87a0c0f400b1f026497321a449b27124 +size 8443918536 diff --git a/model-00023-of-000096.safetensors b/model-00023-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..294acabe72ee216b6fd3e16e3b4d5c447a948e24 --- /dev/null +++ b/model-00023-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cb2c2a3c27252cbfb1b959d9d1ec68ab1823fca91caab1f316f6c2eb06733dc +size 8443918536 diff --git a/model-00024-of-000096.safetensors b/model-00024-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8208c2e0d59059137becf40cf42d9815ca900f9c --- /dev/null +++ b/model-00024-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9ada09296e8d1b34c2f81cd3d7e18838a4690867f4387ce6b9dab2ffe7ecace +size 8035382440 diff --git a/model-00025-of-000096.safetensors b/model-00025-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b2e6c465f4476878a8f0c0fb88b31f5bdbf1f636 --- /dev/null +++ b/model-00025-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e153cc58cf7153c99807f6ce6f0d27e198f2aecf7d76a36d3a94f34b268d613 +size 8443918536 diff --git a/model-00026-of-000096.safetensors b/model-00026-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c072ae58180b926605c9da9485c3ccf7c2a53b52 --- /dev/null +++ b/model-00026-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7cd26fd82533eac62e13511433f3160098eeb29e5e9500c9a3ce4c045870a04 +size 8443918536 diff --git a/model-00027-of-000096.safetensors b/model-00027-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e744017293aee034e6b5acbd4411a77c7b9d7895 --- /dev/null +++ b/model-00027-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ae3cc89ebc51398d607f2548a2d6597a6cfbaa3e1d4b6fdb453657138e9077f +size 8443918536 diff --git a/model-00028-of-000096.safetensors b/model-00028-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..67a7493c5bdde84b8af309f002104be642f45bc0 --- /dev/null +++ b/model-00028-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:119f61e09b65ba60d516e00577026665869481beb5f705322993c1bfbf58ed8a +size 8035382480 diff --git a/model-00029-of-000096.safetensors b/model-00029-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..71ae42de666fd833b983241a68db7bb9b12f0dc4 --- /dev/null +++ b/model-00029-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:485ce0e6dfb4b0fbb298d39fd4bb16d207fa84117f033ac8056cf0ff57b7dafd +size 8443918536 diff --git a/model-00030-of-000096.safetensors b/model-00030-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cf4ec0bc2cf4081bc1ad2c7f8387acddafbf0f93 --- /dev/null +++ b/model-00030-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:277e79a47b8bf9a7857fcabf0532a7f60c54e3d3d340b6c172748e7d7c552ed5 +size 8443918536 diff --git a/model-00031-of-000096.safetensors b/model-00031-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4d8265f05d01eb80dbdd0962ec973fe7b3337313 --- /dev/null +++ b/model-00031-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57e8680d1db9e828d9c71cd43043764784fa6f97b6d2e76a3670b32bb36566cb +size 8443918536 diff --git a/model-00032-of-000096.safetensors b/model-00032-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d0ec65b0483f5fc26eefdb94823c7af4fd078fc7 --- /dev/null +++ b/model-00032-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:898298f125564348dbef22bdaa43dead700a3d34ed1b5c243d09e5e5ddc2f53e +size 8035382480 diff --git a/model-00033-of-000096.safetensors b/model-00033-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9f5d06fc12e3f36113f410f519fc200a14d36d74 --- /dev/null +++ b/model-00033-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2903890c34b68cfb2188217a601f7767756a62951a004c2573d0ad3d9592bb6 +size 8443918536 diff --git a/model-00034-of-000096.safetensors b/model-00034-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..161f481d974864bda71ca6d776c4aebd00dd3b8f --- /dev/null +++ b/model-00034-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d41653ae0bf0e06fbc419fe792bc3f5c4475a3def1f25424bc5f73bc9ad7539 +size 8443918536 diff --git a/model-00035-of-000096.safetensors b/model-00035-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..973b1b43085f8d46120b79358191045d054563bd --- /dev/null +++ b/model-00035-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adfb3e3b10d70c4f5c23a7a70caefcd5df545ac9369e15202140779f76918ec5 +size 8443918536 diff --git a/model-00036-of-000096.safetensors b/model-00036-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..daaad67787f8f3d93f59eb0fe1afb11664a0ebdc --- /dev/null +++ b/model-00036-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:308e9f6a822b3d70f276531fea83fa5ab61e8700f4226609ea0a58a4157857ba +size 8035382536 diff --git a/model-00037-of-000096.safetensors b/model-00037-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..62671e803cf1aedcd3b862887e7b9b7332311ad6 --- /dev/null +++ b/model-00037-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbca8958a4d645d4e8bf5454b49ce248b33aa96fb6c27d67ee7e88775b348a61 +size 8443918536 diff --git a/model-00038-of-000096.safetensors b/model-00038-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6d564c61171a1bd750108f36dfe6b419344cbcb5 --- /dev/null +++ b/model-00038-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6527ff8bf01faa186d9741343ca3494fbad92ba644269f696c0fc0fdc244eb58 +size 8443918536 diff --git a/model-00039-of-000096.safetensors b/model-00039-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7e5a2b8dddf40a6f22818cba8f021eb3fcfed90e --- /dev/null +++ b/model-00039-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0348c800203930e892ffd026e7d23c7f5eda6137063dc34ee1077c7a038fe95a +size 8443918536 diff --git a/model-00040-of-000096.safetensors b/model-00040-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f9cf34e04eeb80a582e68673357ba86155661e10 --- /dev/null +++ b/model-00040-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd6cd85edefdc123c46b2d8945d2d1eff050fc779fc4be635d586304910965dd +size 8035382544 diff --git a/model-00041-of-000096.safetensors b/model-00041-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8bb04b323798c1bb8c62f8a856bc2a0c4d4c7275 --- /dev/null +++ b/model-00041-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d37fcb5de61534657390c6115ff61162e5bad5aec95da8c2db488b65e07bb03a +size 8443918536 diff --git a/model-00042-of-000096.safetensors b/model-00042-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dc5fa4192321027b3a73ac81ea4dd4997af6e254 --- /dev/null +++ b/model-00042-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6887ab1615b709519ed6e9ab0d0873b4452ba2e71e4760fc0411789130288e49 +size 8443918536 diff --git a/model-00043-of-000096.safetensors b/model-00043-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2ad5df70074cc7e2fc004c73d11caa2ecedbc558 --- /dev/null +++ b/model-00043-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:156c2091207ef2d54992e3c605c677ef1d9e3ae7f0043642fc9801492fe0ca76 +size 8443918536 diff --git a/model-00044-of-000096.safetensors b/model-00044-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bb266e6e33bb572317153425d2427e9d715ac1d4 --- /dev/null +++ b/model-00044-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b31dbbeb70abea68af5cc023885f1e89adffbd4246c0bcbaba9a6c3eba6cf4d9 +size 8035382656 diff --git a/model-00045-of-000096.safetensors b/model-00045-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ab66e6c075aa845397a078ef2faade4ce6ed921e --- /dev/null +++ b/model-00045-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff161ce5ad9664c3558f8b9b0cf56c2342033e43181f386d149800615866fc90 +size 8443918536 diff --git a/model-00046-of-000096.safetensors b/model-00046-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fc5116f182c01cd03f4c1ae4ff8c641909e9151a --- /dev/null +++ b/model-00046-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1fb2abd98f2bd584e06c1cb601fd0cc2933ff3ccbde90f085815e53118b6c08 +size 8443918536 diff --git a/model-00047-of-000096.safetensors b/model-00047-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..37a464a36b1ac4424a08392a858e3095280260b3 --- /dev/null +++ b/model-00047-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22bf91c568c0d9314df541b7f60c7f25bd9a6f63d976db048ccbe8bfdaed1adb +size 8443918536 diff --git a/model-00048-of-000096.safetensors b/model-00048-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d358f5eb6e6a3c7e7ebcb0ec7fb084e2e8913b42 --- /dev/null +++ b/model-00048-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a815c2d63fa550a6407bd00183c93763a77497851013c8f2740eff71e03bbe1d +size 8035382320 diff --git a/model-00049-of-000096.safetensors b/model-00049-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..50cabdf190cec4ad1310b3bf08ae4cf33278612f --- /dev/null +++ b/model-00049-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efdfcccbbb4587fa594c64ec76e633446e80cec04efa2b57a3519c0c588b731c +size 8443918536 diff --git a/model-00050-of-000096.safetensors b/model-00050-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c48fd166ee57dc09350f6d9465169e806e32b943 --- /dev/null +++ b/model-00050-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:549ad47370d37123d23592874cafab69d6b39fb0c5ce7aa1060d5163aa2116bf +size 8443918536 diff --git a/model-00051-of-000096.safetensors b/model-00051-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9e16c01446677f5f3858218ebd6c28b5dd6a2a27 --- /dev/null +++ b/model-00051-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8caabd0bab22e505e0c4aee9ad174e2cb402e03437e7c346f6eb17ca739929b2 +size 8443918536 diff --git a/model-00052-of-000096.safetensors b/model-00052-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a0005151dcb975d92e6fdaabbf974fc2aeb20c1a --- /dev/null +++ b/model-00052-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f4359b7adc635f826376725dad3a3c4a358d6eb9c6b4aaad2565e4730667669 +size 8035382480 diff --git a/model-00053-of-000096.safetensors b/model-00053-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b4e815d8179ecb8faa7844ef9331cb1928fcad67 --- /dev/null +++ b/model-00053-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2bc7a09504feab761f59d9d5f69acedad3b718a95d3904774a27c3c3406d403 +size 8443918536 diff --git a/model-00054-of-000096.safetensors b/model-00054-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb67769a49971374c3d242978faa85ea8b247b9e --- /dev/null +++ b/model-00054-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bab04358b3e991ba6085aed26df546260167999f76c668f6ef0fb0686874e12e +size 8443918536 diff --git a/model-00055-of-000096.safetensors b/model-00055-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a69a3ac6cbfd3e561335348eb42e1279f071098d --- /dev/null +++ b/model-00055-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e3a000e27b0597a97ee1f5e6322946f1df300ffcc2bf06dffcf983b76b27e01 +size 8443918536 diff --git a/model-00056-of-000096.safetensors b/model-00056-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..aa52194cb9078f93c5ab5000d43df92a358354a8 --- /dev/null +++ b/model-00056-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:993b134e0f3df1dacef6d919d395aab33865701448cf9357083570487d7464f6 +size 8035382464 diff --git a/model-00057-of-000096.safetensors b/model-00057-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..74c342e7c78856a7af34c7eff9ec2b4c01276e53 --- /dev/null +++ b/model-00057-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c8ab13025795e05fd5ac9bf7deb4056fe10a9cf6b589c15c2c70257416a6ee8 +size 8443918536 diff --git a/model-00058-of-000096.safetensors b/model-00058-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6949019dbb4654cdd5ca6f3562aea3044d1820c9 --- /dev/null +++ b/model-00058-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a7bdcf4a85710f28eae5a04237b17a726817119d24ea2253eeec73a4b1d0970 +size 8443918536 diff --git a/model-00059-of-000096.safetensors b/model-00059-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0efdb3f9bb18e082d7ca4cff423e1a975fdbd087 --- /dev/null +++ b/model-00059-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:766e93e382354a092e45599a07f24b745cc0cefac83bd53861ad18c53f0e0237 +size 8443918536 diff --git a/model-00060-of-000096.safetensors b/model-00060-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..64ba63dc33ef3e73e93075213ce079062123ba68 --- /dev/null +++ b/model-00060-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79b252d5297d068ff798bbcd5d5d705fdf1a13b0a558f59e0770d31585b919e4 +size 8035382512 diff --git a/model-00061-of-000096.safetensors b/model-00061-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..023e5be5add6dbf28b7c91d42c94de8d5b9e292b --- /dev/null +++ b/model-00061-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4d8ed46cca9e51f8d0cb18824b15897996a9f2f270ae0a3b42a58d995b2d4c6 +size 8443918536 diff --git a/model-00062-of-000096.safetensors b/model-00062-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..800c5c25a1243b531e9a40bcbc486c1dc6982b33 --- /dev/null +++ b/model-00062-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c97fe7fa69b27b705aec2c9bc4291ea0decc54f426d8e35001fc27b7c49818b0 +size 8443918536 diff --git a/model-00063-of-000096.safetensors b/model-00063-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d7744e5a77dab15f2e82a345a6c070934e166b08 --- /dev/null +++ b/model-00063-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ada3296e8644a37ef0251c75a155161276786a06ac539607c81bf8861cc20e9 +size 8443918536 diff --git a/model-00064-of-000096.safetensors b/model-00064-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1e7f1d0ce1c0ec1e56bb80f1963c3ae0c67f86b6 --- /dev/null +++ b/model-00064-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60149199c574af3326cd22135c13ed1b1efc28e45921bcadb6bcb931c4313534 +size 8035382288 diff --git a/model-00065-of-000096.safetensors b/model-00065-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0bc2aa680f982513844be15b009c6d04aa9c5380 --- /dev/null +++ b/model-00065-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e58476018d3a7e473840ce11c18a3c462dabedb1d6fcc07b61d5c855e6db1bf +size 8443918536 diff --git a/model-00066-of-000096.safetensors b/model-00066-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5d28bf7e901cfa047a7c660ed3de1b1b40cc9e97 --- /dev/null +++ b/model-00066-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ea97c8820a21046a95ee122eba04aaa1f5a28674a991b5ee607e500d58019e3 +size 8443918536 diff --git a/model-00067-of-000096.safetensors b/model-00067-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5178bd248293b2ef65b95dbb2786fff6a2602294 --- /dev/null +++ b/model-00067-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7990da8888f4613eea67e43e02a709145692b1e5c9a1c3e22b982c9b7d2af7e2 +size 8443918536 diff --git a/model-00068-of-000096.safetensors b/model-00068-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6fdb2e8b4746bf158f76c7b592ef915cb37a377e --- /dev/null +++ b/model-00068-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4825cbab5f5b085c8b1d6695e39c22aa9a425c65ee6e66bbc1925eff52b71d49 +size 8035382472 diff --git a/model-00069-of-000096.safetensors b/model-00069-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..be5ac57bd427a4c8172357fd57f2f705683051ff --- /dev/null +++ b/model-00069-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ed8c2d372f1c268579b5590e8481112c8324f2927cc5eb146cdfb1d15d2a897 +size 8443918536 diff --git a/model-00070-of-000096.safetensors b/model-00070-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9510c0405cd31bb816baa5a8a1758f4cf1250a1a --- /dev/null +++ b/model-00070-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f79db5b299ef4583cd95d27726e30751a1da25275f25dd93d40e1dcadee35c63 +size 8443918536 diff --git a/model-00071-of-000096.safetensors b/model-00071-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..62dc8ef4c74086ec28c3da7f1cb3290994db1da4 --- /dev/null +++ b/model-00071-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:444288d4df9b730faca2ea8a3297932c493a2c9a332b904dfbcb8e73b4b31d43 +size 8443918536 diff --git a/model-00072-of-000096.safetensors b/model-00072-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..279530cee9d97dc06140a6a65b5a9a068165bc0a --- /dev/null +++ b/model-00072-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:590dd671d55bb39a1f30e6668705e2c2c4104fe4c55c5d529c5f5a09e9326d7c +size 8035382568 diff --git a/model-00073-of-000096.safetensors b/model-00073-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4e0d6707e1ed918b948c0ff33d868878a5dbc82e --- /dev/null +++ b/model-00073-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:739bbc74427370a8f1142d1cac50cfa7980b610bf33e7fe2db3ae902b328a726 +size 8443918536 diff --git a/model-00074-of-000096.safetensors b/model-00074-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b4089e46bd2a2590d9c5024c06505d72cc01e641 --- /dev/null +++ b/model-00074-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25b0f91e56a361fb4ea714de0bf54cccbe5e138c77d8f23fc0b8ab1b4b848de8 +size 8443918536 diff --git a/model-00075-of-000096.safetensors b/model-00075-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..329d0bf7f309742e4e5697a437b1904dc786e173 --- /dev/null +++ b/model-00075-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0a07439fedfda916cf0804fef9e1d9421234aa6cf5e808a8ebe089b88489aa6 +size 8443918536 diff --git a/model-00076-of-000096.safetensors b/model-00076-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5e0ec781f6b57c7d18b375a10353092a45f03fa9 --- /dev/null +++ b/model-00076-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dedf3bbd2e1ecac2b2e82d2324f507ab7c444027a2e1e53bcdbdc34afb85041 +size 8035382576 diff --git a/model-00077-of-000096.safetensors b/model-00077-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7fc966e6a58b44e5aeedf6f48b96ec6404225271 --- /dev/null +++ b/model-00077-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d12251f6f408b267d9af01e748d269c151f5d54295e8c5e204b2aaa3e52d691d +size 8443918536 diff --git a/model-00078-of-000096.safetensors b/model-00078-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4acb668e935a81e65f7abac71e0de4b4619ace44 --- /dev/null +++ b/model-00078-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9b95e8faa04100f3eed9e0f64415cc5ba1a98a2e341bb39d1a067215407297d +size 8443918536 diff --git a/model-00079-of-000096.safetensors b/model-00079-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f1901af3b0bc7444bf401896b5809869fb9ef7ef --- /dev/null +++ b/model-00079-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f12f9929ec6fae980197da418f337195c49d6752f770e80f6d9df4eb7abf64aa +size 8443918536 diff --git a/model-00080-of-000096.safetensors b/model-00080-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0ee10b7921661aa0aa9ccbb4a09539577f9f52ba --- /dev/null +++ b/model-00080-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fa9a38b521330384a1ec4a1c7af61e61dc854d2070962643208ce71f80d9386 +size 8035382432 diff --git a/model-00081-of-000096.safetensors b/model-00081-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0ecbb33d3f001afa95ec79e4e101cbcd45958447 --- /dev/null +++ b/model-00081-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e1e4cea857bea83f4ca4e3b91029e7a7b14b0b30780e24227b28cfe87b4f5d8 +size 8443918536 diff --git a/model-00082-of-000096.safetensors b/model-00082-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2a21be767478012ca6bb10f89e09cfcada5bda19 --- /dev/null +++ b/model-00082-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35b8b99af7cfb6709b1a81c7a8c1bd522189357dff4a6a3814ed2caccca10c97 +size 8443918536 diff --git a/model-00083-of-000096.safetensors b/model-00083-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e6f6ca7655e22e7cc8d8a5b2ba946f68ea0e5edd --- /dev/null +++ b/model-00083-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e13e3e4b5db4c3644b541039ced6b55f50a5af9a9b6c74c6105f7fce4d38cf5 +size 8443918536 diff --git a/model-00084-of-000096.safetensors b/model-00084-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cdaaf3136785b20c9b952a085d5241e2c88f4aad --- /dev/null +++ b/model-00084-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59f13e148777ce28ac77f7c2b75fa1ea476719cbfa868e8ebc931e3ae3b3ca81 +size 8035382584 diff --git a/model-00085-of-000096.safetensors b/model-00085-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..73aa200138e20d9372becfcae2947ec9b5f24e92 --- /dev/null +++ b/model-00085-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32a1ad940250bae1801725fa462fa76ab8ea73ee4173960611bb3231a1afd3b6 +size 8443918536 diff --git a/model-00086-of-000096.safetensors b/model-00086-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5e8d77deff979a9f08e48e1e40ce647ae4793691 --- /dev/null +++ b/model-00086-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be5db6a47cc74d7d5521f54e4c874449aa4aaccb1c8d98f714efa6c8ceb82ea8 +size 8443918536 diff --git a/model-00087-of-000096.safetensors b/model-00087-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..27bfa0627fc714d02d5c6274ead426e38a680408 --- /dev/null +++ b/model-00087-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99c6bf8cd02d746e28cb4fd3a518f994029699984f81bb7045c694a54ab7b9c3 +size 8443918536 diff --git a/model-00088-of-000096.safetensors b/model-00088-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..323c1a9a0211343f47751fc1a42c54c475078a35 --- /dev/null +++ b/model-00088-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39878183f767214de0e08fa5e97b657ca66a418e0dfaa605c7a018468b1ac123 +size 8035382648 diff --git a/model-00089-of-000096.safetensors b/model-00089-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bc5e950742c97bfed34f1cd727c25f753509d3bc --- /dev/null +++ b/model-00089-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f79dd4bf2d6b915bd844242886fc02ea2ce74d69b9b008f25c97b90003c592b +size 8443918536 diff --git a/model-00090-of-000096.safetensors b/model-00090-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2a7697a732c4ee31375f2f095b613d4189efee73 --- /dev/null +++ b/model-00090-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56235bfacdd0a582538c2e68d4dd33268673d8dc3dc5b7b847c10032f9fb76d9 +size 8443918536 diff --git a/model-00091-of-000096.safetensors b/model-00091-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..70f5f7bdc3e5eb7d3e41d1f235d4270b29ac7da6 --- /dev/null +++ b/model-00091-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2409e91c1c45d6aa17e98da1230e6d3e5ac243219e653a0f129d62d6be9098ba +size 8443918536 diff --git a/model-00092-of-000096.safetensors b/model-00092-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a63354b62638f5fb9a429f52d6784cc69ecd2bb1 --- /dev/null +++ b/model-00092-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2b2f19ca5f374fce6f927a85b06ab5bbfcb83ff8a3f5977e1427db4508db2c6 +size 8035382512 diff --git a/model-00093-of-000096.safetensors b/model-00093-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..30e1ad57fd6b9e714485f2c6533657935190c2a3 --- /dev/null +++ b/model-00093-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:091be0d17dc69a0093f77ccae659dd039675ee5ebe31c1e416f05980c05abd27 +size 8035382576 diff --git a/model-00094-of-000096.safetensors b/model-00094-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a67302cad5858f6c341271e06f8c4f91336344e9 --- /dev/null +++ b/model-00094-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f60884ddcd7dc3d2c95640cf14626ae17702fa489d2612849433d7290c4358ff +size 4697664072 diff --git a/model-00095-of-000096.safetensors b/model-00095-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b94638ece80201bd6df7fc116a1093867ba3ecfb --- /dev/null +++ b/model-00095-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db95f23534af1cb3b09d321d632b87f2103865fda2e96734c83379d80c114cfe +size 92289328 diff --git a/model-00096-of-000096.safetensors b/model-00096-of-000096.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1efdad31fc1a0dac902585d53b0254be0356a32d --- /dev/null +++ b/model-00096-of-000096.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1912e9c59d01fe3962cab478be9c7546482b35ce2c19bea63dc990ee0fdcd4c2 +size 802448352 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..8d941576cd1fee45e3b12a0d9a7a69282a19f5ab --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1acc7e213ca92ac9bde948ef1b271f8b7a9296c032824847f6236b5b5ebdc339 +size 113399932 diff --git a/preprocessor_config.json b/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8be28c13c93fb8982f81928d3f4fe80568102286 --- /dev/null +++ b/preprocessor_config.json @@ -0,0 +1,38 @@ +{ + "media_proc_cfg": { + "in_patch_limit": 65536, + "patch_size": 14, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "merge_kernel_size": 2, + "fixed_output_tokens": null, + "patch_limit_on_one_side": 512, + "in_patch_limit_each_frame": 16384, + "in_patch_limit_video": 655360, + "sample_fps": 8.0, + "max_num_frames_each_video": null, + "temporal_merge_kernel_size": 4, + "timestamp_mode": "hh:mm:ss.fff", + "transparent_bg_config": { + "pattern": "chessboard", + "chessboard_square_size": 8, + "chessboard_square_on_top_left": true, + "chessboard_white_value": 255, + "chessboard_gray_value": 180 + }, + "transparent_bg_fill_stage": "after_resize", + "config_type": "media_proc.processors.moonvit.MoonViTMediaProcessorConfig" + }, + "auto_map": { + "AutoProcessor": "kimi_k3_processor.KimiK3Processor", + "AutoImageProcessor": "kimi_k3_vision_processing.KimiK3VisionProcessor" + } +} \ No newline at end of file diff --git a/strata_coder/decoder.py b/strata_coder/decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..00af00e1e9395af6b0073776195293f3189b8642 --- /dev/null +++ b/strata_coder/decoder.py @@ -0,0 +1,32 @@ +import torch + +@torch.no_grad() +def unpack_sc_lut_indices(packed: torch.Tensor, n_tiles: int, bits: int) -> torch.Tensor: + R = packed.shape[0] + if bits == 10: + p = packed.view(R, n_tiles // 4, 5).to(torch.int32) + i0 = p[:, :, 0] | ((p[:, :, 1] & 0x03) << 8) + i1 = (p[:, :, 1] >> 2) | ((p[:, :, 2] & 0x0F) << 6) + i2 = (p[:, :, 2] >> 4) | ((p[:, :, 3] & 0x3F) << 4) + i3 = (p[:, :, 3] >> 6) | (p[:, :, 4] << 2) + return torch.stack([i0, i1, i2, i3], dim=-1).reshape(R, n_tiles) + if bits == 9: + p = packed.view(R, n_tiles // 8, 9).to(torch.int64) + lo = torch.zeros((R, n_tiles // 8), dtype=torch.int64, device=packed.device) + for k in range(8): + lo |= (p[:, :, k] & 0xFF) << (8 * k) + out = [((lo >> (9 * t)) & 0x1FF).to(torch.int32) for t in range(7)] + i7 = (((lo >> 63) & 0x01) | ((p[:, :, 8] & 0xFF) << 1)).to(torch.int32) + out.append(i7) + return torch.stack(out, dim=-1).reshape(R, n_tiles) + raise ValueError(f"Unsupported bit width: {bits}") + +@torch.no_grad() +def decode_sc_lut_matrix(idx_or_packed, scales, u_svd, v_svd, lut, q_basis, bits=0, out_dtype=torch.bfloat16): + R, C = u_svd.shape[0], v_svd.shape[1] + tile = lut.shape[1] + n_tiles = C // tile + idx = unpack_sc_lut_indices(idx_or_packed, n_tiles, bits) if bits > 0 else idx_or_packed + w_rot = lut.float()[idx.long()].reshape(R, scales.shape[1], -1) * scales.float().unsqueeze(-1) + w_main = (w_rot.reshape(R, C // 64, 64) @ q_basis.float()).reshape(R, C) + return (w_main + (u_svd.float() @ v_svd.float())).to(out_dtype) diff --git a/tiktoken.model b/tiktoken.model new file mode 100644 index 0000000000000000000000000000000000000000..b4149a6e17a01b6442187f39890f89bc2fe8d309 --- /dev/null +++ b/tiktoken.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6c497a7469b33ced9c38afb1ad6e47f03f5e5dc05f15930799210ec050c5103 +size 2795286 diff --git a/tokenization_kimi.py b/tokenization_kimi.py new file mode 100644 index 0000000000000000000000000000000000000000..7ae891c2e0ef050bbeb2da55e9bc0401c4c508db --- /dev/null +++ b/tokenization_kimi.py @@ -0,0 +1,412 @@ +import os +from logging import getLogger +from pathlib import Path +from shutil import copyfile +from typing import Dict, Iterator, List, Optional, Tuple, Union, cast + +import tiktoken +from tiktoken.load import load_tiktoken_bpe +from tokenizers import AddedToken +from transformers.convert_slow_tokenizer import bytes_to_unicode +from transformers.tokenization_utils import PreTrainedTokenizer + +try: + from .encoding_k3 import build_chat_segments, is_batched_conversation +except ImportError: # pragma: no cover - supports direct file execution/import. + from encoding_k3 import build_chat_segments, is_batched_conversation + +logger = getLogger(__name__) +VOCAB_FILES_NAMES = {"vocab_file": "tiktoken.model"} + + +class TikTokenTokenizer(PreTrainedTokenizer): + """ + Tokenizing and encoding/decoding text using the Tiktoken tokenizer. See megatron/tokenizer/tiktoken_tokenizer.py. + + This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to + this superclass for more information regarding those methods. + + Args: + vocab_file (`str`): + The path to the Tiktoken model file. + bos_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<|begin_of_text|>",`): + The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token. + eos_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<|end_of_text|>"`): + The end of sequence token. + unk_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<|reserved_special_token_249|>"`): + The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this + token instead. The second to last item in special_tokens. + pad_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<|reserved_special_token_250|>"`): + The token used for padding, for example when batching sequences of different lengths. + additional_special_tokens (list of `str`, *optional*): + A tuple or a list of additional tokens, which will be marked as `special`, meaning that they will be + skipped when decoding if `skip_special_tokens` is set to `True`. + """ + + vocab_files_names = VOCAB_FILES_NAMES + + model_input_names = ["input_ids", "attention_mask"] + + special_tokens: Dict[str, int] + + num_reserved_special_tokens = 256 + + pat_str = "|".join([ + r"""[\p{Han}]+""", + r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?""", + r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?""", + r"""\p{N}{1,3}""", + r""" ?[^\s\p{L}\p{N}]+[\r\n]*""", + r"""\s*[\r\n]+""", + r"""\s+(?!\S)""", + r"""\s+""", + ]) + + def __init__( + self, + vocab_file, + bos_token: Union[str, AddedToken] = "[BOS]", + eos_token: Union[str, AddedToken] = "[EOS]", + unk_token: Union[str, AddedToken, None] = None, + pad_token: Union[str, AddedToken, None] = None, + additional_special_tokens: List[str] = None, + added_tokens_decoder: Optional[dict] = None, + **kwargs, + ): + assert os.path.isfile(vocab_file), vocab_file + + if additional_special_tokens is None: + additional_special_tokens = [ + "<|im_end|>", + "<|im_user|>", + "<|im_assistant|>", + "<|start_header_id|>", + "<|end_header_id|>", + "[EOT]", + "<|im_system|>", + "<|im_middle|>", + ] + + if added_tokens_decoder: + special_tokens_mapping = { + i: added_tokens_decoder[i].content + for i in added_tokens_decoder + } + else: + special_tokens_mapping = {} + + self.vocab_file = vocab_file + mergeable_ranks = load_tiktoken_bpe(vocab_file) + num_base_tokens = len(mergeable_ranks) + self.special_tokens = { + special_tokens_mapping.get(i, f"<|reserved_token_{i}|>"): i + for i in range(num_base_tokens, num_base_tokens + + self.num_reserved_special_tokens) + } + + self.model = tiktoken.Encoding( + name=Path(vocab_file).name, + pat_str=self.pat_str, + mergeable_ranks=mergeable_ranks, + special_tokens=self.special_tokens, + ) + logger.info(f"Reloaded tiktoken model from {vocab_file}") + + self.n_words: int = self.model.n_vocab + # BOS / EOS token IDs + self.bos_id: int = self.special_tokens[str(bos_token)] + self.eos_id: int = self.special_tokens[str(eos_token)] + logger.info( + f"#words: {self.n_words} - BOS ID: {self.bos_id} - EOS ID: {self.eos_id}" + ) + + self.pad_id: int = self.special_tokens[str(pad_token)] + self.unk_id: int = self.special_tokens[str(unk_token)] + + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + + self.decoder = {} + for i in range(self.n_words): + # Taken from https://gist.github.com/xenova/a452a6474428de0182b17605a98631ee + decoding = ''.join([ + self.byte_encoder[ord(char)] for char in + self.model.decode_single_token_bytes(i).decode('latin-1') + ]) + self.decoder[i] = decoding + + self.encoder = {} + for i in range(self.n_words): + if i in self.decoder: + self.encoder[self.decoder[i]] = i + + super().__init__( + bos_token=bos_token, + eos_token=eos_token, + unk_token=unk_token, + pad_token=pad_token, + additional_special_tokens=additional_special_tokens, + added_tokens_decoder=added_tokens_decoder, + **kwargs, + ) + self.all_special_ids_set = set(self.all_special_ids) + + def _encode_text_piece(self, + text: str, + allow_special_tokens: bool = True) -> List[int]: + # The tiktoken tokenizer can handle <=400k chars without + # pyo3_runtime.PanicException. + TIKTOKEN_MAX_ENCODE_CHARS = 400_000 + + # https://github.com/openai/tiktoken/issues/195 + # Here we iterate over subsequences and split if we exceed the limit + # of max consecutive non-whitespace or whitespace characters. + MAX_NO_WHITESPACES_CHARS = 25_000 + + t: List[int] = [] + for i in range(0, len(text), TIKTOKEN_MAX_ENCODE_CHARS): + for substr in self._split_whitespaces_or_nonwhitespaces( + text[i:i + TIKTOKEN_MAX_ENCODE_CHARS], + MAX_NO_WHITESPACES_CHARS, + ): + if allow_special_tokens: + t.extend( + # structural markers: encode <|...|> as their special token IDs + self.model.encode( + substr, + allowed_special="all", + )) + else: + t.extend( + # user/tool text: encode any <|...|> as ordinary BPE tokens (never as control tokens) + self.model.encode( + substr, + disallowed_special=(), + )) + + return t + + def encode(self, + text: str, + allow_special_tokens: bool = True, + **kwargs) -> List[int]: + """ + Encodes a string into a list of token IDs. + + Args: + text (str): The input string to be encoded. + + Returns: + list[int]: A list of token IDs. + """ + # If there are other args, we should call super().encode because there are a lot of code + # to handle those args. supper().encode finally will call _tokenize and _convert_token_to_id. + # NOTE: our encode method is not compatible with the super().encode method, + # e.g. split_special_tokens' default is True in our encode method. + if len(kwargs) > 0: + logger.warning(f"Calling super().encode with {kwargs}") + return super().encode(text, **kwargs) + + assert type(text) is str + return self._encode_text_piece( + text, allow_special_tokens=allow_special_tokens) + + def decode(self, token_ids: Union[int, List[int]], **kwargs) -> str: + """ + Decodes a list of token IDs into a string. + + Args: + token_ids (List[int]): The list of token IDs to be decoded. + + Returns: + str: The decoded string. + """ + # If there are other args, we should call super().decode because there are a lot of code + # to handle those args. supper().encode finally will call convert_tokens_to_string and _convert_id_to_token. + if len(kwargs) > 0: + return super().decode(token_ids, **kwargs) + + if type(token_ids) is int: + token_ids = [token_ids] + + return self.model.decode(cast(List[int], token_ids)) + + @staticmethod + def _split_whitespaces_or_nonwhitespaces( + s: str, max_consecutive_slice_len: int) -> Iterator[str]: + """ + Splits the string `s` so that each substring contains no more than `max_consecutive_slice_len` + consecutive whitespaces or consecutive non-whitespaces. + """ + current_slice_len = 0 + current_slice_is_space = s[0].isspace() if len(s) > 0 else False + slice_start = 0 + + for i in range(len(s)): + is_now_space = s[i].isspace() + + if current_slice_is_space ^ is_now_space: + current_slice_len = 1 + current_slice_is_space = is_now_space + else: + current_slice_len += 1 + if current_slice_len > max_consecutive_slice_len: + yield s[slice_start:i] + slice_start = i + current_slice_len = 1 + yield s[slice_start:] + + def _encode_chat_segments(self, segments) -> List[int]: + token_ids: List[int] = [] + for segment in segments: + token_ids.extend( + self._encode_text_piece( + segment.text, + allow_special_tokens=segment.allow_special, + )) + return token_ids + + @staticmethod + def _truncate(ids: List[int], + truncation: bool = False, + max_length: Optional[int] = None) -> List[int]: + if truncation and max_length is not None: + return ids[:max_length] + return ids + + def _format_chat_token_output(self, + encoded_inputs: List[List[int]], + *, + is_batched: bool, + padding=False, + truncation: bool = False, + max_length: Optional[int] = None, + return_tensors=None, + return_dict: bool = False): + encoded_inputs = [ + self._truncate(ids, truncation=truncation, max_length=max_length) + for ids in encoded_inputs + ] + + needs_batch_encoding = (is_batched or padding + or return_tensors is not None or return_dict) + if not needs_batch_encoding: + return encoded_inputs[0] + + features = [{ + "input_ids": ids, + "attention_mask": [1] * len(ids) + } for ids in encoded_inputs] + batch = self.pad(features, + padding=padding, + max_length=max_length if padding else None, + return_attention_mask=True, + return_tensors=return_tensors) + + if return_dict: + return batch + if is_batched: + return batch["input_ids"] + return batch["input_ids"][0] if return_tensors is None else batch[ + "input_ids"] + + """ ----- Below are the abstract methods required by PreTrainedTokenizer ----- """ + + @property + def vocab_size(self) -> int: + return self.n_words + + def get_vocab(self) -> Dict[str, int]: + return self.encoder + + def _tokenize(self, text: str, **kwargs) -> List[str]: + return [self.decoder[t] for t in self.encode(text)] + + def _convert_token_to_id(self, token: str) -> int: + return self.encoder.get(token, self.unk_id) + + def _convert_id_to_token(self, index: int) -> str: + return self.decoder.get(index) + + @staticmethod + def clean_up_tokenization(out_string: str) -> str: + return out_string + + def convert_tokens_to_string(self, tokens: List[str]) -> str: + text = ''.join(tokens) + text = bytearray([self.byte_decoder[c] + for c in text]).decode('utf-8', 'replace') + return text + + def save_vocabulary(self, + save_directory: str, + filename_prefix: Optional[str] = None) -> Tuple[str]: + if not os.path.isdir(save_directory): + raise ValueError( + f"vocabulary path ({save_directory}) should be a directory") + out_vocab_file = os.path.join( + save_directory, + (filename_prefix + "-" if filename_prefix else "") + + VOCAB_FILES_NAMES["vocab_file"]) + + if os.path.abspath(self.vocab_file) != os.path.abspath( + out_vocab_file) and os.path.isfile(self.vocab_file): + copyfile(self.vocab_file, out_vocab_file) + + return (out_vocab_file, ) + + def apply_chat_template(self, + conversation, + tools: Optional[list[dict]] = None, + tokenize: bool = False, + add_generation_prompt: bool = True, + thinking: bool = True, + padding=False, + truncation: bool = False, + max_length: Optional[int] = None, + return_tensors=None, + return_dict: bool = False, + **kwargs): + # Tokenizer-level rendering reorders tool result messages to match + # assistant tool_calls, normalizes per-call arguments and response + # schema, then encodes the resulting XTML structure segment-by-segment. + is_batched = is_batched_conversation(conversation) + conversations = conversation if is_batched else [conversation] + image_prompts = kwargs.pop("image_prompts", None) + if is_batched and image_prompts is not None: + raise ValueError("image_prompts is only supported for one chat.") + + # by default set thinking effort to max + kwargs.setdefault("thinking_effort", "max") + + segment_batches = [ + build_chat_segments( + messages, + tools=tools, + add_generation_prompt=add_generation_prompt, + thinking=thinking, + image_prompts=image_prompts, + **kwargs, + ) for messages in conversations + ] + + if not tokenize: + rendered = [ + "".join(segment.text for segment in segments) + for segments in segment_batches + ] + return rendered if is_batched else rendered[0] + + encoded_inputs = [ + self._encode_chat_segments(segments) + for segments in segment_batches + ] + return self._format_chat_token_output( + encoded_inputs, + is_batched=is_batched, + padding=padding, + truncation=truncation, + max_length=max_length, + return_tensors=return_tensors, + return_dict=return_dict, + ) diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8b4d8c452d716709fe22591bb3fd19c3f5d5f269 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,156 @@ +{ + "added_tokens_decoder": { + "163584": { + "content": "[BOS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163585": { + "content": "[EOS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163586": { + "content": "<|end_of_msg|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163587": { + "content": "<|open|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163588": { + "content": "<|close|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163589": { + "content": "<|sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163590": { + "content": "[start_header_id]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163591": { + "content": "[end_header_id]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163593": { + "content": "[EOT]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163602": { + "content": "<|media_begin|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163603": { + "content": "<|media_content|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163604": { + "content": "<|media_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163605": { + "content": "<|media_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163649": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163838": { + "content": "[UNK]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163839": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|end_of_msg|>", + "[start_header_id]", + "[end_header_id]", + "[EOT]", + "<|media_begin|>", + "<|media_content|>", + "<|media_end|>", + "<|media_pad|>", + "" + ], + "bos_token": "[BOS]", + "clean_up_tokenization_spaces": false, + "eos_token": "[EOS]", + "model_max_length": 1000000000000000019884624838656, + "pad_token": "[PAD]", + "unk_token": "[UNK]", + "tokenizer_class": "TikTokenTokenizer", + "auto_map": { + "AutoTokenizer": [ + "tokenization_kimi.TikTokenTokenizer", + null + ] + } +} \ No newline at end of file