diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..52373fe24473b1aa44333d318f578ae6bf04b49b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..48c81da8544e978f08692c3e29b5c4dbc8663de5 --- /dev/null +++ b/LICENSE @@ -0,0 +1,17 @@ +MINIMAX COMMUNITY LICENSE +Copyright (c) 2026 MiniMax +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software for non-commercial purposes, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or provide copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +1. The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +2. If the Software (or any derivative works thereof) is used for any Commercial Use for your products or services: + 1. you shall prominently display “Built with MiniMax M3.1-Flash” on a related website, user interface, blogpost, about page or product documentation. + 2. you shall obtain a separate, prior written authorization from MiniMax by contacting api@minimax.io with the subject line “M3.1-Flash licensing - authorization request”, if such products and services generate more than 20 million US dollars (or equivalent in other currencies) in yearly revenue; otherwise, you only need to send a one-time notice to api@minimax.io with the subject “M3.1-Flash licensing — notice”. +3. “Commercial Use” means any use of the Software or any derivative work thereof that is primarily intended for commercial advantage or monetary compensation, which includes, without limitation: (i) offering products or services to third parties for a fee, which utilize, incorporate, or rely on the Software or its derivatives, (ii) the commercial use of APIs provided by or for the Software or its derivatives, including to support or enable commercial products, services, or operations, whether in a cloud-based, hosted, or other similar environment, and (iii) the deployment or provision of the Software or its derivatives that have been subjected to post-training, fine-tuning, instruction-tuning, or any other form of modification, for any commercial purpose. +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Appendix: Prohibited Uses +You agree you will not use, or allow others to use, the Software or any derivatives of the Software to: +1. Generate or disseminate content prohibited by applicable laws or regulations. +2. Assist with, engage in or otherwise support any military purpose. +3. Exploit, harm, or attempt to exploit or harm minors. +4. Generate or disseminate false or misleading information with the intent to cause harm. +5. Promote discrimination, hate speech, or harmful behavior against individuals or groups based on race or ethnic origin, religion, disability, age, nationality and national origin, veteran status, sexual orientation, gender or gender identity, caste, immigration status, or any other characteristic that is associated with systemic discrimination or marginalization. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..29bfa5f076cfa5b7a89dbadd83dcdb62a2941ec7 --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +--- +license: other +license_name: minimax-community +license_link: LICENSE +library_name: mlx +pipeline_tag: text-generation +base_model: MiniMaxAI/MiniMax-M3.1-Flash +tags: +- moe +- agent +- coding +- mlx +--- + +# mlx-community/MiniMax-M3.1-Flash-Text-4bit + +This model [mlx-community/MiniMax-M3.1-Flash-Text-4bit](https://huggingface.co/mlx-community/MiniMax-M3.1-Flash-Text-4bit) was converted to MLX format from [MiniMaxAI/MiniMax-M3.1-Flash](https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash) using mlx-lm with [ml-explore/mlx-lm#2003](https://github.com/ml-explore/mlx-lm/pull/2003). Refer to the [original model card](https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash) for more details on the model. + +This is a text-only conversion. The vision tower is not included. + +The weights keep the original quantization. The routed experts are NVFP4 with a per-expert global scale, the attention, shared expert and dense MLP layers are MXFP8, and the router stays in float32. Greedy output matched the original checkpoint token for token in testing. Peak memory was about 249 GB in a short generation test. + +Tested with mlx-lm 0.32.2. + +## Use with mlx + +```bash +pip install -U mlx-lm +``` + +```python +from mlx_lm import load, generate + +model, tokenizer = load("mlx-community/MiniMax-M3.1-Flash-Text-4bit") + +prompt = "hello" + +if tokenizer.chat_template is not None: + messages = [{"role": "user", "content": prompt}] + prompt = tokenizer.apply_chat_template( + messages, add_generation_prompt=True, return_dict=False, + ) + +response = generate(model, tokenizer, prompt=prompt, verbose=True) +``` diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..e51457eedc3910cb9a16d4dde8f2c4a70c5def3a --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,248 @@ +{# ---------- special token variables ---------- #} +{%- set ns_token = ']<]minimax[>[' -%} +{%- set bod_token = ']~!b[' -%} +{%- set bos_token = ']~b]' -%} +{%- set eos_token = '[e~[' -%} +{%- set toolcall_begin_token = ns_token ~ '' -%} +{%- set toolcall_end_token = ns_token ~ '' -%} +{%- set think_begin_token = '' -%} +{%- set think_end_token = '' -%} +{%- set image_token = ']<]image[>[' -%} +{%- set video_token = ']<]video[>[' -%} +{#- Recursive XML renderer for tool_call arguments ======================== -#} +{#- None values are intentionally skipped in mapping iteration so that + `null` (which would round-trip to the literal string "null") + never appears in the rendered tool_call. The convention is: omit the + field entirely. The top-level `_args` loop applies the same rule. + The `val is none` branch below is a safety net only — upstream cleaning + (drop_none_in_tool_arguments) should ensure no None ever reaches here. -#} +{%- macro to_xml(val, ns) -%} +{%- if val is mapping -%} +{%- for k, v in val.items() if v is not none -%} +{{ ns }}<{{ k }}>{{ to_xml(v, ns) }}{{ ns }} +{%- endfor -%} +{%- elif val is iterable and val is not string -%} +{%- for item in val -%} +{{ ns }}{{ to_xml(item, ns) }}{{ ns }} +{%- endfor -%} +{%- elif val is none -%} +{#- Should be unreachable when upstream cleaning is applied. -#} +{%- elif val is boolean -%} +{{ val | tojson }} +{%- else -%} +{{ val }} +{%- endif -%} +{%- endmacro -%} +{#- Tool Rendering Functions ============================================== -#} +{%- macro render_tool_namespace(namespace_name, tool_list) -%} +{%- for tool in tool_list -%} +{{ tool.function | tojson(ensure_ascii=False) }} +{% endfor -%} +{%- endmacro -%} +{%- macro visible_text(content) -%} + {%- if content is string -%} + {{ content }} + {%- elif content is iterable and content is not mapping -%} + {%- for item in content -%} + {%- if item is mapping and item.type == 'text' -%} + {{- item.text }} + {%- elif item is mapping and item.type == 'image' -%} + {{- image_token }} + {%- elif item is mapping and item.type == 'video' -%} + {{- video_token }} + {%- elif item is string -%} + {{- item }} + {%- endif -%} + {%- endfor -%} + {%- elif content is none -%} + {{- '' }} + {%- else -%} + {{- content }} + {%- endif -%} +{%- endmacro -%} +{#- System Message Construction ============================================ -#} +{%- macro build_system_message(system_message) -%} + {%- if system_message and system_message.content -%} + {{- visible_text(system_message.content) }} + {#- Handle current_date -#} + {%- if system_message.current_date -%} + {{- '\n' ~ 'Current date: ' + system_message.current_date }} + {%- endif -%} + {#- Handle current_location -#} + {%- if system_message.current_location -%} + {{- '\n' ~ 'Current location: ' + system_message.current_location }} + {%- endif -%} + {%- else -%} + {{- 'Your model version is MiniMax-M3.1, developed by MiniMax. Knowledge cutoff: January 2026. Founded in early 2022, MiniMax is a global AI foundation model company committed to advancing the frontiers of AI towards AGI.' }} + {%- endif -%} + + {#- Thinking mode instructions -#} + {{- '\n\n\n' }} + {{- 'You have a thinking capability that allows you to reason step by step before responding. When thinking is enabled, wrap your reasoning in ' ~ think_begin_token ~ think_end_token ~ ' tags before your response. When thinking is disabled, begin your response directly after the ' ~ think_end_token ~ ' prefix. When thinking is adaptive, decide on your own whether to think for the current turn.\n' }} + {{- 'Current thinking mode: adaptive. You are encouraged to think for complex decision-making, multi-step reasoning, or when analyzing function/tool results.\n' }} + {{- '' }} + {{- '\n\n8bb8149fe' }} + {%- if reasoning_effort is defined and reasoning_effort is not none -%} + {{- '\n\n' ~ reasoning_effort ~ '' }} + {%- elif effort is defined and effort is not none -%} + {{- '\n\n' ~ effort ~ '' }} + {%- endif -%} +{%- endmacro -%} +{%- macro build_developer_message(developer_message) -%} + {%- if developer_message and developer_message.content -%} + {{- visible_text(developer_message.content) }} + {%- else -%} + {%- if model_identity is not defined -%} + {%- set model_identity = "You are a helpful assistant." -%} + {%- endif -%} + {{- model_identity }} + {%- endif -%} +{%- endmacro -%} +{#- Main Template Logic ================================================= -#} +{#- Role mapping: root -> system sp (high priority), system/developer -> developer sp (low priority) -#} +{%- set system_message = none -%} +{%- set developer_message = none -%} +{%- set conversation_messages = messages -%} +{%- if messages and messages[0].role == "root" -%} + {%- set system_message = messages[0] -%} + {%- set conversation_messages = messages[1:] -%} + {%- if conversation_messages and conversation_messages[0].role in ["system", "developer"] -%} + {%- set developer_message = conversation_messages[0] -%} + {%- set conversation_messages = conversation_messages[1:] -%} + {%- endif -%} +{%- elif messages and messages[0].role in ["system", "developer"] -%} + {%- set developer_message = messages[0] -%} + {%- set conversation_messages = messages[1:] -%} +{%- endif -%} +{#- Render system sp (higher priority, root role only) -#} +{{- bod_token ~ bos_token ~ 'system' ~ '\n' }} +{{- build_system_message(system_message) }} +{{- eos_token ~ '\n' }} + +{#- Render developer sp (lower priority: system/developer role + tools) -#} +{{- bos_token ~ 'developer' ~ '\n' }} +{{- build_developer_message(developer_message) }} +{%- if tools -%} + {{- '\n\n' ~ '# Tools' ~ '\n' ~ 'You may call one or more tools to assist with the user query.\nHere are the tools available in JSONSchema format:' ~ '\n' }} + {{- '\n' ~ '' ~ '\n' }} + {{- render_tool_namespace("functions", tools) }} + {{- '' ~ '\n\n' }} + {{- 'To call tools, wrap all invocations in a single ' ~ toolcall_begin_token ~ toolcall_end_token ~ ' block. Parameter values containing nested objects or arrays are recursively expanded into XML elements. Example:\n' }} + {{- '\n' ~ toolcall_begin_token ~ '\n' }} + {{- ns_token + '' }} + {{- ns_token + 'value-1' + ns_token + '' }} + {{- ns_token + '' }} + {{- ns_token + '' }} + {{- ns_token + 'val-a' + ns_token + '' }} + {{- ns_token + 'val-b' + ns_token + '' }} + {{- ns_token + '' }} + {{- ns_token + '' }} + {{- ns_token + '\n' }} + {{- ns_token + '' }} + {{- ns_token + 'value-1' + ns_token + '' }} + {{- ns_token + '\n' }} + {{- toolcall_end_token }} +{%- endif -%} +{{- eos_token ~ '\n' }} + +{#- Render messages -#} +{%- set last_tool_call = namespace(name=none) -%} +{%- for message in conversation_messages -%} + {%- if message.role == 'assistant' -%} + {{- bos_token ~ 'ai' ~ '\n' }} + + {%- set reasoning_content = '' %} + {%- set content = visible_text(message.content) %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if think_end_token in content %} + {%- set reasoning_content = content.split(think_end_token)[0].strip('\n').split(think_begin_token)[-1].strip('\n') %} + {%- set content = content.split(think_end_token)[-1].strip('\n') %} + {%- endif %} + {%- endif %} + + {%- if reasoning_content -%} + {#- Render thinking for every assistant turn (all-turn visible) -#} + {{- think_begin_token ~ reasoning_content ~ think_end_token }} + {%- else -%} + {#- No thinking rendered → prefix with think_end_token -#} + {{- think_end_token }} + {%- endif -%} + + {%- if content -%} + {{- content }} + {%- endif -%} + {%- if message.tool_calls -%} + {{- toolcall_begin_token ~ '\n' }} + + {%- for tool_call in message.tool_calls -%} + {%- if tool_call.function -%} + {%- set tool_call = tool_call.function -%} + {%- endif -%} +{{- ns_token + '' }} +{%- set _args = tool_call.arguments -%} +{%- for k, v in _args.items() if v is not none %} +{{- ns_token + '<' + k + '>' -}} +{{- to_xml(v, ns_token) -}} +{{- ns_token + '' }} +{%- endfor -%} +{{- ns_token + '' ~ '\n' }} + {%- endfor -%} + + {{- toolcall_end_token }} + {%- if message.tool_calls[-1].function -%} + {%- set last_tool_call.name = message.tool_calls[-1].function.name -%} + {%- else -%} + {%- set last_tool_call.name = message.tool_calls[-1].name -%} + {%- endif -%} + {%- else -%} + {%- set last_tool_call.name = none -%} + {%- endif -%} + {{- eos_token ~ '\n' }} + + {%- elif message.role == 'tool' -%} + {%- if last_tool_call.name is none -%} + {{- raise_exception("Message has tool role, but there was no previous assistant message with a tool call!") }} + {%- endif -%} + {%- if loop.first or (conversation_messages[loop.index0 - 1].role != 'tool') -%} + {{- bos_token ~ 'tool' }} + {%- endif -%} + {{- '\n' }} + {%- if message.content is string -%} + {{- message.content }} + {%- else -%} + {%- for tr in message.content -%} + {%- if tr is mapping and tr.type is defined and tr.type == 'image' -%} + {{- image_token }} + {%- elif tr is mapping and tr.type is defined and tr.type == 'video' -%} + {{- video_token }} + {%- else -%} + {{- tr.output if tr.output is defined else (tr.text if tr.type == 'text' and tr.text is defined else tr) }} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {{- '' }} + {%- if loop.last or (conversation_messages[loop.index0 + 1].role != 'tool') -%} + {{- eos_token ~ '\n' -}} + {%- endif -%} + + {%- elif message.role in ['system', 'developer'] -%} + {#- Mid-conversation system/developer messages keep their position and + map to the lower-priority M3 developer role. -#} + {{- bos_token ~ 'developer' ~ '\n' }} + {{- build_developer_message(message) }} + {{- eos_token ~ '\n' }} + + {%- elif message.role == 'user' -%} + {{- bos_token ~ 'user' ~ '\n' }} + {{- visible_text(message.content) }} + {{- eos_token ~ '\n' }} + {%- endif -%} +{%- endfor -%} + +{#- Generation prompt -#} +{%- if add_generation_prompt -%} +{{- bos_token ~ 'ai' ~ '\n' }} +{#- adaptive: no prefix, let model decide -#} +{%- endif -%} diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a0d3f019b386d0235a6d2f85ebf5cad93a3e02cb --- /dev/null +++ b/config.json @@ -0,0 +1,267 @@ +{ + "architectures": [ + "MiniMaxM3SparseForConditionalGeneration" + ], + "auto_map": { + "AutoConfig": "configuration_minimax_m3_vl.MiniMaxM3VLConfig" + }, + "eos_token_id": 200020, + "image_grid_pinpoints": "[(336, 336), (336, 672), (336, 1008), (336, 1344), (336, 1680), (336, 2016), (672, 336), (672, 672), (672, 1008), (672, 1344), (672, 1680), (672, 2016), (1008, 336), (1008, 672), (1008, 1008), (1008, 1344), (1008, 1680), (1008, 2016), (1344, 336), (1344, 672), (1344, 1008), (1344, 1344), (1344, 1680), (1344, 2016), (1680, 336), (1680, 672), (1680, 1008), (1680, 1344), (1680, 1680), (1680, 2016), (2016, 336), (2016, 672), (2016, 1008), (2016, 1344), (2016, 1680), (2016, 2016)]", + "image_seq_length": 576, + "image_token_index": 200025, + "img_token_compression_config": { + "image_token_compression_method": "patch_merge", + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "model_type": "minimax_m3_vl", + "multimodal_projector_bias": true, + "num_reward_heads": 0, + "process_image_mode": "dynamic_res", + "projector_hidden_act": "gelu", + "projector_hidden_size": 6144, + "quantization": { + "group_size": 32, + "bits": 8, + "mode": "mxfp8" + }, + "quantization_config": { + "group_size": 32, + "bits": 8, + "mode": "mxfp8" + }, + "text_config": { + "hidden_size": 6144, + "intermediate_size": 3072, + "num_hidden_layers": 60, + "num_attention_heads": 64, + "num_key_value_heads": 4, + "head_dim": 128, + "vocab_size": 200064, + "max_position_embeddings": 1048576, + "rms_norm_eps": 1e-06, + "use_gemma_norm": true, + "attention_output_gate": false, + "rope_theta": 10000000, + "rotary_dim": 64, + "partial_rotary_factor": 0.5, + "hidden_act": "swigluoai", + "use_qk_norm": true, + "tie_word_embeddings": false, + "dense_intermediate_size": 12288, + "shared_intermediate_size": 3072, + "num_local_experts": 128, + "num_experts_per_tok": 4, + "n_shared_experts": 1, + "scoring_func": "sigmoid", + "use_routing_bias": true, + "moe_layer_freq": [ + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "qk_norm_type": "per_head", + "swiglu_alpha": 1.702, + "swiglu_limit": 7.0, + "routed_scaling_factor": 2.0, + "sparse_attention_config": { + "use_sparse_attention": true, + "sparse_index_dim": 128, + "sparse_num_index_heads": 4, + "sparse_topk_blocks": 16, + "sparse_block_size": 128, + "sparse_score_type": "max", + "sparse_init_block": 0, + "sparse_local_block": 1, + "sparse_attention_freq": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sparse_disable_index_value": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ] + }, + "architectures": [ + "MiniMaxM3SparseForCausalLM" + ] + }, + "torch_dtype": "bfloat16", + "transformers_version": "4.52.4", + "video_token_index": 200026, + "vision_feature_layer": -1, + "vision_feature_select_strategy": "full" +} \ No newline at end of file diff --git a/configuration_minimax_m3_vl.py b/configuration_minimax_m3_vl.py new file mode 100644 index 0000000000000000000000000000000000000000..0534f70354808a0f93f70546a919770fcca57b56 --- /dev/null +++ b/configuration_minimax_m3_vl.py @@ -0,0 +1,111 @@ +"""HuggingFace configs for the MiniMax VL family (M2 VL / M3 VL). + +This file is bundled into every converted HF checkpoint so that loading via +``AutoConfig.from_pretrained(..., trust_remote_code=True)`` works without any +runtime dependency on sglang or other internal packages — only stock +``transformers`` is required. + +The class definitions intentionally mirror +``sglang.srt.configs.minimax_vl``; if either side changes, keep them in sync. + +The file is named ``configuration_minimax_m3_vl.py`` (matching the legacy +``model_type="minimax_m3_vl"`` and the converter's ``auto_map`` entry) so +that ckpts produced by this converter remain loadable by older sglang versions +that only know the ``MiniMaxM3VL*`` names. The canonical class is +``MiniMaxM3VLConfig``; ``MiniMaxM3VLConfig`` is a thin BC alias whose only +purpose is to be referenced from ``auto_map``. +""" + +from typing import Optional + +from transformers.configuration_utils import PretrainedConfig +from transformers.models.auto import CONFIG_MAPPING + + +def _coerce_sub_config( + sub_config: Optional[dict], default_model_type: str +) -> Optional[PretrainedConfig]: + """Convert a config dict to a ``PretrainedConfig`` instance. + + If ``model_type`` is registered in HF ``CONFIG_MAPPING`` the corresponding + config class is used; otherwise we fall back to a generic + ``PretrainedConfig`` so all dict keys still become real attributes (M3's + text backbone uses ``model_type="minimax_m2"`` which is not in + ``CONFIG_MAPPING``). + """ + if not isinstance(sub_config, dict): + return sub_config + model_type = sub_config.get("model_type", default_model_type) + cls = CONFIG_MAPPING.get(model_type, PretrainedConfig) + return cls(**sub_config) + + +class MiniMaxVLBaseConfig(PretrainedConfig): + """Base config shared by every MiniMax VL variant. + + Handles vision/text sub-config coercion. Concrete subclasses only need to + declare a unique ``model_type`` string. + """ + + def __init__( + self, + vision_config: Optional[dict] = None, + text_config: Optional[dict] = None, + image_token_index: int = 200025, + video_token_index: int = 200026, + image_seq_length: int = 576, + process_image_mode: str = "dynamic_res", + projector_hidden_act: str = "gelu", + multimodal_projector_bias: bool = True, + vision_feature_layer: int = -1, + vision_feature_select_strategy: str = "full", + img_token_compression_config: Optional[dict] = None, + image_grid_pinpoints: Optional[str] = None, + **kwargs, + ): + self.vision_config = _coerce_sub_config(vision_config, "clip_vision_model") + self.text_config = _coerce_sub_config(text_config, "mixtral") + + self.image_token_index = image_token_index + self.video_token_index = video_token_index + self.image_seq_length = image_seq_length + self.process_image_mode = process_image_mode + self.projector_hidden_act = projector_hidden_act + self.multimodal_projector_bias = multimodal_projector_bias + self.vision_feature_layer = vision_feature_layer + self.vision_feature_select_strategy = vision_feature_select_strategy + self.img_token_compression_config = img_token_compression_config or {} + self.image_grid_pinpoints = image_grid_pinpoints + + super().__init__(**kwargs) + + def __post_init__(self, **kwargs): + super().__post_init__(**kwargs) + if hasattr(self, "vision_config"): + self.vision_config = _coerce_sub_config(self.vision_config, "clip_vision_model") + if hasattr(self, "text_config"): + self.text_config = _coerce_sub_config(self.text_config, "mixtral") + + +class MiniMaxM2VLConfig(MiniMaxVLBaseConfig): + """MiniMax M2 VL: vision tower + M2 (Mixtral-style MoE) text backbone.""" + + model_type = "minimax_m2_vl" + + +class MiniMaxM3VLConfig(MiniMaxVLBaseConfig): + """MiniMax M3 VL: vision tower + M3 (mixed sparse/dense MoE) text backbone.""" + + model_type = "minimax_m3_vl" + + +class MiniMaxM2MiniVLConfig(MiniMaxM2VLConfig): + """Legacy alias kept so old ``model_type="minimax_m2_mini_vl"`` ckpts load.""" + + model_type = "minimax_m2_mini_vl" + + +class MiniMaxM3VLConfig(MiniMaxM3VLConfig): + """Legacy alias kept so old ``model_type="minimax_m3_vl"`` ckpts load.""" + + model_type = "minimax_m3_vl" diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..54806f9d4ea503f2d1afa468de37764acbbaa97d --- /dev/null +++ b/generation_config.json @@ -0,0 +1,8 @@ +{ + "bos_token_id": 200019, + "do_sample": true, + "eos_token_id": 200020, + "temperature": 1.0, + "top_p": 0.95, + "transformers_version": "4.46.1" +} \ No newline at end of file diff --git a/image_processor.py b/image_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..aa896bba971dfc6e79c8b4d51730e0c6ea7953dc --- /dev/null +++ b/image_processor.py @@ -0,0 +1,223 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +""" +MiniMax VL family HuggingFace-compatible Processor, ImageProcessor, VideoProcessor. +""" +import math +from typing import List, Tuple + +import torch +from torchvision.transforms import InterpolationMode +from transformers import BatchFeature +from transformers.image_processing_utils_fast import ( + BaseImageProcessorFast, + group_images_by_shape, + reorder_images, +) +from transformers.image_utils import PILImageResampling, SizeDict +from transformers.processing_utils import ( + ImagesKwargs, + Unpack, +) +from transformers.utils import TensorType + +MAX_RATIO = 200 + + +def round_by_factor(number: int, factor: int) -> int: + return round(number / factor) * factor + + +def ceil_by_factor(number: int, factor: int) -> int: + return math.ceil(number / factor) * factor + + +def floor_by_factor(number: int, factor: int) -> int: + return math.floor(number / factor) * factor + + +def smart_resize( + height: int, + width: int, + factor: int = 28, + min_pixels: int = 4 * 28 * 28, + max_pixels: int = 451584, +) -> tuple[int, int]: + if max(height, width) / min(height, width) > MAX_RATIO: + raise ValueError( + f"absolute aspect ratio must be smaller than {MAX_RATIO}, " + f"got {max(height, width) / min(height, width)}" + ) + h_bar = max(factor, round_by_factor(height, factor)) + w_bar = max(factor, round_by_factor(width, factor)) + if h_bar * w_bar > max_pixels: + beta = math.sqrt((height * width) / max_pixels) + h_bar = floor_by_factor(height / beta, factor) + w_bar = floor_by_factor(width / beta, factor) + elif h_bar * w_bar < min_pixels: + beta = math.sqrt(min_pixels / (height * width)) + h_bar = ceil_by_factor(height * beta, factor) + w_bar = ceil_by_factor(width * beta, factor) + return h_bar, w_bar + + +# ============================================================================== +# MiniMax M3 VL Image Processor Fast (Fast Mode - Torch based) +# ============================================================================== + + +class MiniMaxM3VLImageProcessorKwargs(ImagesKwargs, total=False): + patch_size: int + temporal_patch_size: int + merge_size: int + max_pixels: int + + +class MiniMaxM3VLImageProcessor(BaseImageProcessorFast): + do_resize = True + resample = PILImageResampling.BICUBIC + size = {"height": 672, "width": 672} # required by base class validation, not used as resize bound + default_to_square = False + do_rescale = True + rescale_factor = 1 / 255 + do_normalize = True + image_mean = [0.48145466, 0.4578275, 0.40821073] + image_std = [0.26862954, 0.26130258, 0.27577711] + do_convert_rgb = True + patch_size = 14 + temporal_patch_size = 2 + merge_size = 2 + max_pixels = 451584 # 672*672 + valid_kwargs = MiniMaxM3VLImageProcessorKwargs + model_input_names = ["pixel_values", "image_grid_thw"] + + def __init__(self, **kwargs: Unpack[MiniMaxM3VLImageProcessorKwargs]): + super().__init__(**kwargs) + + def preprocess( + self, images, **kwargs: Unpack[MiniMaxM3VLImageProcessorKwargs] + ) -> BatchFeature: + return super().preprocess(images, **kwargs) + + def _preprocess( + self, + images: List[torch.Tensor], + do_resize: bool, + size: SizeDict, + resample: PILImageResampling | InterpolationMode | int | None, + do_rescale: bool, + rescale_factor: float, + do_normalize: bool, + image_mean: float | List[float] | None, + image_std: float | List[float] | None, + patch_size: int, + temporal_patch_size: int, + merge_size: int, + max_pixels: int, + disable_grouping: bool | None, + return_tensors: str | TensorType | None, + **kwargs, + ) -> BatchFeature: + grouped_images, grouped_images_index = group_images_by_shape( + images, disable_grouping=disable_grouping + ) + resized_images_grouped = {} + factor = patch_size * merge_size + for shape, stacked_images in grouped_images.items(): + height, width = stacked_images.shape[-2:] + if do_resize: + resized_height, resized_width = smart_resize( + height, width, factor=factor, + max_pixels=max_pixels, + ) + stacked_images = self.resize( + stacked_images, + size=SizeDict(height=resized_height, width=resized_width), + resample=resample, + ) + resized_images_grouped[shape] = stacked_images + + resized_images = reorder_images(resized_images_grouped, grouped_images_index) + + grouped_images, grouped_images_index = group_images_by_shape( + resized_images, disable_grouping=disable_grouping + ) + processed_images_grouped = {} + processed_grids = {} + + for shape, stacked_images in grouped_images.items(): + resized_height, resized_width = stacked_images.shape[-2:] + + patches = self.rescale_and_normalize( + stacked_images, + do_rescale, + rescale_factor, + do_normalize, + image_mean, + image_std, + ) + if patches.ndim == 4: + patches = patches.unsqueeze(1) + + if patches.shape[1] % temporal_patch_size != 0: + repeats = patches[:, -1:].repeat( + 1, + temporal_patch_size - (patches.shape[1] % temporal_patch_size), + 1, + 1, + 1, + ) + patches = torch.cat([patches, repeats], dim=1) + + batch_size, grid_t, channel = patches.shape[:3] + grid_t = grid_t // temporal_patch_size + grid_h, grid_w = resized_height // patch_size, resized_width // patch_size + + patches = patches.view( + batch_size, + grid_t, + temporal_patch_size, + channel, + grid_h // merge_size, + merge_size, + patch_size, + grid_w // merge_size, + merge_size, + patch_size, + ) + patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9) + + flatten_patches = patches.reshape( + batch_size, + grid_t * grid_h * grid_w, + channel * temporal_patch_size * patch_size * patch_size, + ) + + processed_images_grouped[shape] = flatten_patches + processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size + + processed_images = reorder_images( + processed_images_grouped, grouped_images_index + ) + processed_grids = reorder_images(processed_grids, grouped_images_index) + + pixel_values = torch.cat(processed_images, dim=0) + image_grid_thw = torch.tensor(processed_grids, dtype=torch.long) + + return BatchFeature( + data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}, + tensor_type=return_tensors, + ) + + def get_number_of_image_patches(self, height: int, width: int, images_kwargs=None): + images_kwargs = images_kwargs or {} + patch_size = images_kwargs.get("patch_size", self.patch_size) + merge_size = images_kwargs.get("merge_size", self.merge_size) + max_pixels = images_kwargs.get("max_pixels", self.max_pixels) + + resized_height, resized_width = smart_resize( + height, width, factor=patch_size * merge_size, + max_pixels=max_pixels, + ) + grid_h, grid_w = resized_height // patch_size, resized_width // patch_size + return grid_h * grid_w diff --git a/model-00001-of-00058.safetensors b/model-00001-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bfb75fa561e7b7d3688796c07f1ab874a3c145b0 --- /dev/null +++ b/model-00001-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2e5a8b92e32dfba731a5491825e282ef70347b3b858ae2e3adf566e14800c92 +size 4978697130 diff --git a/model-00002-of-00058.safetensors b/model-00002-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..eebf6188b6040399ecbba867a8b0bab1f251bdb1 --- /dev/null +++ b/model-00002-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:834fc18871bf0295874cfe22dec3482c4d8f972e4c572551e72dd4bc46356481 +size 4252786531 diff --git a/model-00003-of-00058.safetensors b/model-00003-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..812951000899204a5c69673a9358558b62b5a980 --- /dev/null +++ b/model-00003-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d71eabf1e6eff602e63b8d6f684b777a5351c2a7b3b525ef96bb14d06115ea1d +size 4252786517 diff --git a/model-00004-of-00058.safetensors b/model-00004-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5f56e0a302eeb09905ae9a05d5aee35056821215 --- /dev/null +++ b/model-00004-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:662c29bd4cad8789997d5725c5ff8ccde7fb92e9a55d790eb9818303a48c8a00 +size 4252786475 diff --git a/model-00005-of-00058.safetensors b/model-00005-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..563a2755fe4365c233344ca443eb61d7ce51dc9b --- /dev/null +++ b/model-00005-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d26e4910b7b4a92d18ca5c4d0c72a57c98d6f69d0c3abf94acc383f776bf36a +size 4252786517 diff --git a/model-00006-of-00058.safetensors b/model-00006-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a16b9d33bc1dbd6d7fdd1b8f8413ed5546780046 --- /dev/null +++ b/model-00006-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dbd733e4203ee203a6e67fb9cf4a398b8af618cdab09355b03c02fc82f77cdb +size 4252786531 diff --git a/model-00007-of-00058.safetensors b/model-00007-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4d6fb2c98075bdd3ad059b6ae4138b7234ea332e --- /dev/null +++ b/model-00007-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ca0f3d9823a8108b5ed03068418dc5df11a06d395a41f44306a96500e1f989d +size 4252786473 diff --git a/model-00008-of-00058.safetensors b/model-00008-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0fcd6e28f97baeea2ccb71ddac3e405605d4408e --- /dev/null +++ b/model-00008-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:474034764a235dabd554415abfb83e73dfb13c99495b0b33206fe52009c01e90 +size 4252786550 diff --git a/model-00009-of-00058.safetensors b/model-00009-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..433af12e085cad7ba23789ac0cd0d55dc42372fc --- /dev/null +++ b/model-00009-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:221bbb01726f09431c05ff3f9c45d229a47413bda70ee8a4d0fd1b722080ebb3 +size 4252786536 diff --git a/model-00010-of-00058.safetensors b/model-00010-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..691f845220af2a561a4eb124ef3b6844f7eb8c0b --- /dev/null +++ b/model-00010-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2ebfbcb788304ea01a43d3835a979c158792e6e9bea1a43269d30adb852237c +size 4252786560 diff --git a/model-00011-of-00058.safetensors b/model-00011-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e2265223d519a8bfcddbf063e92abfa305e42207 --- /dev/null +++ b/model-00011-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77f236b78b8a298a2643459c75df2ce84f99281c057ee74dc7c9228cf97daaf8 +size 4252786524 diff --git a/model-00012-of-00058.safetensors b/model-00012-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e9ab213d3f2fdb9c811d085f01261739ff91789c --- /dev/null +++ b/model-00012-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32b75f3edf4d045ca693ecf542add31b59fa688a6c66951f81ba557254aab8d9 +size 4252786498 diff --git a/model-00013-of-00058.safetensors b/model-00013-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cf8447d33d02ebcd1d58cd9473fdb40a24576c3d --- /dev/null +++ b/model-00013-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4acd342a1fa82e60b0ee7f5cff24eefd106844e6502db9fb270bdf4c7e4e667 +size 4252786552 diff --git a/model-00014-of-00058.safetensors b/model-00014-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a8307d8c8ca896785d15b742ef5c036b386da730 --- /dev/null +++ b/model-00014-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3865f5ebc1d307e293f2c46a415b254072165f44f778df54824a110260c6ac86 +size 4252786564 diff --git a/model-00015-of-00058.safetensors b/model-00015-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3f5dbb82f475e2f5482b3ce28c19969e8c85451e --- /dev/null +++ b/model-00015-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc68b394208687be8c20aa65f247c469591ae1cd0833913decdbf0bd19876f62 +size 4252786460 diff --git a/model-00016-of-00058.safetensors b/model-00016-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..83286c8d20de75f90c839e280ce060b0be330a4e --- /dev/null +++ b/model-00016-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4868115742d39eedd804696e911ed5963758578bf7fe56bceaf43a708359f722 +size 4252786552 diff --git a/model-00017-of-00058.safetensors b/model-00017-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..384ab06ee5deb8f683e449433e57e80c18dc935d --- /dev/null +++ b/model-00017-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13e0dc823c165ca6610bf1a12790fa6388a6cde771325b427bde448a2d7eeaf7 +size 4252786526 diff --git a/model-00018-of-00058.safetensors b/model-00018-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..de14355d38490c609323cd54e56794442f8d3a3f --- /dev/null +++ b/model-00018-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bc98923b0dea1ced3122630e889c6114cfc29ee0471a60e9728bdab54c20946 +size 4252786518 diff --git a/model-00019-of-00058.safetensors b/model-00019-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a49f524cb2aa2f08853911980d77ef5605ac7e7a --- /dev/null +++ b/model-00019-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd6d850da52ed0543cc3b04ecfee63b0899f2f8220f638401c79db85de3736f7 +size 4252786566 diff --git a/model-00020-of-00058.safetensors b/model-00020-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a73d724baa08256bb92bbed81cb522db059aca1c --- /dev/null +++ b/model-00020-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f22da89eaf6ad9ebd974ac03e0f53bbd7cb0318cdcbcd7860a05a5a1f888ea9e +size 4252786552 diff --git a/model-00021-of-00058.safetensors b/model-00021-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fe2df97d632f3ade17963be0624d20f1a027ce14 --- /dev/null +++ b/model-00021-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8a422afc2488dd9d6ae3157c1097aff35d47681b79428edafbc5e748ec79f27 +size 4252786550 diff --git a/model-00022-of-00058.safetensors b/model-00022-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f41d62ff301b846716c396808e46a35539c7e09f --- /dev/null +++ b/model-00022-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acfd500c7cc374b63e6969d57fedb925bdf0ffa0d478178378c0864d232cfe9b +size 4252786552 diff --git a/model-00023-of-00058.safetensors b/model-00023-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..00808299895439409701a8497e829222884b4bc4 --- /dev/null +++ b/model-00023-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c643337342e43bfd5dba84211705ce2d3eec88ceafa9dc2ebce6b1d94122cd5b +size 4252786550 diff --git a/model-00024-of-00058.safetensors b/model-00024-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a52aa5c104d99fb0bac594484e2a997738a5a24a --- /dev/null +++ b/model-00024-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:827b6dc50e18fa4376721f7ceb54e8aab1b7ef9cfd0977277a3661c039a659f0 +size 4252786552 diff --git a/model-00025-of-00058.safetensors b/model-00025-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..db751402fde71c189fcba2cce9e4ad92f21d996c --- /dev/null +++ b/model-00025-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d00b61c3ce7513cb21e0d7bb81fe48853f9fb12a5c726fc4b04ba638ec868a4e +size 4252786500 diff --git a/model-00026-of-00058.safetensors b/model-00026-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9959626aa7a7282b2c3bea90b0a9880b65587d56 --- /dev/null +++ b/model-00026-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64e66ce120f722791709fac478bc170aad613f7b4ba4f55b6dadf4df3f9e8ef4 +size 4252786566 diff --git a/model-00027-of-00058.safetensors b/model-00027-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bad101a54a72dd2dfb750f38ba4bf278b2ecf2c2 --- /dev/null +++ b/model-00027-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:247ef271bc8628aabba0e2daf482978c1308a5019910ed0af07f65deda2147ab +size 4252786564 diff --git a/model-00028-of-00058.safetensors b/model-00028-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..279efcb7e7a52062ffb0571a4a62202408e7951e --- /dev/null +++ b/model-00028-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf75377f2aaebfe7c939799793d879aca8f92af361ca9d105d8402dc41219ffe +size 4252786490 diff --git a/model-00029-of-00058.safetensors b/model-00029-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..80ab0f209c9cbad7aff79db864485b3ad1938856 --- /dev/null +++ b/model-00029-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:436fa42d0b66128b0bcbba75db45cca74414dcb30746e1a935e427ba26f220b1 +size 4252786504 diff --git a/model-00030-of-00058.safetensors b/model-00030-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0138544ae0030ce5bb6ee1c53ebd862cffeb2568 --- /dev/null +++ b/model-00030-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97aa2a37d1b441792bcca9585e5b561c198fa13817e84e4c74473f9db66ada62 +size 4252786550 diff --git a/model-00031-of-00058.safetensors b/model-00031-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..61b8566c21ac5ee2ac9242cf98c7bc4548a519d2 --- /dev/null +++ b/model-00031-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7c5f99b3094727195107e8495a420d7b54bded13529199700ca47bc6a722788 +size 4252786550 diff --git a/model-00032-of-00058.safetensors b/model-00032-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f4562b0adaba51afe37b77d9ba78f2e8b7cd74cd --- /dev/null +++ b/model-00032-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad603f8b8fdc6305e0dc2d847e7715ddb975875342239bc6c29fe17bd17307d5 +size 4252786528 diff --git a/model-00033-of-00058.safetensors b/model-00033-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4178c33439231c7235f1cce231623c747050b9b2 --- /dev/null +++ b/model-00033-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2979c413868e5875d1e79dcae68a1879f6c6ff2951a8033ee5f91f970b9ec016 +size 4252786552 diff --git a/model-00034-of-00058.safetensors b/model-00034-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a615ecad4099bdc5c2513a3ca37e50952b9871b7 --- /dev/null +++ b/model-00034-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59b270d443a32c61e8f8ad52bc4fd785c7ae5e862d4b862dfb23c6489e346755 +size 4252786550 diff --git a/model-00035-of-00058.safetensors b/model-00035-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..80a3dcc6f84d5953cba033c178b579801cecefda --- /dev/null +++ b/model-00035-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bbf256f80382163572408429eb6eba4d63a92e41263c033c417512d2a59bea9 +size 4252786566 diff --git a/model-00036-of-00058.safetensors b/model-00036-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1306119ccc995ad7a533cdf3763dc6d83d19a30f --- /dev/null +++ b/model-00036-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57182530c04628436856c8dc25fca4b368d91d555d91bcebcc7a50a90eefb63b +size 4252786500 diff --git a/model-00037-of-00058.safetensors b/model-00037-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e0a50f57bb55061c32b5edf885dfcd69e8e3aed3 --- /dev/null +++ b/model-00037-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:670ebacf9ae23237ef58eb169f2cfb561684f1afe4a0b4fd666b5a7b3d4ed4cd +size 4252786512 diff --git a/model-00038-of-00058.safetensors b/model-00038-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..eac58d42686746f43a7fd1e5855ff99728ce8c1e --- /dev/null +++ b/model-00038-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:737dddaf949437951bfd17b67cf6788be1aa5298b886bc9be0aedf7636cf94e1 +size 4252786566 diff --git a/model-00039-of-00058.safetensors b/model-00039-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c90177dc5c0e9b1bdd0fadf5cd9a9f91a7d9beb3 --- /dev/null +++ b/model-00039-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdd17abe3664e1eab182c212270cd9e715f0972a358efd7a389021e699952c78 +size 4252786562 diff --git a/model-00040-of-00058.safetensors b/model-00040-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..30ea754f01dfae602a9e670a2c4c7d7cff6aa89b --- /dev/null +++ b/model-00040-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67dd2d850b7a708d43bfa72477dbeeff4b67776897a684d90ab6d8c90c46ff0c +size 4252786508 diff --git a/model-00041-of-00058.safetensors b/model-00041-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..188f1dd62fae5cdafba827d24878841e81bf377a --- /dev/null +++ b/model-00041-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:523dce9c7db6da0e593d8ea5954ea219bdf8dfc17a650d21b2aa6477c4b3e324 +size 4252786552 diff --git a/model-00042-of-00058.safetensors b/model-00042-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c3f541ad4a04129ffaa58fbea701779719969e45 --- /dev/null +++ b/model-00042-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4330dd2b041cd538939334ac76b509a1dbae6b62737611fae97a91fcc135c5c2 +size 4252786550 diff --git a/model-00043-of-00058.safetensors b/model-00043-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f792f50968698b8db07cee757b317e60eac70ad7 --- /dev/null +++ b/model-00043-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:144cd0e21b89db60d6acea13fc3ae0efb300a9fd11554cdfac8ded12f0a789eb +size 4252786566 diff --git a/model-00044-of-00058.safetensors b/model-00044-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ca09dcb60fc231d283d617e967cf9d674b0f87c9 --- /dev/null +++ b/model-00044-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25ef4be49238ebc49a1a4302efd71abfce4e1cff36a03a6244995bebf579e83c +size 4252786538 diff --git a/model-00045-of-00058.safetensors b/model-00045-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..403d4a4711877fcbc6cff3288dc581352bccc591 --- /dev/null +++ b/model-00045-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f33ce5a553a6be96136e916e4c8d9cedb09d60e7fb23cb66ff050080b7fcd972 +size 4252786550 diff --git a/model-00046-of-00058.safetensors b/model-00046-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..77fffe01736c43940d132a19158ba5a7d78d616a --- /dev/null +++ b/model-00046-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ce97d84f7158c7e10514669a5068c7eec7eb6c71a4e54dfebe94b06089bb35f +size 4252786520 diff --git a/model-00047-of-00058.safetensors b/model-00047-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dfcc1d532e58994f45a14566908e1e82cbd3d18b --- /dev/null +++ b/model-00047-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd3423f42ffa6632604f35e4b3ed4df16ad4c14cf3dadc389715859fb87ff692 +size 4252786566 diff --git a/model-00048-of-00058.safetensors b/model-00048-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e39c444e10e44410fc3f1787f0f595d63bb3fd61 --- /dev/null +++ b/model-00048-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15e7cedd700dd1e9d2a7611f2ecf42673b818d196f90f17168229af18f47ca5e +size 4252786550 diff --git a/model-00049-of-00058.safetensors b/model-00049-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5b39027c8b57b2c1ac167850ba1484970166b011 --- /dev/null +++ b/model-00049-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4bbf974de0c48149da18340d32dd946c32532c1be82615137a6d02dac1ad3e1 +size 4252786550 diff --git a/model-00050-of-00058.safetensors b/model-00050-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c3bff53eaa35115a11d27979690d1faf3e83cbfd --- /dev/null +++ b/model-00050-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50acb18d1c26dfcc7f408948c6f17a7fa129124ea3c1081b86431338fafef9c4 +size 4252786552 diff --git a/model-00051-of-00058.safetensors b/model-00051-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..711132acbaa2ecd6cc9327a49b8e8c78aaa09a8a --- /dev/null +++ b/model-00051-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec5fb3c0d4f31cf7c20617b9a59cb869d31e778d0c4f9643d0ecbfef527b60ba +size 4252786566 diff --git a/model-00052-of-00058.safetensors b/model-00052-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5634ca6eda84faa41cd3eef11cd95714b58d487c --- /dev/null +++ b/model-00052-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a501f057cbb9845f8c7702ff508ae0462680b947daaa7d54ba07c1935c6a9e0d +size 4252786496 diff --git a/model-00053-of-00058.safetensors b/model-00053-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d96d6ed37d3d8031a456b2e2ca3245625a308bf8 --- /dev/null +++ b/model-00053-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61f2c5bbfc9d6a038bbddc22c07779f672ba32e46bcf1d7c07831f8d34313c6c +size 4252786564 diff --git a/model-00054-of-00058.safetensors b/model-00054-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9afa5d6045c2e22530067bf3795711409de7ac9d --- /dev/null +++ b/model-00054-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfa28fdc7316589a2d17309dfbbca5075a4db291694c4896f57f6259f957ee61 +size 4252786460 diff --git a/model-00055-of-00058.safetensors b/model-00055-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ba610b1f7259fc7dc9b8667daa3688fd558b5a55 --- /dev/null +++ b/model-00055-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b50a84f40b728cf063cce5ed1df073ce2863c3815c653a588f702a62724b51b +size 4252786552 diff --git a/model-00056-of-00058.safetensors b/model-00056-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e5123eca7c1c2af1ff005954db02404388064e6f --- /dev/null +++ b/model-00056-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b27dbf77d36bad654a658c8a455d585cb053d0ed4b4856283e584ce6e14b6359 +size 4252786564 diff --git a/model-00057-of-00058.safetensors b/model-00057-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bb57c06047cbae074661a189f2f6b2000a42e611 --- /dev/null +++ b/model-00057-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da9388d10768f69ed950838561a6f8541556563bad8d487d92ad9c55bac2257b +size 4252786520 diff --git a/model-00058-of-00058.safetensors b/model-00058-of-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3ba76bc8395ba3f9eedafd5f6eda7d1e29965352 --- /dev/null +++ b/model-00058-of-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48154a55a60416697d0d6be32eb0b4c51e8924c53bd45b1cedd32e6eafff4aae +size 5234728209 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..48aaa59dc166f742383cbd20a54b0f0dc20bb77b --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,2078 @@ +{ + "metadata": { + "total_size": 248369178624, + "total_parameters": 426186362880 + }, + "weight_map": { + "language_model.lm_head.weight": "model-00058-of-00058.safetensors", + "language_model.model.embed_tokens.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.input_layernorm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.mlp.down_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.mlp.gate_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.mlp.up_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.mlp.up_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.self_attn.index_k_norm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.self_attn.index_k_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.self_attn.index_k_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.self_attn.index_q_norm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.self_attn.index_q_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.self_attn.index_q_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.self_attn.k_norm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.self_attn.k_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.self_attn.o_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.self_attn.q_norm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.self_attn.q_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.self_attn.v_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.input_layernorm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.mlp.down_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.mlp.down_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.mlp.gate_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.mlp.up_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.mlp.up_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.self_attn.index_k_norm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.self_attn.index_k_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.self_attn.index_k_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.self_attn.index_q_norm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.self_attn.index_q_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.self_attn.index_q_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.self_attn.k_norm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.self_attn.k_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.self_attn.o_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.self_attn.q_norm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.self_attn.q_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.self_attn.v_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.10.block_sparse_moe.e_score_correction_bias": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.block_sparse_moe.gate.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.block_sparse_moe.shared_experts.down_proj.scales": "model-00009-of-00058.safetensors", + "language_model.model.layers.10.block_sparse_moe.shared_experts.down_proj.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.10.block_sparse_moe.shared_experts.gate_proj.scales": "model-00009-of-00058.safetensors", + "language_model.model.layers.10.block_sparse_moe.shared_experts.gate_proj.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.10.block_sparse_moe.shared_experts.up_proj.scales": "model-00009-of-00058.safetensors", + "language_model.model.layers.10.block_sparse_moe.shared_experts.up_proj.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.10.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00009-of-00058.safetensors", + "language_model.model.layers.10.block_sparse_moe.switch_mlp.down_proj.scales": "model-00009-of-00058.safetensors", + "language_model.model.layers.10.block_sparse_moe.switch_mlp.down_proj.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.10.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00009-of-00058.safetensors", + "language_model.model.layers.10.block_sparse_moe.switch_mlp.up_proj.scales": "model-00009-of-00058.safetensors", + "language_model.model.layers.10.block_sparse_moe.switch_mlp.up_proj.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.10.input_layernorm.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.10.post_attention_layernorm.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.10.self_attn.index_k_norm.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.self_attn.index_k_proj.scales": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.self_attn.index_k_proj.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.self_attn.index_q_norm.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.self_attn.index_q_proj.scales": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.self_attn.index_q_proj.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.self_attn.k_norm.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.self_attn.k_proj.scales": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.self_attn.k_proj.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.self_attn.o_proj.scales": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.self_attn.o_proj.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.self_attn.q_norm.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.self_attn.q_proj.scales": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.self_attn.q_proj.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.self_attn.v_proj.scales": "model-00008-of-00058.safetensors", + "language_model.model.layers.10.self_attn.v_proj.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.11.block_sparse_moe.e_score_correction_bias": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.block_sparse_moe.gate.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.block_sparse_moe.shared_experts.down_proj.scales": "model-00010-of-00058.safetensors", + "language_model.model.layers.11.block_sparse_moe.shared_experts.down_proj.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.11.block_sparse_moe.shared_experts.gate_proj.scales": "model-00010-of-00058.safetensors", + "language_model.model.layers.11.block_sparse_moe.shared_experts.gate_proj.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.11.block_sparse_moe.shared_experts.up_proj.scales": "model-00010-of-00058.safetensors", + "language_model.model.layers.11.block_sparse_moe.shared_experts.up_proj.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.11.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00010-of-00058.safetensors", + "language_model.model.layers.11.block_sparse_moe.switch_mlp.down_proj.scales": "model-00010-of-00058.safetensors", + "language_model.model.layers.11.block_sparse_moe.switch_mlp.down_proj.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.11.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00010-of-00058.safetensors", + "language_model.model.layers.11.block_sparse_moe.switch_mlp.up_proj.scales": "model-00010-of-00058.safetensors", + "language_model.model.layers.11.block_sparse_moe.switch_mlp.up_proj.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.11.input_layernorm.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.11.post_attention_layernorm.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.11.self_attn.index_k_norm.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.self_attn.index_k_proj.scales": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.self_attn.index_k_proj.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.self_attn.index_q_norm.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.self_attn.index_q_proj.scales": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.self_attn.index_q_proj.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.self_attn.k_norm.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.self_attn.k_proj.scales": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.self_attn.k_proj.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.self_attn.o_proj.scales": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.self_attn.o_proj.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.self_attn.q_norm.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.self_attn.q_proj.scales": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.self_attn.q_proj.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.self_attn.v_proj.scales": "model-00009-of-00058.safetensors", + "language_model.model.layers.11.self_attn.v_proj.weight": "model-00009-of-00058.safetensors", + "language_model.model.layers.12.block_sparse_moe.e_score_correction_bias": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.block_sparse_moe.gate.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.block_sparse_moe.shared_experts.down_proj.scales": "model-00011-of-00058.safetensors", + "language_model.model.layers.12.block_sparse_moe.shared_experts.down_proj.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.12.block_sparse_moe.shared_experts.gate_proj.scales": "model-00011-of-00058.safetensors", + "language_model.model.layers.12.block_sparse_moe.shared_experts.gate_proj.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.12.block_sparse_moe.shared_experts.up_proj.scales": "model-00011-of-00058.safetensors", + "language_model.model.layers.12.block_sparse_moe.shared_experts.up_proj.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.12.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00011-of-00058.safetensors", + "language_model.model.layers.12.block_sparse_moe.switch_mlp.down_proj.scales": "model-00011-of-00058.safetensors", + "language_model.model.layers.12.block_sparse_moe.switch_mlp.down_proj.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.12.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00011-of-00058.safetensors", + "language_model.model.layers.12.block_sparse_moe.switch_mlp.up_proj.scales": "model-00011-of-00058.safetensors", + "language_model.model.layers.12.block_sparse_moe.switch_mlp.up_proj.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.12.input_layernorm.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.12.post_attention_layernorm.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.12.self_attn.index_k_norm.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.self_attn.index_k_proj.scales": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.self_attn.index_k_proj.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.self_attn.index_q_norm.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.self_attn.index_q_proj.scales": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.self_attn.index_q_proj.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.self_attn.k_norm.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.self_attn.k_proj.scales": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.self_attn.k_proj.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.self_attn.o_proj.scales": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.self_attn.o_proj.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.self_attn.q_norm.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.self_attn.q_proj.scales": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.self_attn.q_proj.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.self_attn.v_proj.scales": "model-00010-of-00058.safetensors", + "language_model.model.layers.12.self_attn.v_proj.weight": "model-00010-of-00058.safetensors", + "language_model.model.layers.13.block_sparse_moe.e_score_correction_bias": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.block_sparse_moe.gate.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.block_sparse_moe.shared_experts.down_proj.scales": "model-00012-of-00058.safetensors", + "language_model.model.layers.13.block_sparse_moe.shared_experts.down_proj.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.13.block_sparse_moe.shared_experts.gate_proj.scales": "model-00012-of-00058.safetensors", + "language_model.model.layers.13.block_sparse_moe.shared_experts.gate_proj.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.13.block_sparse_moe.shared_experts.up_proj.scales": "model-00012-of-00058.safetensors", + "language_model.model.layers.13.block_sparse_moe.shared_experts.up_proj.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.13.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00012-of-00058.safetensors", + "language_model.model.layers.13.block_sparse_moe.switch_mlp.down_proj.scales": "model-00012-of-00058.safetensors", + "language_model.model.layers.13.block_sparse_moe.switch_mlp.down_proj.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.13.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00012-of-00058.safetensors", + "language_model.model.layers.13.block_sparse_moe.switch_mlp.up_proj.scales": "model-00012-of-00058.safetensors", + "language_model.model.layers.13.block_sparse_moe.switch_mlp.up_proj.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.13.input_layernorm.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.13.post_attention_layernorm.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.13.self_attn.index_k_norm.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.self_attn.index_k_proj.scales": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.self_attn.index_k_proj.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.self_attn.index_q_norm.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.self_attn.index_q_proj.scales": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.self_attn.index_q_proj.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.self_attn.k_norm.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.self_attn.k_proj.scales": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.self_attn.k_proj.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.self_attn.o_proj.scales": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.self_attn.o_proj.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.self_attn.q_norm.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.self_attn.q_proj.scales": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.self_attn.q_proj.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.self_attn.v_proj.scales": "model-00011-of-00058.safetensors", + "language_model.model.layers.13.self_attn.v_proj.weight": "model-00011-of-00058.safetensors", + "language_model.model.layers.14.block_sparse_moe.e_score_correction_bias": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.block_sparse_moe.gate.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.block_sparse_moe.shared_experts.down_proj.scales": "model-00013-of-00058.safetensors", + "language_model.model.layers.14.block_sparse_moe.shared_experts.down_proj.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.14.block_sparse_moe.shared_experts.gate_proj.scales": "model-00013-of-00058.safetensors", + "language_model.model.layers.14.block_sparse_moe.shared_experts.gate_proj.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.14.block_sparse_moe.shared_experts.up_proj.scales": "model-00013-of-00058.safetensors", + "language_model.model.layers.14.block_sparse_moe.shared_experts.up_proj.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.14.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00013-of-00058.safetensors", + "language_model.model.layers.14.block_sparse_moe.switch_mlp.down_proj.scales": "model-00013-of-00058.safetensors", + "language_model.model.layers.14.block_sparse_moe.switch_mlp.down_proj.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.14.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00013-of-00058.safetensors", + "language_model.model.layers.14.block_sparse_moe.switch_mlp.up_proj.scales": "model-00013-of-00058.safetensors", + "language_model.model.layers.14.block_sparse_moe.switch_mlp.up_proj.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.14.input_layernorm.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.14.post_attention_layernorm.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.14.self_attn.index_k_norm.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.self_attn.index_k_proj.scales": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.self_attn.index_k_proj.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.self_attn.index_q_norm.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.self_attn.index_q_proj.scales": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.self_attn.index_q_proj.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.self_attn.k_norm.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.self_attn.k_proj.scales": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.self_attn.k_proj.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.self_attn.o_proj.scales": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.self_attn.o_proj.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.self_attn.q_norm.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.self_attn.q_proj.scales": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.self_attn.q_proj.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.self_attn.v_proj.scales": "model-00012-of-00058.safetensors", + "language_model.model.layers.14.self_attn.v_proj.weight": "model-00012-of-00058.safetensors", + "language_model.model.layers.15.block_sparse_moe.e_score_correction_bias": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.block_sparse_moe.gate.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.block_sparse_moe.shared_experts.down_proj.scales": "model-00014-of-00058.safetensors", + "language_model.model.layers.15.block_sparse_moe.shared_experts.down_proj.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.15.block_sparse_moe.shared_experts.gate_proj.scales": "model-00014-of-00058.safetensors", + "language_model.model.layers.15.block_sparse_moe.shared_experts.gate_proj.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.15.block_sparse_moe.shared_experts.up_proj.scales": "model-00014-of-00058.safetensors", + "language_model.model.layers.15.block_sparse_moe.shared_experts.up_proj.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.15.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00014-of-00058.safetensors", + "language_model.model.layers.15.block_sparse_moe.switch_mlp.down_proj.scales": "model-00014-of-00058.safetensors", + "language_model.model.layers.15.block_sparse_moe.switch_mlp.down_proj.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.15.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00014-of-00058.safetensors", + "language_model.model.layers.15.block_sparse_moe.switch_mlp.up_proj.scales": "model-00014-of-00058.safetensors", + "language_model.model.layers.15.block_sparse_moe.switch_mlp.up_proj.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.15.input_layernorm.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.15.post_attention_layernorm.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.15.self_attn.index_k_norm.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.self_attn.index_k_proj.scales": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.self_attn.index_k_proj.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.self_attn.index_q_norm.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.self_attn.index_q_proj.scales": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.self_attn.index_q_proj.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.self_attn.k_norm.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.self_attn.k_proj.scales": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.self_attn.k_proj.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.self_attn.o_proj.scales": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.self_attn.o_proj.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.self_attn.q_norm.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.self_attn.q_proj.scales": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.self_attn.q_proj.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.self_attn.v_proj.scales": "model-00013-of-00058.safetensors", + "language_model.model.layers.15.self_attn.v_proj.weight": "model-00013-of-00058.safetensors", + "language_model.model.layers.16.block_sparse_moe.e_score_correction_bias": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.block_sparse_moe.gate.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.block_sparse_moe.shared_experts.down_proj.scales": "model-00015-of-00058.safetensors", + "language_model.model.layers.16.block_sparse_moe.shared_experts.down_proj.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.16.block_sparse_moe.shared_experts.gate_proj.scales": "model-00015-of-00058.safetensors", + "language_model.model.layers.16.block_sparse_moe.shared_experts.gate_proj.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.16.block_sparse_moe.shared_experts.up_proj.scales": "model-00015-of-00058.safetensors", + "language_model.model.layers.16.block_sparse_moe.shared_experts.up_proj.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.16.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00015-of-00058.safetensors", + "language_model.model.layers.16.block_sparse_moe.switch_mlp.down_proj.scales": "model-00015-of-00058.safetensors", + "language_model.model.layers.16.block_sparse_moe.switch_mlp.down_proj.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.16.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00015-of-00058.safetensors", + "language_model.model.layers.16.block_sparse_moe.switch_mlp.up_proj.scales": "model-00015-of-00058.safetensors", + "language_model.model.layers.16.block_sparse_moe.switch_mlp.up_proj.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.16.input_layernorm.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.16.post_attention_layernorm.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.16.self_attn.index_k_norm.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.self_attn.index_k_proj.scales": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.self_attn.index_k_proj.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.self_attn.index_q_norm.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.self_attn.index_q_proj.scales": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.self_attn.index_q_proj.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.self_attn.k_norm.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.self_attn.k_proj.scales": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.self_attn.k_proj.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.self_attn.o_proj.scales": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.self_attn.o_proj.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.self_attn.q_norm.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.self_attn.q_proj.scales": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.self_attn.q_proj.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.self_attn.v_proj.scales": "model-00014-of-00058.safetensors", + "language_model.model.layers.16.self_attn.v_proj.weight": "model-00014-of-00058.safetensors", + "language_model.model.layers.17.block_sparse_moe.e_score_correction_bias": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.block_sparse_moe.gate.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.block_sparse_moe.shared_experts.down_proj.scales": "model-00016-of-00058.safetensors", + "language_model.model.layers.17.block_sparse_moe.shared_experts.down_proj.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.17.block_sparse_moe.shared_experts.gate_proj.scales": "model-00016-of-00058.safetensors", + "language_model.model.layers.17.block_sparse_moe.shared_experts.gate_proj.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.17.block_sparse_moe.shared_experts.up_proj.scales": "model-00016-of-00058.safetensors", + "language_model.model.layers.17.block_sparse_moe.shared_experts.up_proj.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.17.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00016-of-00058.safetensors", + "language_model.model.layers.17.block_sparse_moe.switch_mlp.down_proj.scales": "model-00016-of-00058.safetensors", + "language_model.model.layers.17.block_sparse_moe.switch_mlp.down_proj.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.17.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00016-of-00058.safetensors", + "language_model.model.layers.17.block_sparse_moe.switch_mlp.up_proj.scales": "model-00016-of-00058.safetensors", + "language_model.model.layers.17.block_sparse_moe.switch_mlp.up_proj.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.17.input_layernorm.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.17.post_attention_layernorm.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.17.self_attn.index_k_norm.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.self_attn.index_k_proj.scales": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.self_attn.index_k_proj.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.self_attn.index_q_norm.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.self_attn.index_q_proj.scales": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.self_attn.index_q_proj.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.self_attn.k_norm.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.self_attn.k_proj.scales": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.self_attn.k_proj.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.self_attn.o_proj.scales": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.self_attn.o_proj.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.self_attn.q_norm.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.self_attn.q_proj.scales": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.self_attn.q_proj.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.self_attn.v_proj.scales": "model-00015-of-00058.safetensors", + "language_model.model.layers.17.self_attn.v_proj.weight": "model-00015-of-00058.safetensors", + "language_model.model.layers.18.block_sparse_moe.e_score_correction_bias": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.block_sparse_moe.gate.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.block_sparse_moe.shared_experts.down_proj.scales": "model-00017-of-00058.safetensors", + "language_model.model.layers.18.block_sparse_moe.shared_experts.down_proj.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.18.block_sparse_moe.shared_experts.gate_proj.scales": "model-00017-of-00058.safetensors", + "language_model.model.layers.18.block_sparse_moe.shared_experts.gate_proj.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.18.block_sparse_moe.shared_experts.up_proj.scales": "model-00017-of-00058.safetensors", + "language_model.model.layers.18.block_sparse_moe.shared_experts.up_proj.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.18.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00017-of-00058.safetensors", + "language_model.model.layers.18.block_sparse_moe.switch_mlp.down_proj.scales": "model-00017-of-00058.safetensors", + "language_model.model.layers.18.block_sparse_moe.switch_mlp.down_proj.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.18.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00017-of-00058.safetensors", + "language_model.model.layers.18.block_sparse_moe.switch_mlp.up_proj.scales": "model-00017-of-00058.safetensors", + "language_model.model.layers.18.block_sparse_moe.switch_mlp.up_proj.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.18.input_layernorm.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.18.post_attention_layernorm.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.18.self_attn.index_k_norm.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.self_attn.index_k_proj.scales": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.self_attn.index_k_proj.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.self_attn.index_q_norm.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.self_attn.index_q_proj.scales": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.self_attn.index_q_proj.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.self_attn.k_norm.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.self_attn.k_proj.scales": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.self_attn.k_proj.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.self_attn.o_proj.scales": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.self_attn.o_proj.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.self_attn.q_norm.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.self_attn.q_proj.scales": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.self_attn.q_proj.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.self_attn.v_proj.scales": "model-00016-of-00058.safetensors", + "language_model.model.layers.18.self_attn.v_proj.weight": "model-00016-of-00058.safetensors", + "language_model.model.layers.19.block_sparse_moe.e_score_correction_bias": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.block_sparse_moe.gate.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.block_sparse_moe.shared_experts.down_proj.scales": "model-00018-of-00058.safetensors", + "language_model.model.layers.19.block_sparse_moe.shared_experts.down_proj.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.19.block_sparse_moe.shared_experts.gate_proj.scales": "model-00018-of-00058.safetensors", + "language_model.model.layers.19.block_sparse_moe.shared_experts.gate_proj.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.19.block_sparse_moe.shared_experts.up_proj.scales": "model-00018-of-00058.safetensors", + "language_model.model.layers.19.block_sparse_moe.shared_experts.up_proj.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.19.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00018-of-00058.safetensors", + "language_model.model.layers.19.block_sparse_moe.switch_mlp.down_proj.scales": "model-00018-of-00058.safetensors", + "language_model.model.layers.19.block_sparse_moe.switch_mlp.down_proj.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.19.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00018-of-00058.safetensors", + "language_model.model.layers.19.block_sparse_moe.switch_mlp.up_proj.scales": "model-00018-of-00058.safetensors", + "language_model.model.layers.19.block_sparse_moe.switch_mlp.up_proj.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.19.input_layernorm.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.19.post_attention_layernorm.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.19.self_attn.index_k_norm.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.self_attn.index_k_proj.scales": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.self_attn.index_k_proj.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.self_attn.index_q_norm.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.self_attn.index_q_proj.scales": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.self_attn.index_q_proj.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.self_attn.k_norm.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.self_attn.k_proj.scales": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.self_attn.k_proj.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.self_attn.o_proj.scales": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.self_attn.o_proj.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.self_attn.q_norm.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.self_attn.q_proj.scales": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.self_attn.q_proj.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.self_attn.v_proj.scales": "model-00017-of-00058.safetensors", + "language_model.model.layers.19.self_attn.v_proj.weight": "model-00017-of-00058.safetensors", + "language_model.model.layers.2.input_layernorm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.mlp.down_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.mlp.down_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.mlp.gate_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.mlp.up_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.mlp.up_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.self_attn.index_k_norm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.self_attn.index_k_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.self_attn.index_k_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.self_attn.index_q_norm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.self_attn.index_q_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.self_attn.index_q_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.self_attn.k_norm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.self_attn.k_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.self_attn.o_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.self_attn.q_norm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.self_attn.q_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.self_attn.v_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.20.block_sparse_moe.e_score_correction_bias": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.block_sparse_moe.gate.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.block_sparse_moe.shared_experts.down_proj.scales": "model-00019-of-00058.safetensors", + "language_model.model.layers.20.block_sparse_moe.shared_experts.down_proj.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.20.block_sparse_moe.shared_experts.gate_proj.scales": "model-00019-of-00058.safetensors", + "language_model.model.layers.20.block_sparse_moe.shared_experts.gate_proj.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.20.block_sparse_moe.shared_experts.up_proj.scales": "model-00019-of-00058.safetensors", + "language_model.model.layers.20.block_sparse_moe.shared_experts.up_proj.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.20.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00019-of-00058.safetensors", + "language_model.model.layers.20.block_sparse_moe.switch_mlp.down_proj.scales": "model-00019-of-00058.safetensors", + "language_model.model.layers.20.block_sparse_moe.switch_mlp.down_proj.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.20.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00019-of-00058.safetensors", + "language_model.model.layers.20.block_sparse_moe.switch_mlp.up_proj.scales": "model-00019-of-00058.safetensors", + "language_model.model.layers.20.block_sparse_moe.switch_mlp.up_proj.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.20.input_layernorm.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.20.post_attention_layernorm.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.20.self_attn.index_k_norm.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.self_attn.index_k_proj.scales": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.self_attn.index_k_proj.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.self_attn.index_q_norm.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.self_attn.index_q_proj.scales": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.self_attn.index_q_proj.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.self_attn.k_norm.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.self_attn.k_proj.scales": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.self_attn.k_proj.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.self_attn.o_proj.scales": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.self_attn.o_proj.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.self_attn.q_norm.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.self_attn.q_proj.scales": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.self_attn.q_proj.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.self_attn.v_proj.scales": "model-00018-of-00058.safetensors", + "language_model.model.layers.20.self_attn.v_proj.weight": "model-00018-of-00058.safetensors", + "language_model.model.layers.21.block_sparse_moe.e_score_correction_bias": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.block_sparse_moe.gate.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.block_sparse_moe.shared_experts.down_proj.scales": "model-00020-of-00058.safetensors", + "language_model.model.layers.21.block_sparse_moe.shared_experts.down_proj.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.21.block_sparse_moe.shared_experts.gate_proj.scales": "model-00020-of-00058.safetensors", + "language_model.model.layers.21.block_sparse_moe.shared_experts.gate_proj.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.21.block_sparse_moe.shared_experts.up_proj.scales": "model-00020-of-00058.safetensors", + "language_model.model.layers.21.block_sparse_moe.shared_experts.up_proj.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.21.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00020-of-00058.safetensors", + "language_model.model.layers.21.block_sparse_moe.switch_mlp.down_proj.scales": "model-00020-of-00058.safetensors", + "language_model.model.layers.21.block_sparse_moe.switch_mlp.down_proj.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.21.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00020-of-00058.safetensors", + "language_model.model.layers.21.block_sparse_moe.switch_mlp.up_proj.scales": "model-00020-of-00058.safetensors", + "language_model.model.layers.21.block_sparse_moe.switch_mlp.up_proj.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.21.input_layernorm.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.21.post_attention_layernorm.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.21.self_attn.index_k_norm.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.self_attn.index_k_proj.scales": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.self_attn.index_k_proj.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.self_attn.index_q_norm.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.self_attn.index_q_proj.scales": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.self_attn.index_q_proj.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.self_attn.k_norm.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.self_attn.k_proj.scales": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.self_attn.k_proj.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.self_attn.o_proj.scales": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.self_attn.o_proj.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.self_attn.q_norm.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.self_attn.q_proj.scales": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.self_attn.q_proj.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.self_attn.v_proj.scales": "model-00019-of-00058.safetensors", + "language_model.model.layers.21.self_attn.v_proj.weight": "model-00019-of-00058.safetensors", + "language_model.model.layers.22.block_sparse_moe.e_score_correction_bias": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.block_sparse_moe.gate.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.block_sparse_moe.shared_experts.down_proj.scales": "model-00021-of-00058.safetensors", + "language_model.model.layers.22.block_sparse_moe.shared_experts.down_proj.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.22.block_sparse_moe.shared_experts.gate_proj.scales": "model-00021-of-00058.safetensors", + "language_model.model.layers.22.block_sparse_moe.shared_experts.gate_proj.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.22.block_sparse_moe.shared_experts.up_proj.scales": "model-00021-of-00058.safetensors", + "language_model.model.layers.22.block_sparse_moe.shared_experts.up_proj.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.22.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00021-of-00058.safetensors", + "language_model.model.layers.22.block_sparse_moe.switch_mlp.down_proj.scales": "model-00021-of-00058.safetensors", + "language_model.model.layers.22.block_sparse_moe.switch_mlp.down_proj.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.22.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00021-of-00058.safetensors", + "language_model.model.layers.22.block_sparse_moe.switch_mlp.up_proj.scales": "model-00021-of-00058.safetensors", + "language_model.model.layers.22.block_sparse_moe.switch_mlp.up_proj.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.22.input_layernorm.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.22.post_attention_layernorm.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.22.self_attn.index_k_norm.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.self_attn.index_k_proj.scales": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.self_attn.index_k_proj.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.self_attn.index_q_norm.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.self_attn.index_q_proj.scales": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.self_attn.index_q_proj.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.self_attn.k_norm.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.self_attn.k_proj.scales": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.self_attn.k_proj.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.self_attn.o_proj.scales": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.self_attn.o_proj.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.self_attn.q_norm.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.self_attn.q_proj.scales": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.self_attn.q_proj.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.self_attn.v_proj.scales": "model-00020-of-00058.safetensors", + "language_model.model.layers.22.self_attn.v_proj.weight": "model-00020-of-00058.safetensors", + "language_model.model.layers.23.block_sparse_moe.e_score_correction_bias": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.block_sparse_moe.gate.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.block_sparse_moe.shared_experts.down_proj.scales": "model-00022-of-00058.safetensors", + "language_model.model.layers.23.block_sparse_moe.shared_experts.down_proj.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.23.block_sparse_moe.shared_experts.gate_proj.scales": "model-00022-of-00058.safetensors", + "language_model.model.layers.23.block_sparse_moe.shared_experts.gate_proj.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.23.block_sparse_moe.shared_experts.up_proj.scales": "model-00022-of-00058.safetensors", + "language_model.model.layers.23.block_sparse_moe.shared_experts.up_proj.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.23.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00022-of-00058.safetensors", + "language_model.model.layers.23.block_sparse_moe.switch_mlp.down_proj.scales": "model-00022-of-00058.safetensors", + "language_model.model.layers.23.block_sparse_moe.switch_mlp.down_proj.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.23.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00022-of-00058.safetensors", + "language_model.model.layers.23.block_sparse_moe.switch_mlp.up_proj.scales": "model-00022-of-00058.safetensors", + "language_model.model.layers.23.block_sparse_moe.switch_mlp.up_proj.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.23.input_layernorm.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.23.post_attention_layernorm.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.23.self_attn.index_k_norm.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.self_attn.index_k_proj.scales": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.self_attn.index_k_proj.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.self_attn.index_q_norm.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.self_attn.index_q_proj.scales": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.self_attn.index_q_proj.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.self_attn.k_norm.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.self_attn.k_proj.scales": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.self_attn.k_proj.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.self_attn.o_proj.scales": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.self_attn.o_proj.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.self_attn.q_norm.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.self_attn.q_proj.scales": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.self_attn.q_proj.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.self_attn.v_proj.scales": "model-00021-of-00058.safetensors", + "language_model.model.layers.23.self_attn.v_proj.weight": "model-00021-of-00058.safetensors", + "language_model.model.layers.24.block_sparse_moe.e_score_correction_bias": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.block_sparse_moe.gate.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.block_sparse_moe.shared_experts.down_proj.scales": "model-00023-of-00058.safetensors", + "language_model.model.layers.24.block_sparse_moe.shared_experts.down_proj.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.24.block_sparse_moe.shared_experts.gate_proj.scales": "model-00023-of-00058.safetensors", + "language_model.model.layers.24.block_sparse_moe.shared_experts.gate_proj.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.24.block_sparse_moe.shared_experts.up_proj.scales": "model-00023-of-00058.safetensors", + "language_model.model.layers.24.block_sparse_moe.shared_experts.up_proj.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.24.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00023-of-00058.safetensors", + "language_model.model.layers.24.block_sparse_moe.switch_mlp.down_proj.scales": "model-00023-of-00058.safetensors", + "language_model.model.layers.24.block_sparse_moe.switch_mlp.down_proj.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.24.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00023-of-00058.safetensors", + "language_model.model.layers.24.block_sparse_moe.switch_mlp.up_proj.scales": "model-00023-of-00058.safetensors", + "language_model.model.layers.24.block_sparse_moe.switch_mlp.up_proj.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.24.input_layernorm.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.24.post_attention_layernorm.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.24.self_attn.index_k_norm.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.self_attn.index_k_proj.scales": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.self_attn.index_k_proj.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.self_attn.index_q_norm.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.self_attn.index_q_proj.scales": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.self_attn.index_q_proj.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.self_attn.k_norm.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.self_attn.k_proj.scales": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.self_attn.k_proj.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.self_attn.o_proj.scales": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.self_attn.o_proj.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.self_attn.q_norm.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.self_attn.q_proj.scales": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.self_attn.q_proj.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.self_attn.v_proj.scales": "model-00022-of-00058.safetensors", + "language_model.model.layers.24.self_attn.v_proj.weight": "model-00022-of-00058.safetensors", + "language_model.model.layers.25.block_sparse_moe.e_score_correction_bias": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.block_sparse_moe.gate.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.block_sparse_moe.shared_experts.down_proj.scales": "model-00024-of-00058.safetensors", + "language_model.model.layers.25.block_sparse_moe.shared_experts.down_proj.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.25.block_sparse_moe.shared_experts.gate_proj.scales": "model-00024-of-00058.safetensors", + "language_model.model.layers.25.block_sparse_moe.shared_experts.gate_proj.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.25.block_sparse_moe.shared_experts.up_proj.scales": "model-00024-of-00058.safetensors", + "language_model.model.layers.25.block_sparse_moe.shared_experts.up_proj.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.25.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00024-of-00058.safetensors", + "language_model.model.layers.25.block_sparse_moe.switch_mlp.down_proj.scales": "model-00024-of-00058.safetensors", + "language_model.model.layers.25.block_sparse_moe.switch_mlp.down_proj.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.25.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00024-of-00058.safetensors", + "language_model.model.layers.25.block_sparse_moe.switch_mlp.up_proj.scales": "model-00024-of-00058.safetensors", + "language_model.model.layers.25.block_sparse_moe.switch_mlp.up_proj.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.25.input_layernorm.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.25.post_attention_layernorm.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.25.self_attn.index_k_norm.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.self_attn.index_k_proj.scales": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.self_attn.index_k_proj.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.self_attn.index_q_norm.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.self_attn.index_q_proj.scales": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.self_attn.index_q_proj.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.self_attn.k_norm.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.self_attn.k_proj.scales": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.self_attn.k_proj.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.self_attn.o_proj.scales": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.self_attn.o_proj.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.self_attn.q_norm.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.self_attn.q_proj.scales": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.self_attn.q_proj.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.self_attn.v_proj.scales": "model-00023-of-00058.safetensors", + "language_model.model.layers.25.self_attn.v_proj.weight": "model-00023-of-00058.safetensors", + "language_model.model.layers.26.block_sparse_moe.e_score_correction_bias": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.block_sparse_moe.gate.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.block_sparse_moe.shared_experts.down_proj.scales": "model-00025-of-00058.safetensors", + "language_model.model.layers.26.block_sparse_moe.shared_experts.down_proj.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.26.block_sparse_moe.shared_experts.gate_proj.scales": "model-00025-of-00058.safetensors", + "language_model.model.layers.26.block_sparse_moe.shared_experts.gate_proj.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.26.block_sparse_moe.shared_experts.up_proj.scales": "model-00025-of-00058.safetensors", + "language_model.model.layers.26.block_sparse_moe.shared_experts.up_proj.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.26.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00025-of-00058.safetensors", + "language_model.model.layers.26.block_sparse_moe.switch_mlp.down_proj.scales": "model-00025-of-00058.safetensors", + "language_model.model.layers.26.block_sparse_moe.switch_mlp.down_proj.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.26.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00025-of-00058.safetensors", + "language_model.model.layers.26.block_sparse_moe.switch_mlp.up_proj.scales": "model-00025-of-00058.safetensors", + "language_model.model.layers.26.block_sparse_moe.switch_mlp.up_proj.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.26.input_layernorm.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.26.post_attention_layernorm.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.26.self_attn.index_k_norm.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.self_attn.index_k_proj.scales": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.self_attn.index_k_proj.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.self_attn.index_q_norm.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.self_attn.index_q_proj.scales": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.self_attn.index_q_proj.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.self_attn.k_norm.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.self_attn.k_proj.scales": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.self_attn.k_proj.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.self_attn.o_proj.scales": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.self_attn.o_proj.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.self_attn.q_norm.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.self_attn.q_proj.scales": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.self_attn.q_proj.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.self_attn.v_proj.scales": "model-00024-of-00058.safetensors", + "language_model.model.layers.26.self_attn.v_proj.weight": "model-00024-of-00058.safetensors", + "language_model.model.layers.27.block_sparse_moe.e_score_correction_bias": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.block_sparse_moe.gate.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.block_sparse_moe.shared_experts.down_proj.scales": "model-00026-of-00058.safetensors", + "language_model.model.layers.27.block_sparse_moe.shared_experts.down_proj.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.27.block_sparse_moe.shared_experts.gate_proj.scales": "model-00026-of-00058.safetensors", + "language_model.model.layers.27.block_sparse_moe.shared_experts.gate_proj.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.27.block_sparse_moe.shared_experts.up_proj.scales": "model-00026-of-00058.safetensors", + "language_model.model.layers.27.block_sparse_moe.shared_experts.up_proj.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.27.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00026-of-00058.safetensors", + "language_model.model.layers.27.block_sparse_moe.switch_mlp.down_proj.scales": "model-00026-of-00058.safetensors", + "language_model.model.layers.27.block_sparse_moe.switch_mlp.down_proj.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.27.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00026-of-00058.safetensors", + "language_model.model.layers.27.block_sparse_moe.switch_mlp.up_proj.scales": "model-00026-of-00058.safetensors", + "language_model.model.layers.27.block_sparse_moe.switch_mlp.up_proj.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.27.input_layernorm.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.27.post_attention_layernorm.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.27.self_attn.index_k_norm.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.self_attn.index_k_proj.scales": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.self_attn.index_k_proj.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.self_attn.index_q_norm.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.self_attn.index_q_proj.scales": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.self_attn.index_q_proj.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.self_attn.k_norm.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.self_attn.k_proj.scales": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.self_attn.k_proj.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.self_attn.o_proj.scales": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.self_attn.o_proj.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.self_attn.q_norm.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.self_attn.q_proj.scales": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.self_attn.q_proj.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.self_attn.v_proj.scales": "model-00025-of-00058.safetensors", + "language_model.model.layers.27.self_attn.v_proj.weight": "model-00025-of-00058.safetensors", + "language_model.model.layers.28.block_sparse_moe.e_score_correction_bias": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.block_sparse_moe.gate.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.block_sparse_moe.shared_experts.down_proj.scales": "model-00027-of-00058.safetensors", + "language_model.model.layers.28.block_sparse_moe.shared_experts.down_proj.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.28.block_sparse_moe.shared_experts.gate_proj.scales": "model-00027-of-00058.safetensors", + "language_model.model.layers.28.block_sparse_moe.shared_experts.gate_proj.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.28.block_sparse_moe.shared_experts.up_proj.scales": "model-00027-of-00058.safetensors", + "language_model.model.layers.28.block_sparse_moe.shared_experts.up_proj.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.28.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00027-of-00058.safetensors", + "language_model.model.layers.28.block_sparse_moe.switch_mlp.down_proj.scales": "model-00027-of-00058.safetensors", + "language_model.model.layers.28.block_sparse_moe.switch_mlp.down_proj.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.28.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00027-of-00058.safetensors", + "language_model.model.layers.28.block_sparse_moe.switch_mlp.up_proj.scales": "model-00027-of-00058.safetensors", + "language_model.model.layers.28.block_sparse_moe.switch_mlp.up_proj.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.28.input_layernorm.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.28.post_attention_layernorm.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.28.self_attn.index_k_norm.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.self_attn.index_k_proj.scales": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.self_attn.index_k_proj.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.self_attn.index_q_norm.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.self_attn.index_q_proj.scales": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.self_attn.index_q_proj.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.self_attn.k_norm.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.self_attn.k_proj.scales": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.self_attn.k_proj.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.self_attn.o_proj.scales": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.self_attn.o_proj.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.self_attn.q_norm.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.self_attn.q_proj.scales": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.self_attn.q_proj.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.self_attn.v_proj.scales": "model-00026-of-00058.safetensors", + "language_model.model.layers.28.self_attn.v_proj.weight": "model-00026-of-00058.safetensors", + "language_model.model.layers.29.block_sparse_moe.e_score_correction_bias": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.block_sparse_moe.gate.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.block_sparse_moe.shared_experts.down_proj.scales": "model-00028-of-00058.safetensors", + "language_model.model.layers.29.block_sparse_moe.shared_experts.down_proj.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.29.block_sparse_moe.shared_experts.gate_proj.scales": "model-00028-of-00058.safetensors", + "language_model.model.layers.29.block_sparse_moe.shared_experts.gate_proj.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.29.block_sparse_moe.shared_experts.up_proj.scales": "model-00028-of-00058.safetensors", + "language_model.model.layers.29.block_sparse_moe.shared_experts.up_proj.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.29.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00028-of-00058.safetensors", + "language_model.model.layers.29.block_sparse_moe.switch_mlp.down_proj.scales": "model-00028-of-00058.safetensors", + "language_model.model.layers.29.block_sparse_moe.switch_mlp.down_proj.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.29.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00028-of-00058.safetensors", + "language_model.model.layers.29.block_sparse_moe.switch_mlp.up_proj.scales": "model-00028-of-00058.safetensors", + "language_model.model.layers.29.block_sparse_moe.switch_mlp.up_proj.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.29.input_layernorm.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.29.post_attention_layernorm.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.29.self_attn.index_k_norm.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.self_attn.index_k_proj.scales": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.self_attn.index_k_proj.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.self_attn.index_q_norm.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.self_attn.index_q_proj.scales": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.self_attn.index_q_proj.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.self_attn.k_norm.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.self_attn.k_proj.scales": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.self_attn.k_proj.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.self_attn.o_proj.scales": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.self_attn.o_proj.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.self_attn.q_norm.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.self_attn.q_proj.scales": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.self_attn.q_proj.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.self_attn.v_proj.scales": "model-00027-of-00058.safetensors", + "language_model.model.layers.29.self_attn.v_proj.weight": "model-00027-of-00058.safetensors", + "language_model.model.layers.3.block_sparse_moe.e_score_correction_bias": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.block_sparse_moe.gate.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.block_sparse_moe.shared_experts.down_proj.scales": "model-00002-of-00058.safetensors", + "language_model.model.layers.3.block_sparse_moe.shared_experts.down_proj.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.3.block_sparse_moe.shared_experts.gate_proj.scales": "model-00002-of-00058.safetensors", + "language_model.model.layers.3.block_sparse_moe.shared_experts.gate_proj.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.3.block_sparse_moe.shared_experts.up_proj.scales": "model-00002-of-00058.safetensors", + "language_model.model.layers.3.block_sparse_moe.shared_experts.up_proj.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.3.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00002-of-00058.safetensors", + "language_model.model.layers.3.block_sparse_moe.switch_mlp.down_proj.scales": "model-00002-of-00058.safetensors", + "language_model.model.layers.3.block_sparse_moe.switch_mlp.down_proj.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.3.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00002-of-00058.safetensors", + "language_model.model.layers.3.block_sparse_moe.switch_mlp.up_proj.scales": "model-00002-of-00058.safetensors", + "language_model.model.layers.3.block_sparse_moe.switch_mlp.up_proj.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.3.input_layernorm.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.3.post_attention_layernorm.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.3.self_attn.index_k_norm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.self_attn.index_k_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.self_attn.index_k_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.self_attn.index_q_norm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.self_attn.index_q_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.self_attn.index_q_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.self_attn.k_norm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.self_attn.k_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.self_attn.o_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.self_attn.q_norm.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.self_attn.q_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.self_attn.v_proj.scales": "model-00001-of-00058.safetensors", + "language_model.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00058.safetensors", + "language_model.model.layers.30.block_sparse_moe.e_score_correction_bias": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.block_sparse_moe.gate.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.block_sparse_moe.shared_experts.down_proj.scales": "model-00029-of-00058.safetensors", + "language_model.model.layers.30.block_sparse_moe.shared_experts.down_proj.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.30.block_sparse_moe.shared_experts.gate_proj.scales": "model-00029-of-00058.safetensors", + "language_model.model.layers.30.block_sparse_moe.shared_experts.gate_proj.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.30.block_sparse_moe.shared_experts.up_proj.scales": "model-00029-of-00058.safetensors", + "language_model.model.layers.30.block_sparse_moe.shared_experts.up_proj.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.30.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00029-of-00058.safetensors", + "language_model.model.layers.30.block_sparse_moe.switch_mlp.down_proj.scales": "model-00029-of-00058.safetensors", + "language_model.model.layers.30.block_sparse_moe.switch_mlp.down_proj.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.30.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00029-of-00058.safetensors", + "language_model.model.layers.30.block_sparse_moe.switch_mlp.up_proj.scales": "model-00029-of-00058.safetensors", + "language_model.model.layers.30.block_sparse_moe.switch_mlp.up_proj.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.30.input_layernorm.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.30.post_attention_layernorm.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.30.self_attn.index_k_norm.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.self_attn.index_k_proj.scales": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.self_attn.index_k_proj.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.self_attn.index_q_norm.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.self_attn.index_q_proj.scales": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.self_attn.index_q_proj.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.self_attn.k_norm.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.self_attn.k_proj.scales": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.self_attn.k_proj.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.self_attn.o_proj.scales": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.self_attn.o_proj.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.self_attn.q_norm.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.self_attn.q_proj.scales": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.self_attn.q_proj.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.self_attn.v_proj.scales": "model-00028-of-00058.safetensors", + "language_model.model.layers.30.self_attn.v_proj.weight": "model-00028-of-00058.safetensors", + "language_model.model.layers.31.block_sparse_moe.e_score_correction_bias": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.block_sparse_moe.gate.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.block_sparse_moe.shared_experts.down_proj.scales": "model-00030-of-00058.safetensors", + "language_model.model.layers.31.block_sparse_moe.shared_experts.down_proj.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.31.block_sparse_moe.shared_experts.gate_proj.scales": "model-00030-of-00058.safetensors", + "language_model.model.layers.31.block_sparse_moe.shared_experts.gate_proj.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.31.block_sparse_moe.shared_experts.up_proj.scales": "model-00030-of-00058.safetensors", + "language_model.model.layers.31.block_sparse_moe.shared_experts.up_proj.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.31.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00030-of-00058.safetensors", + "language_model.model.layers.31.block_sparse_moe.switch_mlp.down_proj.scales": "model-00030-of-00058.safetensors", + "language_model.model.layers.31.block_sparse_moe.switch_mlp.down_proj.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.31.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00030-of-00058.safetensors", + "language_model.model.layers.31.block_sparse_moe.switch_mlp.up_proj.scales": "model-00030-of-00058.safetensors", + "language_model.model.layers.31.block_sparse_moe.switch_mlp.up_proj.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.31.input_layernorm.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.31.post_attention_layernorm.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.31.self_attn.index_k_norm.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.self_attn.index_k_proj.scales": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.self_attn.index_k_proj.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.self_attn.index_q_norm.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.self_attn.index_q_proj.scales": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.self_attn.index_q_proj.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.self_attn.k_norm.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.self_attn.k_proj.scales": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.self_attn.k_proj.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.self_attn.o_proj.scales": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.self_attn.o_proj.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.self_attn.q_norm.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.self_attn.q_proj.scales": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.self_attn.q_proj.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.self_attn.v_proj.scales": "model-00029-of-00058.safetensors", + "language_model.model.layers.31.self_attn.v_proj.weight": "model-00029-of-00058.safetensors", + "language_model.model.layers.32.block_sparse_moe.e_score_correction_bias": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.block_sparse_moe.gate.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.block_sparse_moe.shared_experts.down_proj.scales": "model-00031-of-00058.safetensors", + "language_model.model.layers.32.block_sparse_moe.shared_experts.down_proj.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.32.block_sparse_moe.shared_experts.gate_proj.scales": "model-00031-of-00058.safetensors", + "language_model.model.layers.32.block_sparse_moe.shared_experts.gate_proj.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.32.block_sparse_moe.shared_experts.up_proj.scales": "model-00031-of-00058.safetensors", + "language_model.model.layers.32.block_sparse_moe.shared_experts.up_proj.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.32.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00031-of-00058.safetensors", + "language_model.model.layers.32.block_sparse_moe.switch_mlp.down_proj.scales": "model-00031-of-00058.safetensors", + "language_model.model.layers.32.block_sparse_moe.switch_mlp.down_proj.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.32.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00031-of-00058.safetensors", + "language_model.model.layers.32.block_sparse_moe.switch_mlp.up_proj.scales": "model-00031-of-00058.safetensors", + "language_model.model.layers.32.block_sparse_moe.switch_mlp.up_proj.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.32.input_layernorm.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.32.post_attention_layernorm.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.32.self_attn.index_k_norm.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.self_attn.index_k_proj.scales": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.self_attn.index_k_proj.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.self_attn.index_q_norm.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.self_attn.index_q_proj.scales": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.self_attn.index_q_proj.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.self_attn.k_norm.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.self_attn.k_proj.scales": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.self_attn.k_proj.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.self_attn.o_proj.scales": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.self_attn.o_proj.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.self_attn.q_norm.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.self_attn.q_proj.scales": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.self_attn.q_proj.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.self_attn.v_proj.scales": "model-00030-of-00058.safetensors", + "language_model.model.layers.32.self_attn.v_proj.weight": "model-00030-of-00058.safetensors", + "language_model.model.layers.33.block_sparse_moe.e_score_correction_bias": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.block_sparse_moe.gate.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.block_sparse_moe.shared_experts.down_proj.scales": "model-00032-of-00058.safetensors", + "language_model.model.layers.33.block_sparse_moe.shared_experts.down_proj.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.33.block_sparse_moe.shared_experts.gate_proj.scales": "model-00032-of-00058.safetensors", + "language_model.model.layers.33.block_sparse_moe.shared_experts.gate_proj.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.33.block_sparse_moe.shared_experts.up_proj.scales": "model-00032-of-00058.safetensors", + "language_model.model.layers.33.block_sparse_moe.shared_experts.up_proj.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.33.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00032-of-00058.safetensors", + "language_model.model.layers.33.block_sparse_moe.switch_mlp.down_proj.scales": "model-00032-of-00058.safetensors", + "language_model.model.layers.33.block_sparse_moe.switch_mlp.down_proj.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.33.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00032-of-00058.safetensors", + "language_model.model.layers.33.block_sparse_moe.switch_mlp.up_proj.scales": "model-00032-of-00058.safetensors", + "language_model.model.layers.33.block_sparse_moe.switch_mlp.up_proj.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.33.input_layernorm.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.33.post_attention_layernorm.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.33.self_attn.index_k_norm.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.self_attn.index_k_proj.scales": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.self_attn.index_k_proj.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.self_attn.index_q_norm.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.self_attn.index_q_proj.scales": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.self_attn.index_q_proj.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.self_attn.k_norm.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.self_attn.k_proj.scales": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.self_attn.k_proj.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.self_attn.o_proj.scales": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.self_attn.o_proj.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.self_attn.q_norm.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.self_attn.q_proj.scales": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.self_attn.q_proj.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.self_attn.v_proj.scales": "model-00031-of-00058.safetensors", + "language_model.model.layers.33.self_attn.v_proj.weight": "model-00031-of-00058.safetensors", + "language_model.model.layers.34.block_sparse_moe.e_score_correction_bias": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.block_sparse_moe.gate.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.block_sparse_moe.shared_experts.down_proj.scales": "model-00033-of-00058.safetensors", + "language_model.model.layers.34.block_sparse_moe.shared_experts.down_proj.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.34.block_sparse_moe.shared_experts.gate_proj.scales": "model-00033-of-00058.safetensors", + "language_model.model.layers.34.block_sparse_moe.shared_experts.gate_proj.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.34.block_sparse_moe.shared_experts.up_proj.scales": "model-00033-of-00058.safetensors", + "language_model.model.layers.34.block_sparse_moe.shared_experts.up_proj.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.34.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00033-of-00058.safetensors", + "language_model.model.layers.34.block_sparse_moe.switch_mlp.down_proj.scales": "model-00033-of-00058.safetensors", + "language_model.model.layers.34.block_sparse_moe.switch_mlp.down_proj.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.34.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00033-of-00058.safetensors", + "language_model.model.layers.34.block_sparse_moe.switch_mlp.up_proj.scales": "model-00033-of-00058.safetensors", + "language_model.model.layers.34.block_sparse_moe.switch_mlp.up_proj.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.34.input_layernorm.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.34.post_attention_layernorm.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.34.self_attn.index_k_norm.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.self_attn.index_k_proj.scales": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.self_attn.index_k_proj.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.self_attn.index_q_norm.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.self_attn.index_q_proj.scales": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.self_attn.index_q_proj.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.self_attn.k_norm.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.self_attn.k_proj.scales": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.self_attn.k_proj.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.self_attn.o_proj.scales": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.self_attn.o_proj.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.self_attn.q_norm.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.self_attn.q_proj.scales": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.self_attn.q_proj.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.self_attn.v_proj.scales": "model-00032-of-00058.safetensors", + "language_model.model.layers.34.self_attn.v_proj.weight": "model-00032-of-00058.safetensors", + "language_model.model.layers.35.block_sparse_moe.e_score_correction_bias": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.block_sparse_moe.gate.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.block_sparse_moe.shared_experts.down_proj.scales": "model-00034-of-00058.safetensors", + "language_model.model.layers.35.block_sparse_moe.shared_experts.down_proj.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.35.block_sparse_moe.shared_experts.gate_proj.scales": "model-00034-of-00058.safetensors", + "language_model.model.layers.35.block_sparse_moe.shared_experts.gate_proj.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.35.block_sparse_moe.shared_experts.up_proj.scales": "model-00034-of-00058.safetensors", + "language_model.model.layers.35.block_sparse_moe.shared_experts.up_proj.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.35.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00034-of-00058.safetensors", + "language_model.model.layers.35.block_sparse_moe.switch_mlp.down_proj.scales": "model-00034-of-00058.safetensors", + "language_model.model.layers.35.block_sparse_moe.switch_mlp.down_proj.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.35.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00034-of-00058.safetensors", + "language_model.model.layers.35.block_sparse_moe.switch_mlp.up_proj.scales": "model-00034-of-00058.safetensors", + "language_model.model.layers.35.block_sparse_moe.switch_mlp.up_proj.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.35.input_layernorm.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.35.post_attention_layernorm.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.35.self_attn.index_k_norm.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.self_attn.index_k_proj.scales": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.self_attn.index_k_proj.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.self_attn.index_q_norm.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.self_attn.index_q_proj.scales": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.self_attn.index_q_proj.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.self_attn.k_norm.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.self_attn.k_proj.scales": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.self_attn.k_proj.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.self_attn.o_proj.scales": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.self_attn.o_proj.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.self_attn.q_norm.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.self_attn.q_proj.scales": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.self_attn.q_proj.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.self_attn.v_proj.scales": "model-00033-of-00058.safetensors", + "language_model.model.layers.35.self_attn.v_proj.weight": "model-00033-of-00058.safetensors", + "language_model.model.layers.36.block_sparse_moe.e_score_correction_bias": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.block_sparse_moe.gate.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.block_sparse_moe.shared_experts.down_proj.scales": "model-00035-of-00058.safetensors", + "language_model.model.layers.36.block_sparse_moe.shared_experts.down_proj.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.36.block_sparse_moe.shared_experts.gate_proj.scales": "model-00035-of-00058.safetensors", + "language_model.model.layers.36.block_sparse_moe.shared_experts.gate_proj.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.36.block_sparse_moe.shared_experts.up_proj.scales": "model-00035-of-00058.safetensors", + "language_model.model.layers.36.block_sparse_moe.shared_experts.up_proj.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.36.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00035-of-00058.safetensors", + "language_model.model.layers.36.block_sparse_moe.switch_mlp.down_proj.scales": "model-00035-of-00058.safetensors", + "language_model.model.layers.36.block_sparse_moe.switch_mlp.down_proj.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.36.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00035-of-00058.safetensors", + "language_model.model.layers.36.block_sparse_moe.switch_mlp.up_proj.scales": "model-00035-of-00058.safetensors", + "language_model.model.layers.36.block_sparse_moe.switch_mlp.up_proj.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.36.input_layernorm.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.36.post_attention_layernorm.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.36.self_attn.index_k_norm.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.self_attn.index_k_proj.scales": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.self_attn.index_k_proj.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.self_attn.index_q_norm.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.self_attn.index_q_proj.scales": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.self_attn.index_q_proj.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.self_attn.k_norm.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.self_attn.k_proj.scales": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.self_attn.k_proj.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.self_attn.o_proj.scales": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.self_attn.o_proj.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.self_attn.q_norm.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.self_attn.q_proj.scales": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.self_attn.q_proj.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.self_attn.v_proj.scales": "model-00034-of-00058.safetensors", + "language_model.model.layers.36.self_attn.v_proj.weight": "model-00034-of-00058.safetensors", + "language_model.model.layers.37.block_sparse_moe.e_score_correction_bias": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.block_sparse_moe.gate.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.block_sparse_moe.shared_experts.down_proj.scales": "model-00036-of-00058.safetensors", + "language_model.model.layers.37.block_sparse_moe.shared_experts.down_proj.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.37.block_sparse_moe.shared_experts.gate_proj.scales": "model-00036-of-00058.safetensors", + "language_model.model.layers.37.block_sparse_moe.shared_experts.gate_proj.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.37.block_sparse_moe.shared_experts.up_proj.scales": "model-00036-of-00058.safetensors", + "language_model.model.layers.37.block_sparse_moe.shared_experts.up_proj.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.37.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00036-of-00058.safetensors", + "language_model.model.layers.37.block_sparse_moe.switch_mlp.down_proj.scales": "model-00036-of-00058.safetensors", + "language_model.model.layers.37.block_sparse_moe.switch_mlp.down_proj.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.37.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00036-of-00058.safetensors", + "language_model.model.layers.37.block_sparse_moe.switch_mlp.up_proj.scales": "model-00036-of-00058.safetensors", + "language_model.model.layers.37.block_sparse_moe.switch_mlp.up_proj.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.37.input_layernorm.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.37.post_attention_layernorm.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.37.self_attn.index_k_norm.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.self_attn.index_k_proj.scales": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.self_attn.index_k_proj.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.self_attn.index_q_norm.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.self_attn.index_q_proj.scales": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.self_attn.index_q_proj.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.self_attn.k_norm.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.self_attn.k_proj.scales": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.self_attn.k_proj.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.self_attn.o_proj.scales": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.self_attn.o_proj.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.self_attn.q_norm.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.self_attn.q_proj.scales": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.self_attn.q_proj.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.self_attn.v_proj.scales": "model-00035-of-00058.safetensors", + "language_model.model.layers.37.self_attn.v_proj.weight": "model-00035-of-00058.safetensors", + "language_model.model.layers.38.block_sparse_moe.e_score_correction_bias": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.block_sparse_moe.gate.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.block_sparse_moe.shared_experts.down_proj.scales": "model-00037-of-00058.safetensors", + "language_model.model.layers.38.block_sparse_moe.shared_experts.down_proj.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.38.block_sparse_moe.shared_experts.gate_proj.scales": "model-00037-of-00058.safetensors", + "language_model.model.layers.38.block_sparse_moe.shared_experts.gate_proj.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.38.block_sparse_moe.shared_experts.up_proj.scales": "model-00037-of-00058.safetensors", + "language_model.model.layers.38.block_sparse_moe.shared_experts.up_proj.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.38.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00037-of-00058.safetensors", + "language_model.model.layers.38.block_sparse_moe.switch_mlp.down_proj.scales": "model-00037-of-00058.safetensors", + "language_model.model.layers.38.block_sparse_moe.switch_mlp.down_proj.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.38.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00037-of-00058.safetensors", + "language_model.model.layers.38.block_sparse_moe.switch_mlp.up_proj.scales": "model-00037-of-00058.safetensors", + "language_model.model.layers.38.block_sparse_moe.switch_mlp.up_proj.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.38.input_layernorm.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.38.post_attention_layernorm.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.38.self_attn.index_k_norm.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.self_attn.index_k_proj.scales": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.self_attn.index_k_proj.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.self_attn.index_q_norm.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.self_attn.index_q_proj.scales": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.self_attn.index_q_proj.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.self_attn.k_norm.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.self_attn.k_proj.scales": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.self_attn.k_proj.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.self_attn.o_proj.scales": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.self_attn.o_proj.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.self_attn.q_norm.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.self_attn.q_proj.scales": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.self_attn.q_proj.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.self_attn.v_proj.scales": "model-00036-of-00058.safetensors", + "language_model.model.layers.38.self_attn.v_proj.weight": "model-00036-of-00058.safetensors", + "language_model.model.layers.39.block_sparse_moe.e_score_correction_bias": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.block_sparse_moe.gate.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.block_sparse_moe.shared_experts.down_proj.scales": "model-00038-of-00058.safetensors", + "language_model.model.layers.39.block_sparse_moe.shared_experts.down_proj.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.39.block_sparse_moe.shared_experts.gate_proj.scales": "model-00038-of-00058.safetensors", + "language_model.model.layers.39.block_sparse_moe.shared_experts.gate_proj.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.39.block_sparse_moe.shared_experts.up_proj.scales": "model-00038-of-00058.safetensors", + "language_model.model.layers.39.block_sparse_moe.shared_experts.up_proj.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.39.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00038-of-00058.safetensors", + "language_model.model.layers.39.block_sparse_moe.switch_mlp.down_proj.scales": "model-00038-of-00058.safetensors", + "language_model.model.layers.39.block_sparse_moe.switch_mlp.down_proj.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.39.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00038-of-00058.safetensors", + "language_model.model.layers.39.block_sparse_moe.switch_mlp.up_proj.scales": "model-00038-of-00058.safetensors", + "language_model.model.layers.39.block_sparse_moe.switch_mlp.up_proj.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.39.input_layernorm.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.39.post_attention_layernorm.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.39.self_attn.index_k_norm.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.self_attn.index_k_proj.scales": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.self_attn.index_k_proj.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.self_attn.index_q_norm.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.self_attn.index_q_proj.scales": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.self_attn.index_q_proj.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.self_attn.k_norm.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.self_attn.k_proj.scales": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.self_attn.k_proj.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.self_attn.o_proj.scales": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.self_attn.o_proj.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.self_attn.q_norm.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.self_attn.q_proj.scales": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.self_attn.q_proj.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.self_attn.v_proj.scales": "model-00037-of-00058.safetensors", + "language_model.model.layers.39.self_attn.v_proj.weight": "model-00037-of-00058.safetensors", + "language_model.model.layers.4.block_sparse_moe.e_score_correction_bias": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.block_sparse_moe.gate.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.block_sparse_moe.shared_experts.down_proj.scales": "model-00003-of-00058.safetensors", + "language_model.model.layers.4.block_sparse_moe.shared_experts.down_proj.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.4.block_sparse_moe.shared_experts.gate_proj.scales": "model-00003-of-00058.safetensors", + "language_model.model.layers.4.block_sparse_moe.shared_experts.gate_proj.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.4.block_sparse_moe.shared_experts.up_proj.scales": "model-00003-of-00058.safetensors", + "language_model.model.layers.4.block_sparse_moe.shared_experts.up_proj.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.4.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00003-of-00058.safetensors", + "language_model.model.layers.4.block_sparse_moe.switch_mlp.down_proj.scales": "model-00003-of-00058.safetensors", + "language_model.model.layers.4.block_sparse_moe.switch_mlp.down_proj.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.4.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00003-of-00058.safetensors", + "language_model.model.layers.4.block_sparse_moe.switch_mlp.up_proj.scales": "model-00003-of-00058.safetensors", + "language_model.model.layers.4.block_sparse_moe.switch_mlp.up_proj.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.4.input_layernorm.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.4.post_attention_layernorm.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.4.self_attn.index_k_norm.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.self_attn.index_k_proj.scales": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.self_attn.index_k_proj.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.self_attn.index_q_norm.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.self_attn.index_q_proj.scales": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.self_attn.index_q_proj.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.self_attn.k_norm.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.self_attn.k_proj.scales": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.self_attn.k_proj.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.self_attn.o_proj.scales": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.self_attn.o_proj.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.self_attn.q_norm.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.self_attn.q_proj.scales": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.self_attn.q_proj.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.self_attn.v_proj.scales": "model-00002-of-00058.safetensors", + "language_model.model.layers.4.self_attn.v_proj.weight": "model-00002-of-00058.safetensors", + "language_model.model.layers.40.block_sparse_moe.e_score_correction_bias": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.block_sparse_moe.gate.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.block_sparse_moe.shared_experts.down_proj.scales": "model-00039-of-00058.safetensors", + "language_model.model.layers.40.block_sparse_moe.shared_experts.down_proj.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.40.block_sparse_moe.shared_experts.gate_proj.scales": "model-00039-of-00058.safetensors", + "language_model.model.layers.40.block_sparse_moe.shared_experts.gate_proj.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.40.block_sparse_moe.shared_experts.up_proj.scales": "model-00039-of-00058.safetensors", + "language_model.model.layers.40.block_sparse_moe.shared_experts.up_proj.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.40.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00039-of-00058.safetensors", + "language_model.model.layers.40.block_sparse_moe.switch_mlp.down_proj.scales": "model-00039-of-00058.safetensors", + "language_model.model.layers.40.block_sparse_moe.switch_mlp.down_proj.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.40.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00039-of-00058.safetensors", + "language_model.model.layers.40.block_sparse_moe.switch_mlp.up_proj.scales": "model-00039-of-00058.safetensors", + "language_model.model.layers.40.block_sparse_moe.switch_mlp.up_proj.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.40.input_layernorm.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.40.post_attention_layernorm.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.40.self_attn.index_k_norm.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.self_attn.index_k_proj.scales": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.self_attn.index_k_proj.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.self_attn.index_q_norm.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.self_attn.index_q_proj.scales": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.self_attn.index_q_proj.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.self_attn.k_norm.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.self_attn.k_proj.scales": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.self_attn.k_proj.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.self_attn.o_proj.scales": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.self_attn.o_proj.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.self_attn.q_norm.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.self_attn.q_proj.scales": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.self_attn.q_proj.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.self_attn.v_proj.scales": "model-00038-of-00058.safetensors", + "language_model.model.layers.40.self_attn.v_proj.weight": "model-00038-of-00058.safetensors", + "language_model.model.layers.41.block_sparse_moe.e_score_correction_bias": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.block_sparse_moe.gate.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.block_sparse_moe.shared_experts.down_proj.scales": "model-00040-of-00058.safetensors", + "language_model.model.layers.41.block_sparse_moe.shared_experts.down_proj.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.41.block_sparse_moe.shared_experts.gate_proj.scales": "model-00040-of-00058.safetensors", + "language_model.model.layers.41.block_sparse_moe.shared_experts.gate_proj.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.41.block_sparse_moe.shared_experts.up_proj.scales": "model-00040-of-00058.safetensors", + "language_model.model.layers.41.block_sparse_moe.shared_experts.up_proj.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.41.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00040-of-00058.safetensors", + "language_model.model.layers.41.block_sparse_moe.switch_mlp.down_proj.scales": "model-00040-of-00058.safetensors", + "language_model.model.layers.41.block_sparse_moe.switch_mlp.down_proj.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.41.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00040-of-00058.safetensors", + "language_model.model.layers.41.block_sparse_moe.switch_mlp.up_proj.scales": "model-00040-of-00058.safetensors", + "language_model.model.layers.41.block_sparse_moe.switch_mlp.up_proj.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.41.input_layernorm.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.41.post_attention_layernorm.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.41.self_attn.index_k_norm.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.self_attn.index_k_proj.scales": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.self_attn.index_k_proj.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.self_attn.index_q_norm.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.self_attn.index_q_proj.scales": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.self_attn.index_q_proj.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.self_attn.k_norm.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.self_attn.k_proj.scales": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.self_attn.k_proj.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.self_attn.o_proj.scales": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.self_attn.o_proj.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.self_attn.q_norm.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.self_attn.q_proj.scales": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.self_attn.q_proj.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.self_attn.v_proj.scales": "model-00039-of-00058.safetensors", + "language_model.model.layers.41.self_attn.v_proj.weight": "model-00039-of-00058.safetensors", + "language_model.model.layers.42.block_sparse_moe.e_score_correction_bias": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.block_sparse_moe.gate.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.block_sparse_moe.shared_experts.down_proj.scales": "model-00041-of-00058.safetensors", + "language_model.model.layers.42.block_sparse_moe.shared_experts.down_proj.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.42.block_sparse_moe.shared_experts.gate_proj.scales": "model-00041-of-00058.safetensors", + "language_model.model.layers.42.block_sparse_moe.shared_experts.gate_proj.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.42.block_sparse_moe.shared_experts.up_proj.scales": "model-00041-of-00058.safetensors", + "language_model.model.layers.42.block_sparse_moe.shared_experts.up_proj.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.42.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00041-of-00058.safetensors", + "language_model.model.layers.42.block_sparse_moe.switch_mlp.down_proj.scales": "model-00041-of-00058.safetensors", + "language_model.model.layers.42.block_sparse_moe.switch_mlp.down_proj.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.42.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00041-of-00058.safetensors", + "language_model.model.layers.42.block_sparse_moe.switch_mlp.up_proj.scales": "model-00041-of-00058.safetensors", + "language_model.model.layers.42.block_sparse_moe.switch_mlp.up_proj.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.42.input_layernorm.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.42.post_attention_layernorm.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.42.self_attn.index_k_norm.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.self_attn.index_k_proj.scales": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.self_attn.index_k_proj.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.self_attn.index_q_norm.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.self_attn.index_q_proj.scales": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.self_attn.index_q_proj.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.self_attn.k_norm.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.self_attn.k_proj.scales": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.self_attn.k_proj.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.self_attn.o_proj.scales": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.self_attn.o_proj.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.self_attn.q_norm.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.self_attn.q_proj.scales": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.self_attn.q_proj.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.self_attn.v_proj.scales": "model-00040-of-00058.safetensors", + "language_model.model.layers.42.self_attn.v_proj.weight": "model-00040-of-00058.safetensors", + "language_model.model.layers.43.block_sparse_moe.e_score_correction_bias": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.block_sparse_moe.gate.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.block_sparse_moe.shared_experts.down_proj.scales": "model-00042-of-00058.safetensors", + "language_model.model.layers.43.block_sparse_moe.shared_experts.down_proj.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.43.block_sparse_moe.shared_experts.gate_proj.scales": "model-00042-of-00058.safetensors", + "language_model.model.layers.43.block_sparse_moe.shared_experts.gate_proj.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.43.block_sparse_moe.shared_experts.up_proj.scales": "model-00042-of-00058.safetensors", + "language_model.model.layers.43.block_sparse_moe.shared_experts.up_proj.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.43.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00042-of-00058.safetensors", + "language_model.model.layers.43.block_sparse_moe.switch_mlp.down_proj.scales": "model-00042-of-00058.safetensors", + "language_model.model.layers.43.block_sparse_moe.switch_mlp.down_proj.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.43.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00042-of-00058.safetensors", + "language_model.model.layers.43.block_sparse_moe.switch_mlp.up_proj.scales": "model-00042-of-00058.safetensors", + "language_model.model.layers.43.block_sparse_moe.switch_mlp.up_proj.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.43.input_layernorm.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.43.post_attention_layernorm.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.43.self_attn.index_k_norm.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.self_attn.index_k_proj.scales": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.self_attn.index_k_proj.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.self_attn.index_q_norm.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.self_attn.index_q_proj.scales": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.self_attn.index_q_proj.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.self_attn.k_norm.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.self_attn.k_proj.scales": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.self_attn.k_proj.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.self_attn.o_proj.scales": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.self_attn.o_proj.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.self_attn.q_norm.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.self_attn.q_proj.scales": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.self_attn.q_proj.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.self_attn.v_proj.scales": "model-00041-of-00058.safetensors", + "language_model.model.layers.43.self_attn.v_proj.weight": "model-00041-of-00058.safetensors", + "language_model.model.layers.44.block_sparse_moe.e_score_correction_bias": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.block_sparse_moe.gate.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.block_sparse_moe.shared_experts.down_proj.scales": "model-00043-of-00058.safetensors", + "language_model.model.layers.44.block_sparse_moe.shared_experts.down_proj.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.44.block_sparse_moe.shared_experts.gate_proj.scales": "model-00043-of-00058.safetensors", + "language_model.model.layers.44.block_sparse_moe.shared_experts.gate_proj.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.44.block_sparse_moe.shared_experts.up_proj.scales": "model-00043-of-00058.safetensors", + "language_model.model.layers.44.block_sparse_moe.shared_experts.up_proj.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.44.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00043-of-00058.safetensors", + "language_model.model.layers.44.block_sparse_moe.switch_mlp.down_proj.scales": "model-00043-of-00058.safetensors", + "language_model.model.layers.44.block_sparse_moe.switch_mlp.down_proj.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.44.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00043-of-00058.safetensors", + "language_model.model.layers.44.block_sparse_moe.switch_mlp.up_proj.scales": "model-00043-of-00058.safetensors", + "language_model.model.layers.44.block_sparse_moe.switch_mlp.up_proj.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.44.input_layernorm.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.44.post_attention_layernorm.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.44.self_attn.index_k_norm.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.self_attn.index_k_proj.scales": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.self_attn.index_k_proj.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.self_attn.index_q_norm.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.self_attn.index_q_proj.scales": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.self_attn.index_q_proj.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.self_attn.k_norm.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.self_attn.k_proj.scales": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.self_attn.k_proj.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.self_attn.o_proj.scales": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.self_attn.o_proj.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.self_attn.q_norm.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.self_attn.q_proj.scales": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.self_attn.q_proj.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.self_attn.v_proj.scales": "model-00042-of-00058.safetensors", + "language_model.model.layers.44.self_attn.v_proj.weight": "model-00042-of-00058.safetensors", + "language_model.model.layers.45.block_sparse_moe.e_score_correction_bias": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.block_sparse_moe.gate.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.block_sparse_moe.shared_experts.down_proj.scales": "model-00044-of-00058.safetensors", + "language_model.model.layers.45.block_sparse_moe.shared_experts.down_proj.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.45.block_sparse_moe.shared_experts.gate_proj.scales": "model-00044-of-00058.safetensors", + "language_model.model.layers.45.block_sparse_moe.shared_experts.gate_proj.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.45.block_sparse_moe.shared_experts.up_proj.scales": "model-00044-of-00058.safetensors", + "language_model.model.layers.45.block_sparse_moe.shared_experts.up_proj.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.45.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00044-of-00058.safetensors", + "language_model.model.layers.45.block_sparse_moe.switch_mlp.down_proj.scales": "model-00044-of-00058.safetensors", + "language_model.model.layers.45.block_sparse_moe.switch_mlp.down_proj.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.45.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00044-of-00058.safetensors", + "language_model.model.layers.45.block_sparse_moe.switch_mlp.up_proj.scales": "model-00044-of-00058.safetensors", + "language_model.model.layers.45.block_sparse_moe.switch_mlp.up_proj.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.45.input_layernorm.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.45.post_attention_layernorm.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.45.self_attn.index_k_norm.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.self_attn.index_k_proj.scales": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.self_attn.index_k_proj.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.self_attn.index_q_norm.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.self_attn.index_q_proj.scales": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.self_attn.index_q_proj.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.self_attn.k_norm.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.self_attn.k_proj.scales": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.self_attn.k_proj.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.self_attn.o_proj.scales": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.self_attn.o_proj.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.self_attn.q_norm.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.self_attn.q_proj.scales": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.self_attn.q_proj.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.self_attn.v_proj.scales": "model-00043-of-00058.safetensors", + "language_model.model.layers.45.self_attn.v_proj.weight": "model-00043-of-00058.safetensors", + "language_model.model.layers.46.block_sparse_moe.e_score_correction_bias": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.block_sparse_moe.gate.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.block_sparse_moe.shared_experts.down_proj.scales": "model-00045-of-00058.safetensors", + "language_model.model.layers.46.block_sparse_moe.shared_experts.down_proj.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.46.block_sparse_moe.shared_experts.gate_proj.scales": "model-00045-of-00058.safetensors", + "language_model.model.layers.46.block_sparse_moe.shared_experts.gate_proj.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.46.block_sparse_moe.shared_experts.up_proj.scales": "model-00045-of-00058.safetensors", + "language_model.model.layers.46.block_sparse_moe.shared_experts.up_proj.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.46.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00045-of-00058.safetensors", + "language_model.model.layers.46.block_sparse_moe.switch_mlp.down_proj.scales": "model-00045-of-00058.safetensors", + "language_model.model.layers.46.block_sparse_moe.switch_mlp.down_proj.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.46.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00045-of-00058.safetensors", + "language_model.model.layers.46.block_sparse_moe.switch_mlp.up_proj.scales": "model-00045-of-00058.safetensors", + "language_model.model.layers.46.block_sparse_moe.switch_mlp.up_proj.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.46.input_layernorm.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.46.post_attention_layernorm.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.46.self_attn.index_k_norm.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.self_attn.index_k_proj.scales": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.self_attn.index_k_proj.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.self_attn.index_q_norm.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.self_attn.index_q_proj.scales": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.self_attn.index_q_proj.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.self_attn.k_norm.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.self_attn.k_proj.scales": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.self_attn.k_proj.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.self_attn.o_proj.scales": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.self_attn.o_proj.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.self_attn.q_norm.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.self_attn.q_proj.scales": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.self_attn.q_proj.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.self_attn.v_proj.scales": "model-00044-of-00058.safetensors", + "language_model.model.layers.46.self_attn.v_proj.weight": "model-00044-of-00058.safetensors", + "language_model.model.layers.47.block_sparse_moe.e_score_correction_bias": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.block_sparse_moe.gate.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.block_sparse_moe.shared_experts.down_proj.scales": "model-00046-of-00058.safetensors", + "language_model.model.layers.47.block_sparse_moe.shared_experts.down_proj.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.47.block_sparse_moe.shared_experts.gate_proj.scales": "model-00046-of-00058.safetensors", + "language_model.model.layers.47.block_sparse_moe.shared_experts.gate_proj.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.47.block_sparse_moe.shared_experts.up_proj.scales": "model-00046-of-00058.safetensors", + "language_model.model.layers.47.block_sparse_moe.shared_experts.up_proj.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.47.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00046-of-00058.safetensors", + "language_model.model.layers.47.block_sparse_moe.switch_mlp.down_proj.scales": "model-00046-of-00058.safetensors", + "language_model.model.layers.47.block_sparse_moe.switch_mlp.down_proj.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.47.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00046-of-00058.safetensors", + "language_model.model.layers.47.block_sparse_moe.switch_mlp.up_proj.scales": "model-00046-of-00058.safetensors", + "language_model.model.layers.47.block_sparse_moe.switch_mlp.up_proj.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.47.input_layernorm.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.47.post_attention_layernorm.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.47.self_attn.index_k_norm.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.self_attn.index_k_proj.scales": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.self_attn.index_k_proj.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.self_attn.index_q_norm.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.self_attn.index_q_proj.scales": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.self_attn.index_q_proj.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.self_attn.k_norm.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.self_attn.k_proj.scales": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.self_attn.k_proj.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.self_attn.o_proj.scales": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.self_attn.o_proj.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.self_attn.q_norm.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.self_attn.q_proj.scales": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.self_attn.q_proj.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.self_attn.v_proj.scales": "model-00045-of-00058.safetensors", + "language_model.model.layers.47.self_attn.v_proj.weight": "model-00045-of-00058.safetensors", + "language_model.model.layers.48.block_sparse_moe.e_score_correction_bias": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.block_sparse_moe.gate.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.block_sparse_moe.shared_experts.down_proj.scales": "model-00047-of-00058.safetensors", + "language_model.model.layers.48.block_sparse_moe.shared_experts.down_proj.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.48.block_sparse_moe.shared_experts.gate_proj.scales": "model-00047-of-00058.safetensors", + "language_model.model.layers.48.block_sparse_moe.shared_experts.gate_proj.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.48.block_sparse_moe.shared_experts.up_proj.scales": "model-00047-of-00058.safetensors", + "language_model.model.layers.48.block_sparse_moe.shared_experts.up_proj.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.48.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00047-of-00058.safetensors", + "language_model.model.layers.48.block_sparse_moe.switch_mlp.down_proj.scales": "model-00047-of-00058.safetensors", + "language_model.model.layers.48.block_sparse_moe.switch_mlp.down_proj.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.48.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00047-of-00058.safetensors", + "language_model.model.layers.48.block_sparse_moe.switch_mlp.up_proj.scales": "model-00047-of-00058.safetensors", + "language_model.model.layers.48.block_sparse_moe.switch_mlp.up_proj.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.48.input_layernorm.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.48.post_attention_layernorm.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.48.self_attn.index_k_norm.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.self_attn.index_k_proj.scales": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.self_attn.index_k_proj.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.self_attn.index_q_norm.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.self_attn.index_q_proj.scales": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.self_attn.index_q_proj.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.self_attn.k_norm.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.self_attn.k_proj.scales": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.self_attn.k_proj.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.self_attn.o_proj.scales": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.self_attn.o_proj.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.self_attn.q_norm.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.self_attn.q_proj.scales": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.self_attn.q_proj.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.self_attn.v_proj.scales": "model-00046-of-00058.safetensors", + "language_model.model.layers.48.self_attn.v_proj.weight": "model-00046-of-00058.safetensors", + "language_model.model.layers.49.block_sparse_moe.e_score_correction_bias": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.block_sparse_moe.gate.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.block_sparse_moe.shared_experts.down_proj.scales": "model-00048-of-00058.safetensors", + "language_model.model.layers.49.block_sparse_moe.shared_experts.down_proj.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.49.block_sparse_moe.shared_experts.gate_proj.scales": "model-00048-of-00058.safetensors", + "language_model.model.layers.49.block_sparse_moe.shared_experts.gate_proj.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.49.block_sparse_moe.shared_experts.up_proj.scales": "model-00048-of-00058.safetensors", + "language_model.model.layers.49.block_sparse_moe.shared_experts.up_proj.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.49.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00048-of-00058.safetensors", + "language_model.model.layers.49.block_sparse_moe.switch_mlp.down_proj.scales": "model-00048-of-00058.safetensors", + "language_model.model.layers.49.block_sparse_moe.switch_mlp.down_proj.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.49.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00048-of-00058.safetensors", + "language_model.model.layers.49.block_sparse_moe.switch_mlp.up_proj.scales": "model-00048-of-00058.safetensors", + "language_model.model.layers.49.block_sparse_moe.switch_mlp.up_proj.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.49.input_layernorm.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.49.post_attention_layernorm.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.49.self_attn.index_k_norm.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.self_attn.index_k_proj.scales": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.self_attn.index_k_proj.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.self_attn.index_q_norm.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.self_attn.index_q_proj.scales": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.self_attn.index_q_proj.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.self_attn.k_norm.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.self_attn.k_proj.scales": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.self_attn.k_proj.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.self_attn.o_proj.scales": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.self_attn.o_proj.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.self_attn.q_norm.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.self_attn.q_proj.scales": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.self_attn.q_proj.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.self_attn.v_proj.scales": "model-00047-of-00058.safetensors", + "language_model.model.layers.49.self_attn.v_proj.weight": "model-00047-of-00058.safetensors", + "language_model.model.layers.5.block_sparse_moe.e_score_correction_bias": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.block_sparse_moe.gate.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.block_sparse_moe.shared_experts.down_proj.scales": "model-00004-of-00058.safetensors", + "language_model.model.layers.5.block_sparse_moe.shared_experts.down_proj.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.5.block_sparse_moe.shared_experts.gate_proj.scales": "model-00004-of-00058.safetensors", + "language_model.model.layers.5.block_sparse_moe.shared_experts.gate_proj.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.5.block_sparse_moe.shared_experts.up_proj.scales": "model-00004-of-00058.safetensors", + "language_model.model.layers.5.block_sparse_moe.shared_experts.up_proj.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.5.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00004-of-00058.safetensors", + "language_model.model.layers.5.block_sparse_moe.switch_mlp.down_proj.scales": "model-00004-of-00058.safetensors", + "language_model.model.layers.5.block_sparse_moe.switch_mlp.down_proj.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.5.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00004-of-00058.safetensors", + "language_model.model.layers.5.block_sparse_moe.switch_mlp.up_proj.scales": "model-00004-of-00058.safetensors", + "language_model.model.layers.5.block_sparse_moe.switch_mlp.up_proj.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.5.input_layernorm.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.5.post_attention_layernorm.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.5.self_attn.index_k_norm.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.self_attn.index_k_proj.scales": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.self_attn.index_k_proj.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.self_attn.index_q_norm.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.self_attn.index_q_proj.scales": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.self_attn.index_q_proj.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.self_attn.k_norm.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.self_attn.k_proj.scales": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.self_attn.k_proj.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.self_attn.o_proj.scales": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.self_attn.o_proj.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.self_attn.q_norm.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.self_attn.q_proj.scales": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.self_attn.q_proj.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.self_attn.v_proj.scales": "model-00003-of-00058.safetensors", + "language_model.model.layers.5.self_attn.v_proj.weight": "model-00003-of-00058.safetensors", + "language_model.model.layers.50.block_sparse_moe.e_score_correction_bias": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.block_sparse_moe.gate.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.block_sparse_moe.shared_experts.down_proj.scales": "model-00049-of-00058.safetensors", + "language_model.model.layers.50.block_sparse_moe.shared_experts.down_proj.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.50.block_sparse_moe.shared_experts.gate_proj.scales": "model-00049-of-00058.safetensors", + "language_model.model.layers.50.block_sparse_moe.shared_experts.gate_proj.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.50.block_sparse_moe.shared_experts.up_proj.scales": "model-00049-of-00058.safetensors", + "language_model.model.layers.50.block_sparse_moe.shared_experts.up_proj.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.50.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00049-of-00058.safetensors", + "language_model.model.layers.50.block_sparse_moe.switch_mlp.down_proj.scales": "model-00049-of-00058.safetensors", + "language_model.model.layers.50.block_sparse_moe.switch_mlp.down_proj.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.50.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00049-of-00058.safetensors", + "language_model.model.layers.50.block_sparse_moe.switch_mlp.up_proj.scales": "model-00049-of-00058.safetensors", + "language_model.model.layers.50.block_sparse_moe.switch_mlp.up_proj.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.50.input_layernorm.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.50.post_attention_layernorm.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.50.self_attn.index_k_norm.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.self_attn.index_k_proj.scales": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.self_attn.index_k_proj.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.self_attn.index_q_norm.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.self_attn.index_q_proj.scales": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.self_attn.index_q_proj.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.self_attn.k_norm.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.self_attn.k_proj.scales": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.self_attn.k_proj.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.self_attn.o_proj.scales": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.self_attn.o_proj.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.self_attn.q_norm.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.self_attn.q_proj.scales": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.self_attn.q_proj.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.self_attn.v_proj.scales": "model-00048-of-00058.safetensors", + "language_model.model.layers.50.self_attn.v_proj.weight": "model-00048-of-00058.safetensors", + "language_model.model.layers.51.block_sparse_moe.e_score_correction_bias": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.block_sparse_moe.gate.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.block_sparse_moe.shared_experts.down_proj.scales": "model-00050-of-00058.safetensors", + "language_model.model.layers.51.block_sparse_moe.shared_experts.down_proj.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.51.block_sparse_moe.shared_experts.gate_proj.scales": "model-00050-of-00058.safetensors", + "language_model.model.layers.51.block_sparse_moe.shared_experts.gate_proj.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.51.block_sparse_moe.shared_experts.up_proj.scales": "model-00050-of-00058.safetensors", + "language_model.model.layers.51.block_sparse_moe.shared_experts.up_proj.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.51.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00050-of-00058.safetensors", + "language_model.model.layers.51.block_sparse_moe.switch_mlp.down_proj.scales": "model-00050-of-00058.safetensors", + "language_model.model.layers.51.block_sparse_moe.switch_mlp.down_proj.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.51.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00050-of-00058.safetensors", + "language_model.model.layers.51.block_sparse_moe.switch_mlp.up_proj.scales": "model-00050-of-00058.safetensors", + "language_model.model.layers.51.block_sparse_moe.switch_mlp.up_proj.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.51.input_layernorm.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.51.post_attention_layernorm.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.51.self_attn.index_k_norm.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.self_attn.index_k_proj.scales": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.self_attn.index_k_proj.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.self_attn.index_q_norm.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.self_attn.index_q_proj.scales": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.self_attn.index_q_proj.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.self_attn.k_norm.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.self_attn.k_proj.scales": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.self_attn.k_proj.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.self_attn.o_proj.scales": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.self_attn.o_proj.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.self_attn.q_norm.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.self_attn.q_proj.scales": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.self_attn.q_proj.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.self_attn.v_proj.scales": "model-00049-of-00058.safetensors", + "language_model.model.layers.51.self_attn.v_proj.weight": "model-00049-of-00058.safetensors", + "language_model.model.layers.52.block_sparse_moe.e_score_correction_bias": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.block_sparse_moe.gate.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.block_sparse_moe.shared_experts.down_proj.scales": "model-00051-of-00058.safetensors", + "language_model.model.layers.52.block_sparse_moe.shared_experts.down_proj.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.52.block_sparse_moe.shared_experts.gate_proj.scales": "model-00051-of-00058.safetensors", + "language_model.model.layers.52.block_sparse_moe.shared_experts.gate_proj.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.52.block_sparse_moe.shared_experts.up_proj.scales": "model-00051-of-00058.safetensors", + "language_model.model.layers.52.block_sparse_moe.shared_experts.up_proj.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.52.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00051-of-00058.safetensors", + "language_model.model.layers.52.block_sparse_moe.switch_mlp.down_proj.scales": "model-00051-of-00058.safetensors", + "language_model.model.layers.52.block_sparse_moe.switch_mlp.down_proj.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.52.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00051-of-00058.safetensors", + "language_model.model.layers.52.block_sparse_moe.switch_mlp.up_proj.scales": "model-00051-of-00058.safetensors", + "language_model.model.layers.52.block_sparse_moe.switch_mlp.up_proj.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.52.input_layernorm.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.52.post_attention_layernorm.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.52.self_attn.index_k_norm.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.self_attn.index_k_proj.scales": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.self_attn.index_k_proj.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.self_attn.index_q_norm.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.self_attn.index_q_proj.scales": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.self_attn.index_q_proj.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.self_attn.k_norm.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.self_attn.k_proj.scales": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.self_attn.k_proj.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.self_attn.o_proj.scales": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.self_attn.o_proj.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.self_attn.q_norm.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.self_attn.q_proj.scales": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.self_attn.q_proj.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.self_attn.v_proj.scales": "model-00050-of-00058.safetensors", + "language_model.model.layers.52.self_attn.v_proj.weight": "model-00050-of-00058.safetensors", + "language_model.model.layers.53.block_sparse_moe.e_score_correction_bias": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.block_sparse_moe.gate.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.block_sparse_moe.shared_experts.down_proj.scales": "model-00052-of-00058.safetensors", + "language_model.model.layers.53.block_sparse_moe.shared_experts.down_proj.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.53.block_sparse_moe.shared_experts.gate_proj.scales": "model-00052-of-00058.safetensors", + "language_model.model.layers.53.block_sparse_moe.shared_experts.gate_proj.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.53.block_sparse_moe.shared_experts.up_proj.scales": "model-00052-of-00058.safetensors", + "language_model.model.layers.53.block_sparse_moe.shared_experts.up_proj.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.53.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00052-of-00058.safetensors", + "language_model.model.layers.53.block_sparse_moe.switch_mlp.down_proj.scales": "model-00052-of-00058.safetensors", + "language_model.model.layers.53.block_sparse_moe.switch_mlp.down_proj.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.53.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00052-of-00058.safetensors", + "language_model.model.layers.53.block_sparse_moe.switch_mlp.up_proj.scales": "model-00052-of-00058.safetensors", + "language_model.model.layers.53.block_sparse_moe.switch_mlp.up_proj.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.53.input_layernorm.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.53.post_attention_layernorm.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.53.self_attn.index_k_norm.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.self_attn.index_k_proj.scales": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.self_attn.index_k_proj.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.self_attn.index_q_norm.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.self_attn.index_q_proj.scales": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.self_attn.index_q_proj.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.self_attn.k_norm.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.self_attn.k_proj.scales": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.self_attn.k_proj.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.self_attn.o_proj.scales": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.self_attn.o_proj.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.self_attn.q_norm.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.self_attn.q_proj.scales": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.self_attn.q_proj.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.self_attn.v_proj.scales": "model-00051-of-00058.safetensors", + "language_model.model.layers.53.self_attn.v_proj.weight": "model-00051-of-00058.safetensors", + "language_model.model.layers.54.block_sparse_moe.e_score_correction_bias": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.block_sparse_moe.gate.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.block_sparse_moe.shared_experts.down_proj.scales": "model-00053-of-00058.safetensors", + "language_model.model.layers.54.block_sparse_moe.shared_experts.down_proj.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.54.block_sparse_moe.shared_experts.gate_proj.scales": "model-00053-of-00058.safetensors", + "language_model.model.layers.54.block_sparse_moe.shared_experts.gate_proj.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.54.block_sparse_moe.shared_experts.up_proj.scales": "model-00053-of-00058.safetensors", + "language_model.model.layers.54.block_sparse_moe.shared_experts.up_proj.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.54.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00053-of-00058.safetensors", + "language_model.model.layers.54.block_sparse_moe.switch_mlp.down_proj.scales": "model-00053-of-00058.safetensors", + "language_model.model.layers.54.block_sparse_moe.switch_mlp.down_proj.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.54.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00053-of-00058.safetensors", + "language_model.model.layers.54.block_sparse_moe.switch_mlp.up_proj.scales": "model-00053-of-00058.safetensors", + "language_model.model.layers.54.block_sparse_moe.switch_mlp.up_proj.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.54.input_layernorm.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.54.post_attention_layernorm.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.54.self_attn.index_k_norm.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.self_attn.index_k_proj.scales": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.self_attn.index_k_proj.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.self_attn.index_q_norm.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.self_attn.index_q_proj.scales": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.self_attn.index_q_proj.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.self_attn.k_norm.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.self_attn.k_proj.scales": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.self_attn.k_proj.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.self_attn.o_proj.scales": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.self_attn.o_proj.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.self_attn.q_norm.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.self_attn.q_proj.scales": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.self_attn.q_proj.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.self_attn.v_proj.scales": "model-00052-of-00058.safetensors", + "language_model.model.layers.54.self_attn.v_proj.weight": "model-00052-of-00058.safetensors", + "language_model.model.layers.55.block_sparse_moe.e_score_correction_bias": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.block_sparse_moe.gate.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.block_sparse_moe.shared_experts.down_proj.scales": "model-00054-of-00058.safetensors", + "language_model.model.layers.55.block_sparse_moe.shared_experts.down_proj.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.55.block_sparse_moe.shared_experts.gate_proj.scales": "model-00054-of-00058.safetensors", + "language_model.model.layers.55.block_sparse_moe.shared_experts.gate_proj.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.55.block_sparse_moe.shared_experts.up_proj.scales": "model-00054-of-00058.safetensors", + "language_model.model.layers.55.block_sparse_moe.shared_experts.up_proj.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.55.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00054-of-00058.safetensors", + "language_model.model.layers.55.block_sparse_moe.switch_mlp.down_proj.scales": "model-00054-of-00058.safetensors", + "language_model.model.layers.55.block_sparse_moe.switch_mlp.down_proj.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.55.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00054-of-00058.safetensors", + "language_model.model.layers.55.block_sparse_moe.switch_mlp.up_proj.scales": "model-00054-of-00058.safetensors", + "language_model.model.layers.55.block_sparse_moe.switch_mlp.up_proj.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.55.input_layernorm.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.55.post_attention_layernorm.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.55.self_attn.index_k_norm.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.self_attn.index_k_proj.scales": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.self_attn.index_k_proj.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.self_attn.index_q_norm.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.self_attn.index_q_proj.scales": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.self_attn.index_q_proj.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.self_attn.k_norm.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.self_attn.k_proj.scales": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.self_attn.k_proj.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.self_attn.o_proj.scales": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.self_attn.o_proj.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.self_attn.q_norm.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.self_attn.q_proj.scales": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.self_attn.q_proj.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.self_attn.v_proj.scales": "model-00053-of-00058.safetensors", + "language_model.model.layers.55.self_attn.v_proj.weight": "model-00053-of-00058.safetensors", + "language_model.model.layers.56.block_sparse_moe.e_score_correction_bias": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.block_sparse_moe.gate.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.block_sparse_moe.shared_experts.down_proj.scales": "model-00055-of-00058.safetensors", + "language_model.model.layers.56.block_sparse_moe.shared_experts.down_proj.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.56.block_sparse_moe.shared_experts.gate_proj.scales": "model-00055-of-00058.safetensors", + "language_model.model.layers.56.block_sparse_moe.shared_experts.gate_proj.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.56.block_sparse_moe.shared_experts.up_proj.scales": "model-00055-of-00058.safetensors", + "language_model.model.layers.56.block_sparse_moe.shared_experts.up_proj.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.56.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00055-of-00058.safetensors", + "language_model.model.layers.56.block_sparse_moe.switch_mlp.down_proj.scales": "model-00055-of-00058.safetensors", + "language_model.model.layers.56.block_sparse_moe.switch_mlp.down_proj.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.56.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00055-of-00058.safetensors", + "language_model.model.layers.56.block_sparse_moe.switch_mlp.up_proj.scales": "model-00055-of-00058.safetensors", + "language_model.model.layers.56.block_sparse_moe.switch_mlp.up_proj.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.56.input_layernorm.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.56.post_attention_layernorm.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.56.self_attn.index_k_norm.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.self_attn.index_k_proj.scales": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.self_attn.index_k_proj.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.self_attn.index_q_norm.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.self_attn.index_q_proj.scales": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.self_attn.index_q_proj.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.self_attn.k_norm.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.self_attn.k_proj.scales": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.self_attn.k_proj.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.self_attn.o_proj.scales": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.self_attn.o_proj.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.self_attn.q_norm.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.self_attn.q_proj.scales": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.self_attn.q_proj.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.self_attn.v_proj.scales": "model-00054-of-00058.safetensors", + "language_model.model.layers.56.self_attn.v_proj.weight": "model-00054-of-00058.safetensors", + "language_model.model.layers.57.block_sparse_moe.e_score_correction_bias": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.block_sparse_moe.gate.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.block_sparse_moe.shared_experts.down_proj.scales": "model-00056-of-00058.safetensors", + "language_model.model.layers.57.block_sparse_moe.shared_experts.down_proj.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.57.block_sparse_moe.shared_experts.gate_proj.scales": "model-00056-of-00058.safetensors", + "language_model.model.layers.57.block_sparse_moe.shared_experts.gate_proj.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.57.block_sparse_moe.shared_experts.up_proj.scales": "model-00056-of-00058.safetensors", + "language_model.model.layers.57.block_sparse_moe.shared_experts.up_proj.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.57.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00056-of-00058.safetensors", + "language_model.model.layers.57.block_sparse_moe.switch_mlp.down_proj.scales": "model-00056-of-00058.safetensors", + "language_model.model.layers.57.block_sparse_moe.switch_mlp.down_proj.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.57.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00056-of-00058.safetensors", + "language_model.model.layers.57.block_sparse_moe.switch_mlp.up_proj.scales": "model-00056-of-00058.safetensors", + "language_model.model.layers.57.block_sparse_moe.switch_mlp.up_proj.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.57.input_layernorm.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.57.post_attention_layernorm.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.57.self_attn.index_k_norm.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.self_attn.index_k_proj.scales": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.self_attn.index_k_proj.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.self_attn.index_q_norm.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.self_attn.index_q_proj.scales": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.self_attn.index_q_proj.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.self_attn.k_norm.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.self_attn.k_proj.scales": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.self_attn.k_proj.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.self_attn.o_proj.scales": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.self_attn.o_proj.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.self_attn.q_norm.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.self_attn.q_proj.scales": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.self_attn.q_proj.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.self_attn.v_proj.scales": "model-00055-of-00058.safetensors", + "language_model.model.layers.57.self_attn.v_proj.weight": "model-00055-of-00058.safetensors", + "language_model.model.layers.58.block_sparse_moe.e_score_correction_bias": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.block_sparse_moe.gate.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.block_sparse_moe.shared_experts.down_proj.scales": "model-00057-of-00058.safetensors", + "language_model.model.layers.58.block_sparse_moe.shared_experts.down_proj.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.58.block_sparse_moe.shared_experts.gate_proj.scales": "model-00057-of-00058.safetensors", + "language_model.model.layers.58.block_sparse_moe.shared_experts.gate_proj.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.58.block_sparse_moe.shared_experts.up_proj.scales": "model-00057-of-00058.safetensors", + "language_model.model.layers.58.block_sparse_moe.shared_experts.up_proj.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.58.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00057-of-00058.safetensors", + "language_model.model.layers.58.block_sparse_moe.switch_mlp.down_proj.scales": "model-00057-of-00058.safetensors", + "language_model.model.layers.58.block_sparse_moe.switch_mlp.down_proj.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.58.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00057-of-00058.safetensors", + "language_model.model.layers.58.block_sparse_moe.switch_mlp.up_proj.scales": "model-00057-of-00058.safetensors", + "language_model.model.layers.58.block_sparse_moe.switch_mlp.up_proj.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.58.input_layernorm.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.58.post_attention_layernorm.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.58.self_attn.index_k_norm.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.self_attn.index_k_proj.scales": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.self_attn.index_k_proj.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.self_attn.index_q_norm.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.self_attn.index_q_proj.scales": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.self_attn.index_q_proj.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.self_attn.k_norm.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.self_attn.k_proj.scales": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.self_attn.k_proj.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.self_attn.o_proj.scales": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.self_attn.o_proj.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.self_attn.q_norm.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.self_attn.q_proj.scales": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.self_attn.q_proj.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.self_attn.v_proj.scales": "model-00056-of-00058.safetensors", + "language_model.model.layers.58.self_attn.v_proj.weight": "model-00056-of-00058.safetensors", + "language_model.model.layers.59.block_sparse_moe.e_score_correction_bias": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.block_sparse_moe.gate.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.block_sparse_moe.shared_experts.down_proj.scales": "model-00058-of-00058.safetensors", + "language_model.model.layers.59.block_sparse_moe.shared_experts.down_proj.weight": "model-00058-of-00058.safetensors", + "language_model.model.layers.59.block_sparse_moe.shared_experts.gate_proj.scales": "model-00058-of-00058.safetensors", + "language_model.model.layers.59.block_sparse_moe.shared_experts.gate_proj.weight": "model-00058-of-00058.safetensors", + "language_model.model.layers.59.block_sparse_moe.shared_experts.up_proj.scales": "model-00058-of-00058.safetensors", + "language_model.model.layers.59.block_sparse_moe.shared_experts.up_proj.weight": "model-00058-of-00058.safetensors", + "language_model.model.layers.59.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00058-of-00058.safetensors", + "language_model.model.layers.59.block_sparse_moe.switch_mlp.down_proj.scales": "model-00058-of-00058.safetensors", + "language_model.model.layers.59.block_sparse_moe.switch_mlp.down_proj.weight": "model-00058-of-00058.safetensors", + "language_model.model.layers.59.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00058-of-00058.safetensors", + "language_model.model.layers.59.block_sparse_moe.switch_mlp.up_proj.scales": "model-00058-of-00058.safetensors", + "language_model.model.layers.59.block_sparse_moe.switch_mlp.up_proj.weight": "model-00058-of-00058.safetensors", + "language_model.model.layers.59.input_layernorm.weight": "model-00058-of-00058.safetensors", + "language_model.model.layers.59.post_attention_layernorm.weight": "model-00058-of-00058.safetensors", + "language_model.model.layers.59.self_attn.index_k_norm.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.self_attn.index_k_proj.scales": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.self_attn.index_k_proj.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.self_attn.index_q_norm.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.self_attn.index_q_proj.scales": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.self_attn.index_q_proj.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.self_attn.k_norm.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.self_attn.k_proj.scales": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.self_attn.k_proj.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.self_attn.o_proj.scales": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.self_attn.o_proj.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.self_attn.q_norm.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.self_attn.q_proj.scales": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.self_attn.q_proj.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.self_attn.v_proj.scales": "model-00057-of-00058.safetensors", + "language_model.model.layers.59.self_attn.v_proj.weight": "model-00057-of-00058.safetensors", + "language_model.model.layers.6.block_sparse_moe.e_score_correction_bias": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.block_sparse_moe.gate.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.block_sparse_moe.shared_experts.down_proj.scales": "model-00005-of-00058.safetensors", + "language_model.model.layers.6.block_sparse_moe.shared_experts.down_proj.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.6.block_sparse_moe.shared_experts.gate_proj.scales": "model-00005-of-00058.safetensors", + "language_model.model.layers.6.block_sparse_moe.shared_experts.gate_proj.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.6.block_sparse_moe.shared_experts.up_proj.scales": "model-00005-of-00058.safetensors", + "language_model.model.layers.6.block_sparse_moe.shared_experts.up_proj.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.6.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00005-of-00058.safetensors", + "language_model.model.layers.6.block_sparse_moe.switch_mlp.down_proj.scales": "model-00005-of-00058.safetensors", + "language_model.model.layers.6.block_sparse_moe.switch_mlp.down_proj.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.6.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00005-of-00058.safetensors", + "language_model.model.layers.6.block_sparse_moe.switch_mlp.up_proj.scales": "model-00005-of-00058.safetensors", + "language_model.model.layers.6.block_sparse_moe.switch_mlp.up_proj.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.6.input_layernorm.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.6.post_attention_layernorm.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.6.self_attn.index_k_norm.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.self_attn.index_k_proj.scales": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.self_attn.index_k_proj.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.self_attn.index_q_norm.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.self_attn.index_q_proj.scales": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.self_attn.index_q_proj.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.self_attn.k_norm.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.self_attn.k_proj.scales": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.self_attn.k_proj.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.self_attn.o_proj.scales": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.self_attn.o_proj.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.self_attn.q_norm.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.self_attn.q_proj.scales": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.self_attn.q_proj.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.self_attn.v_proj.scales": "model-00004-of-00058.safetensors", + "language_model.model.layers.6.self_attn.v_proj.weight": "model-00004-of-00058.safetensors", + "language_model.model.layers.7.block_sparse_moe.e_score_correction_bias": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.block_sparse_moe.gate.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.block_sparse_moe.shared_experts.down_proj.scales": "model-00006-of-00058.safetensors", + "language_model.model.layers.7.block_sparse_moe.shared_experts.down_proj.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.7.block_sparse_moe.shared_experts.gate_proj.scales": "model-00006-of-00058.safetensors", + "language_model.model.layers.7.block_sparse_moe.shared_experts.gate_proj.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.7.block_sparse_moe.shared_experts.up_proj.scales": "model-00006-of-00058.safetensors", + "language_model.model.layers.7.block_sparse_moe.shared_experts.up_proj.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.7.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00006-of-00058.safetensors", + "language_model.model.layers.7.block_sparse_moe.switch_mlp.down_proj.scales": "model-00006-of-00058.safetensors", + "language_model.model.layers.7.block_sparse_moe.switch_mlp.down_proj.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.7.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00006-of-00058.safetensors", + "language_model.model.layers.7.block_sparse_moe.switch_mlp.up_proj.scales": "model-00006-of-00058.safetensors", + "language_model.model.layers.7.block_sparse_moe.switch_mlp.up_proj.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.7.input_layernorm.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.7.post_attention_layernorm.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.7.self_attn.index_k_norm.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.self_attn.index_k_proj.scales": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.self_attn.index_k_proj.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.self_attn.index_q_norm.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.self_attn.index_q_proj.scales": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.self_attn.index_q_proj.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.self_attn.k_norm.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.self_attn.k_proj.scales": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.self_attn.k_proj.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.self_attn.o_proj.scales": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.self_attn.o_proj.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.self_attn.q_norm.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.self_attn.q_proj.scales": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.self_attn.q_proj.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.self_attn.v_proj.scales": "model-00005-of-00058.safetensors", + "language_model.model.layers.7.self_attn.v_proj.weight": "model-00005-of-00058.safetensors", + "language_model.model.layers.8.block_sparse_moe.e_score_correction_bias": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.block_sparse_moe.gate.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.block_sparse_moe.shared_experts.down_proj.scales": "model-00007-of-00058.safetensors", + "language_model.model.layers.8.block_sparse_moe.shared_experts.down_proj.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.8.block_sparse_moe.shared_experts.gate_proj.scales": "model-00007-of-00058.safetensors", + "language_model.model.layers.8.block_sparse_moe.shared_experts.gate_proj.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.8.block_sparse_moe.shared_experts.up_proj.scales": "model-00007-of-00058.safetensors", + "language_model.model.layers.8.block_sparse_moe.shared_experts.up_proj.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.8.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00007-of-00058.safetensors", + "language_model.model.layers.8.block_sparse_moe.switch_mlp.down_proj.scales": "model-00007-of-00058.safetensors", + "language_model.model.layers.8.block_sparse_moe.switch_mlp.down_proj.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.8.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00007-of-00058.safetensors", + "language_model.model.layers.8.block_sparse_moe.switch_mlp.up_proj.scales": "model-00007-of-00058.safetensors", + "language_model.model.layers.8.block_sparse_moe.switch_mlp.up_proj.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.8.input_layernorm.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.8.post_attention_layernorm.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.8.self_attn.index_k_norm.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.self_attn.index_k_proj.scales": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.self_attn.index_k_proj.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.self_attn.index_q_norm.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.self_attn.index_q_proj.scales": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.self_attn.index_q_proj.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.self_attn.k_norm.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.self_attn.k_proj.scales": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.self_attn.k_proj.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.self_attn.o_proj.scales": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.self_attn.o_proj.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.self_attn.q_norm.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.self_attn.q_proj.scales": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.self_attn.q_proj.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.self_attn.v_proj.scales": "model-00006-of-00058.safetensors", + "language_model.model.layers.8.self_attn.v_proj.weight": "model-00006-of-00058.safetensors", + "language_model.model.layers.9.block_sparse_moe.e_score_correction_bias": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.block_sparse_moe.gate.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.block_sparse_moe.shared_experts.down_proj.scales": "model-00008-of-00058.safetensors", + "language_model.model.layers.9.block_sparse_moe.shared_experts.down_proj.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.9.block_sparse_moe.shared_experts.gate_proj.scales": "model-00008-of-00058.safetensors", + "language_model.model.layers.9.block_sparse_moe.shared_experts.gate_proj.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.9.block_sparse_moe.shared_experts.up_proj.scales": "model-00008-of-00058.safetensors", + "language_model.model.layers.9.block_sparse_moe.shared_experts.up_proj.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.9.block_sparse_moe.switch_mlp.down_proj.global_scale": "model-00008-of-00058.safetensors", + "language_model.model.layers.9.block_sparse_moe.switch_mlp.down_proj.scales": "model-00008-of-00058.safetensors", + "language_model.model.layers.9.block_sparse_moe.switch_mlp.down_proj.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.9.block_sparse_moe.switch_mlp.gate_proj.global_scale": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.block_sparse_moe.switch_mlp.gate_proj.scales": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.block_sparse_moe.switch_mlp.gate_proj.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.block_sparse_moe.switch_mlp.up_proj.global_scale": "model-00008-of-00058.safetensors", + "language_model.model.layers.9.block_sparse_moe.switch_mlp.up_proj.scales": "model-00008-of-00058.safetensors", + "language_model.model.layers.9.block_sparse_moe.switch_mlp.up_proj.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.9.input_layernorm.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.9.post_attention_layernorm.weight": "model-00008-of-00058.safetensors", + "language_model.model.layers.9.self_attn.index_k_norm.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.self_attn.index_k_proj.scales": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.self_attn.index_k_proj.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.self_attn.index_q_norm.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.self_attn.index_q_proj.scales": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.self_attn.index_q_proj.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.self_attn.k_norm.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.self_attn.k_proj.scales": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.self_attn.k_proj.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.self_attn.o_proj.scales": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.self_attn.o_proj.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.self_attn.q_norm.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.self_attn.q_proj.scales": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.self_attn.q_proj.weight": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.self_attn.v_proj.scales": "model-00007-of-00058.safetensors", + "language_model.model.layers.9.self_attn.v_proj.weight": "model-00007-of-00058.safetensors", + "language_model.model.norm.weight": "model-00058-of-00058.safetensors" + } +} \ No newline at end of file diff --git a/processing_minimax.py b/processing_minimax.py new file mode 100644 index 0000000000000000000000000000000000000000..c08cc624a63dfeced8d31686daeed89bf9ede2cc --- /dev/null +++ b/processing_minimax.py @@ -0,0 +1,254 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +""" +MiniMax VL family HuggingFace-compatible Processor, ImageProcessor, VideoProcessor. +""" + +import math +import re +from typing import List, Optional, Tuple, Union + +import torch +import torchvision +from torchvision.transforms import InterpolationMode +from transformers import BatchFeature +from transformers.image_processing_utils_fast import ( + BaseImageProcessorFast, + group_images_by_shape, + reorder_images, +) +from transformers.image_utils import PILImageResampling, SizeDict +from transformers.processing_utils import ( + ImagesKwargs, + ProcessingKwargs, + ProcessorMixin, + Unpack, + VideosKwargs, +) +from transformers.utils import TensorType +from transformers.video_processing_utils import BaseVideoProcessor +from transformers.video_utils import group_videos_by_shape, reorder_videos + + +class MiniMaxVLProcessorKwargs(ProcessingKwargs, total=False): + _defaults = { + "videos_kwargs": { + "do_resize": False, + "return_metadata": True, + }, + } + + +class MiniMaxVLProcessor(ProcessorMixin): + IMAGE_TOKEN = "]<]image[>[" + VIDEO_TOKEN = "]<]video[>[" + VISION_START_TOKEN = "]<]start of image[>[" + VISION_END_TOKEN = "]<]end of image[>[" + + def __init__( + self, image_processor=None, tokenizer=None, video_processor=None, **kwargs + ): + self.image_token_id = tokenizer.convert_tokens_to_ids(self.IMAGE_TOKEN) + self.video_token_id = tokenizer.convert_tokens_to_ids(self.VIDEO_TOKEN) + super().__init__(image_processor, tokenizer, video_processor) + # Video expansion also uses image start/end tokens. Separate video + # start/end tokens exist in the tokenizer, but the original MiniMax + # serving path did not use them; keep that behavior for compatibility. + self.vision_start_token_id = tokenizer.convert_tokens_to_ids( + self.VISION_START_TOKEN + ) + self.vision_end_token_id = tokenizer.convert_tokens_to_ids( + self.VISION_END_TOKEN + ) + + def _prune_video_tokens( + self, + input_text: str, + video_segments: List[int], + video_token: str, + ) -> str: + """ + Prune video tokens by temporal_patch_size (e.g., 2:1). + + Expects the prompt to carry exactly sum(video_segments) video + tokens — i.e. one token per *sampled* frame. Then drops token. + + Args: + input_text: prompt with N video_tokens per segment + video_segments: actual sampled frame count per video segment + video_token: the video token string, e.g. ']<]video[>[' + + Returns: + Pruned input_text with ~N/temporal_patch_size tokens per segment. + """ + # If no videos or temporal_patch_size <= 1, no pruning needed + if not video_segments or self.video_processor.temporal_patch_size <= 1: + return input_text + + # Split while keeping delimiters + special_tokens = [video_token] # , image_token] + pattern = "|".join(map(re.escape, special_tokens)) + parts = re.split(f"({pattern})", input_text) + + def is_timestamp(text: str) -> bool: + """Check if text ends with timestamp format like ']<]0.0 seconds[>['""" + return ( + text.endswith("seconds[>[") + or text.endswith("seconds[>[ ") + or text.endswith("seconds [>[") + or text.endswith("seconds [>[ ") + ) + + def extract_timestamp(text: str) -> str: + """Extract timestamp text from the end, starting from ']<]'""" + start_index = text.rfind("]<]") + if start_index == -1: + raise ValueError(f"Failed to extract timestamp: {text}") + return text[start_index:] + + # Build new text with pruned video tokens + final_parts = [] + current_seg_idx = 0 # Which video segment we're in + frame_in_seg = 0 # Frame index within current segment + last_timestamp_len = 0 # Length of timestamp to potentially remove + + for part in parts: + if part == video_token: + if current_seg_idx < len(video_segments): + if frame_in_seg % self.video_processor.temporal_patch_size == 0: + # Keep this video token + final_parts.append(part) + frame_in_seg += 1 + if frame_in_seg >= video_segments[current_seg_idx]: + current_seg_idx += 1 + frame_in_seg = 0 + last_timestamp_len = 0 + else: + # Skip this video token + frame_in_seg += 1 + if frame_in_seg >= video_segments[current_seg_idx]: + current_seg_idx += 1 + frame_in_seg = 0 + # Remove the timestamp that was already appended + if last_timestamp_len > 0: + # Truncate the last part to remove timestamp + assert len(final_parts) > 0 + final_parts[-1] = final_parts[-1][:-last_timestamp_len] + last_timestamp_len = 0 + else: + # No more video segments, keep as is + final_parts.append(part) + last_timestamp_len = 0 + else: + # Text part + final_parts.append(part) + # Check if this text ends with a timestamp + if is_timestamp(part): + last_timestamp_len = len(extract_timestamp(part)) + else: + last_timestamp_len = 0 + + return "".join(final_parts) + + def __call__( + self, + images=None, + text=None, + videos=None, + **kwargs: Unpack[MiniMaxVLProcessorKwargs], + ) -> BatchFeature: + output_kwargs = self._merge_kwargs( + MiniMaxVLProcessorKwargs, + tokenizer_init_kwargs=self.tokenizer.init_kwargs, + **kwargs, + ) + + if images is not None: + images_kwargs = output_kwargs["images_kwargs"] + image_inputs = self.image_processor(images=images, **images_kwargs) + image_grid_thw = image_inputs["image_grid_thw"] + + else: + image_inputs = {} + image_grid_thw = None + + if videos is not None: + videos_kwargs = output_kwargs["videos_kwargs"] + video_inputs = self.video_processor(videos=videos, **videos_kwargs) + video_grid_thw = video_inputs["video_grid_thw"] + if not kwargs.get("return_metadata"): + video_metadata = video_inputs.pop("video_metadata") + else: + video_metadata = video_inputs["video_metadata"] + else: + video_inputs = {} + video_grid_thw = None + + if not isinstance(text, list): + text = [text] + text = text.copy() + + # Expand image tokens + if image_grid_thw is not None: + merge_length = self.image_processor.merge_size**2 + placeholder = "]<]placeholder[>[" + index = 0 + for i in range(len(text)): + while self.IMAGE_TOKEN in text[i]: + num_tokens = image_grid_thw[index].prod() // merge_length + text[i] = text[i].replace( + self.IMAGE_TOKEN, + self.VISION_START_TOKEN + + placeholder * num_tokens + + self.VISION_END_TOKEN, + 1, + ) + index += 1 + text[i] = text[i].replace(placeholder, self.IMAGE_TOKEN) + + # Expand video tokens + if video_grid_thw is not None: + merge_length = self.image_processor.merge_size**2 + placeholder = "]<]placeholder[>[" + index = 0 + for i in range(len(text)): + while self.VIDEO_TOKEN in text[i]: + metadata = video_metadata[index] + grid_t = video_grid_thw[index][0] + frame_seqlen = video_grid_thw[index][1:].prod() // merge_length + + video_placeholder = "" + for frame_idx in range(grid_t): + if ( + metadata.fps is not None + and metadata.frames_indices is not None + ): + ts = ( + metadata.frames_indices[ + min( + frame_idx + * self.video_processor.temporal_patch_size, + len(metadata.frames_indices) - 1, + ) + ] + / metadata.fps + ) + video_placeholder += f"]<]{ts:.1f} seconds[>[" + video_placeholder += ( + self.VISION_START_TOKEN + + placeholder * frame_seqlen + + self.VISION_END_TOKEN + ) + + text[i] = text[i].replace(self.VIDEO_TOKEN, video_placeholder, 1) + index += 1 + text[i] = text[i].replace(placeholder, self.VIDEO_TOKEN) + + # Tokenize + return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None) + text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"]) + + return BatchFeature( + data={**text_inputs, **image_inputs, **video_inputs}, + tensor_type=return_tensors, + ) diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..70254305f9baa409d26b90b168598e23198e39ab --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae73e31fefce04b005cb41c6781389426fae1a8553b6e58d29f133eaa31ebfb5 +size 15524484 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9af19456279bd9c3b3f8175777feefeb96933fbd --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,14 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": "]~b]", + "clean_up_tokenization_spaces": false, + "eos_token": "[e~[", + "is_local": true, + "local_files_only": false, + "model_max_length": 40960000, + "pad_token": "]!p~[", + "tokenizer_class": "TokenizersBackend", + "tool_parser_type": "json_tools", + "unk_token": "[e~[" +} diff --git a/video_processor.py b/video_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..d09de12560d60a64047e025c7f8397bf5609197c --- /dev/null +++ b/video_processor.py @@ -0,0 +1,208 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +""" +MiniMax VL family HuggingFace-compatible VideoProcessor. +""" + +import math +from typing import List, Optional, Tuple, Union + +import torch +import torchvision +from torchvision.transforms import InterpolationMode +from transformers import BatchFeature +from transformers.image_utils import PILImageResampling, SizeDict +from transformers.processing_utils import ( + Unpack, + VideosKwargs, +) +from transformers.utils import TensorType +from transformers.video_processing_utils import BaseVideoProcessor +from transformers.video_utils import group_videos_by_shape, reorder_videos + +MAX_RATIO = 200 + + +def round_by_factor(number: int, factor: int) -> int: + return round(number / factor) * factor + + +def ceil_by_factor(number: int, factor: int) -> int: + return math.ceil(number / factor) * factor + + +def floor_by_factor(number: int, factor: int) -> int: + return math.floor(number / factor) * factor + + +def smart_resize( + height: int, + width: int, + factor: int = 28, + min_pixels: int = 4 * 28 * 28, + max_pixels: int = 451584, +) -> tuple[int, int]: + if max(height, width) / min(height, width) > MAX_RATIO: + raise ValueError( + f"absolute aspect ratio must be smaller than {MAX_RATIO}, " + f"got {max(height, width) / min(height, width)}" + ) + h_bar = max(factor, round_by_factor(height, factor)) + w_bar = max(factor, round_by_factor(width, factor)) + if h_bar * w_bar > max_pixels: + beta = math.sqrt((height * width) / max_pixels) + h_bar = floor_by_factor(height / beta, factor) + w_bar = floor_by_factor(width / beta, factor) + elif h_bar * w_bar < min_pixels: + beta = math.sqrt(min_pixels / (height * width)) + h_bar = ceil_by_factor(height * beta, factor) + w_bar = ceil_by_factor(width * beta, factor) + return h_bar, w_bar + + +class MiniMaxM3VLVideoProcessorKwargs(VideosKwargs, total=False): + patch_size: int + temporal_patch_size: int + merge_size: int + min_pixels: int + max_pixels: int + total_pixels: int + min_frames: int + max_frames: int + fps: float | int + + +class MiniMaxM3VLVideoProcessor(BaseVideoProcessor): + do_resize = True + resample = PILImageResampling.BICUBIC + size = {"height": 672, "width": 672} + default_to_square = False + do_rescale = True + rescale_factor = 1 / 255 + do_normalize = True + image_mean = [0.48145466, 0.4578275, 0.40821073] + image_std = [0.26862954, 0.26130258, 0.27577711] + do_convert_rgb = True + do_sample_frames = False + patch_size = 14 + temporal_patch_size = 2 + merge_size = 2 + min_pixels = 4 * 28 * 28 + max_pixels = 768 * 28 * 28 # 602,112 + total_pixels = int(64000 * 28 * 28 * 0.9) # ~45M, ~64k tokens budget + fps = 1.0 + min_frames = 4 + max_frames = 768 + valid_kwargs = MiniMaxM3VLVideoProcessorKwargs + model_input_names = ["pixel_values_videos", "video_grid_thw"] + + def __init__(self, **kwargs: Unpack[MiniMaxM3VLVideoProcessorKwargs]): + super().__init__(**kwargs) + + def _preprocess( + self, + videos: List[torch.Tensor], + do_convert_rgb: bool, + do_resize: bool, + size: SizeDict, + resample: PILImageResampling | InterpolationMode | int | None, + do_rescale: bool, + rescale_factor: float, + do_normalize: bool, + image_mean: float | List[float] | None, + image_std: float | List[float] | None, + patch_size: int, + temporal_patch_size: int, + merge_size: int, + min_pixels: int, + max_pixels: int, + return_tensors: str | TensorType | None = None, + **kwargs, + ) -> BatchFeature: + grouped_videos, grouped_videos_index = group_videos_by_shape(videos) + resized_videos_grouped = {} + factor = patch_size * merge_size + for shape, stacked_videos in grouped_videos.items(): + batch_size, num_frames, channels, height, width = stacked_videos.shape + resized_height, resized_width = height, width + if do_resize: + resized_height, resized_width = smart_resize( + height, width, factor=factor, + min_pixels=min_pixels, max_pixels=max_pixels, + ) + stacked_videos = stacked_videos.view( + batch_size * num_frames, channels, height, width + ) + stacked_videos = self.resize( + stacked_videos, + size=SizeDict(height=resized_height, width=resized_width), + resample=resample, + ) + stacked_videos = stacked_videos.view( + batch_size, + num_frames, + channels, + resized_height, + resized_width, + ) + resized_videos_grouped[shape] = stacked_videos + resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index) + + grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos) + processed_videos_grouped = {} + processed_grids = {} + for shape, stacked_videos in grouped_videos.items(): + resized_height, resized_width = stacked_videos.shape[-2:] + patches = self.rescale_and_normalize( + stacked_videos, + do_rescale, + rescale_factor, + do_normalize, + image_mean, + image_std, + ) + + if pad := -patches.shape[1] % temporal_patch_size: + repeats = patches[:, -1:].expand(-1, pad, -1, -1, -1) + patches = torch.cat([patches, repeats], dim=1) + + batch_size, grid_t, channels = patches.shape[:3] + grid_t = grid_t // temporal_patch_size + grid_h, grid_w = resized_height // patch_size, resized_width // patch_size + + patches = patches.view( + batch_size, + grid_t, + temporal_patch_size, + channels, + grid_h // merge_size, + merge_size, + patch_size, + grid_w // merge_size, + merge_size, + patch_size, + ) + patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9) + flatten_patches = patches.reshape( + batch_size, + grid_t * grid_h * grid_w, + channels * temporal_patch_size * patch_size * patch_size, + ) + + processed_videos_grouped[shape] = flatten_patches + processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size + + processed_videos = reorder_videos( + processed_videos_grouped, grouped_videos_index + ) + processed_grids = reorder_videos(processed_grids, grouped_videos_index) + pixel_values_videos = torch.cat(processed_videos, dim=0) + video_grid_thw = torch.tensor(processed_grids, dtype=torch.long) + + return BatchFeature( + data={ + "pixel_values_videos": pixel_values_videos, + "video_grid_thw": video_grid_thw, + }, + tensor_type=return_tensors, + )