ArchiveStudio BigDong commited on
Commit
b277ab6
·
0 Parent(s):

Duplicate from openbmb/MiniCPM4.1-8B

Browse files

Co-authored-by: Yudong Wang <BigDong@users.noreply.huggingface.co>

.gitattributes ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,558 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ language:
4
+ - zh
5
+ - en
6
+ pipeline_tag: text-generation
7
+ library_name: transformers
8
+ ---
9
+ <div align="center">
10
+ <img src="https://github.com/OpenBMB/MiniCPM/blob/main/assets/minicpm_logo.png?raw=true" width="500em" ></img>
11
+ </div>
12
+
13
+ <p align="center">
14
+ <a href="https://github.com/OpenBMB/MiniCPM/" target="_blank">GitHub Repo</a> |
15
+ <a href="https://arxiv.org/abs/2506.07900" target="_blank">Technical Report</a> |
16
+ <a href="https://mp.weixin.qq.com/s/KIhH2nCURBXuFXAtYRpuXg?poc_token=HBIsUWijxino8oJ5s6HcjcfXFRi0Xj2LJlxPYD9c">Join Us</a>
17
+ </p>
18
+ <p align="center">
19
+ 👋 Contact us in <a href="https://discord.gg/3cGQn9b3YM" target="_blank">Discord</a> and <a href="https://github.com/OpenBMB/MiniCPM/blob/main/assets/wechat.jpg" target="_blank">WeChat</a>
20
+ </p>
21
+
22
+ ## What's New
23
+ - [2025.09.29] **[InfLLM-V2 paper](https://arxiv.org/abs/2509.24663) is released!** We can train a sparse attention model with only 5B long-text tokens. 🔥🔥🔥
24
+ - [2025.09.05] **MiniCPM4.1** series are released! This series is a hybrid reasoning model with trainable sparse attention, which can be used in both deep reasoning mode and non-reasoning mode. 🔥🔥🔥
25
+ - [2025.06.06] **MiniCPM4** series are released! This model achieves ultimate efficiency improvements while maintaining optimal performance at the same scale! It can achieve over 5x generation acceleration on typical end-side chips! You can find technical report [here](https://arxiv.org/abs/2506.07900).🔥🔥🔥
26
+
27
+ ## Highlights
28
+ MiniCPM4.1 is highlighted with following features:
29
+
30
+ ✅ Strong Reasoning Capability: Surpasses similar-sized models on 15 tasks!
31
+
32
+ ✅ Fast Generation: 3x decoding speedup for reasoning!
33
+
34
+ ✅ Efficient Architecture: Trainable sparse attention, frequency-ranked speculative decoding!
35
+
36
+ - [MiniCPM4.1-8B](https://huggingface.co/openbmb/MiniCPM4.1-8B): The latest version of MiniCPM4, with 8B parameters, support fusion thinking. (**<-- you are here**)
37
+ - [MiniCPM4.1-8B-GPTQ](https://huggingface.co/openbmb/MiniCPM4.1-8B-GPTQ): MiniCPM4.1-8B in GPTQ format.
38
+ - [MiniCPM4.1-8B-AutoAWQ](https://huggingface.co/openbmb/MiniCPM4.1-8B-AutoAWQ): MiniCPM4.1-8B in AutoAWQ format.
39
+ - [MiniCPM-4.1-8B-Marlin](https://huggingface.co/openbmb/MiniCPM-4.1-8B-Marlin): MiniCPM4.1-8B in Marlin format.
40
+ - [MiniCPM4.1-8B-GGUF](https://huggingface.co/openbmb/MiniCPM4.1-8B-GGUF): MiniCPM4.1-8B in GGUF format.
41
+ - [MiniCPM4.1-8B-MLX](https://huggingface.co/openbmb/MiniCPM4.1-8B-MLX): MiniCPM4.1-8B in MLX format.
42
+ - [MiniCPM4.1-8B-Eagle3](https://huggingface.co/openbmb/MiniCPM4.1-8B-Eagle3): Eagle3 model for MiniCPM4.1-8B.
43
+ - **MiniCPM4 Series**
44
+ <details>
45
+ <summary>Click to expand all MiniCPM4 series models</summary>
46
+
47
+ - [**MiniCPM4-8B**](https://huggingface.co/openbmb/MiniCPM4-8B): The flagship model with 8B parameters, trained on 8T tokens
48
+ - [**MiniCPM4-0.5B**](https://huggingface.co/openbmb/MiniCPM4-0.5B): Lightweight version with 0.5B parameters, trained on 1T tokens
49
+ - [**MiniCPM4-8B-Eagle-FRSpec**](https://huggingface.co/openbmb/MiniCPM4-8B-Eagle-FRSpec): Eagle head for FRSpec, accelerating speculative inference
50
+ - [**MiniCPM4-8B-Eagle-FRSpec-QAT-cpmcu**](https://huggingface.co/openbmb/MiniCPM4-8B-Eagle-FRSpec-QAT-cpmcu): Eagle head with QAT for FRSpec, integrating speculation and quantization for ultra acceleration
51
+ - [**MiniCPM4-8B-Eagle-vLLM**](https://huggingface.co/openbmb/MiniCPM4-8B-Eagle-vLLM): Eagle head in vLLM format for speculative inference
52
+ - [**MiniCPM4-8B-marlin-Eagle-vLLM**](https://huggingface.co/openbmb/MiniCPM4-8B-marlin-Eagle-vLLM): Quantized Eagle head for vLLM format
53
+ - [**BitCPM4-0.5B**](https://huggingface.co/openbmb/BitCPM4-0.5B): Extreme ternary quantization of MiniCPM4-0.5B, achieving 90% bit width reduction
54
+ - [**BitCPM4-1B**](https://huggingface.co/openbmb/BitCPM4-1B): Extreme ternary quantization of MiniCPM3-1B, achieving 90% bit width reduction
55
+ - [**MiniCPM4-Survey**](https://huggingface.co/openbmb/MiniCPM4-Survey): Generates trustworthy, long-form survey papers from user queries
56
+ - [**MiniCPM4-MCP**](https://huggingface.co/openbmb/MiniCPM4-MCP): Integrates MCP tools to autonomously satisfy user requirements
57
+ </details>
58
+
59
+
60
+ ## Evaluation Results
61
+
62
+ ### Performance Evaluation
63
+ MiniCPM4.1 launches end-side versions with 8B parameter scale, both achieving best-in-class performance in their respective categories.
64
+
65
+ ![benchmark](https://github.com/OpenBMB/MiniCPM/blob/main/assets/minicpm4/benchmark4.1.png?raw=true)
66
+
67
+ ### Best Practices
68
+ 1. It is advisable to use temperature=0.9, topp=0.95. And we suggest setting max_output_token to 65,536 tokens.
69
+ 2. For math problems, we recommend using "Please reason step by step, and put your final answer within \boxed{}."
70
+ 3. And for English multiple-choice questions, we recommend starting with "Answer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering." And "你回���的最后一行必须是以下格式 '答案:$选项' (不带引号), 其中选项是ABCD之一。请在回答之前一步步思考" for Chinese MCQ.
71
+
72
+ ### Efficiency Evaluation
73
+ MiniCPM4.1 adopts sparse attention and speculative decoding to improve the inference efficiency. On RTX 4090, MiniCPM4.1 achieves 3x decoding speed improvement in reasoning.
74
+
75
+ ![benchmark](https://github.com/OpenBMB/MiniCPM/blob/main/assets/minicpm4/minicpm4.1_speed.png?raw=true)
76
+
77
+ #### Examples
78
+ <div align="center">
79
+ <a href="https://www.youtube.com/watch?v=VouXjLHKDUY"><img src="https://img.youtube.com/vi/VouXjLHKDUY/0.jpg", width=70%></a>
80
+ </div>
81
+
82
+
83
+
84
+ ## Usage
85
+ MiniCPM 4.1 can be used with following frameworks: Huggingface Transformers, SGLang, vLLM, and CPM.cu. For the ultimate inference speed, we highly recommend CPM.cu.
86
+
87
+ MiniCPM4/MiniCPM4.1 supports both dense attention inference and sparse attention inference modes, where vLLM and SGLang currently only support dense inference mode. If you want to use sparse inference mode, please use Huggingface Transformers and CPM.cu.
88
+
89
+ - Dense attention inference: vLLM, SGLang, Huggingface Transformers
90
+ - Sparse attention inference: Huggingface Transformers, CPM.cu
91
+
92
+ **To facilitate researches in sparse attention, we provide [InfLLM-V2 Training Kernels](https://github.com/OpenBMB/infllmv2_cuda_impl) and [InfLLM-V2 Inference Kernels](https://github.com/openbmb/cpm.cu).**
93
+
94
+ ### Inference with Transformers
95
+ MiniCPM4.1-8B requires `transformers>=4.56`.
96
+
97
+ - **Inference with Dense Attention**
98
+ ```python
99
+ from transformers import AutoModelForCausalLM, AutoTokenizer
100
+ import torch
101
+ torch.manual_seed(0)
102
+
103
+ path = 'openbmb/MiniCPM4.1-8B'
104
+ device = "cuda"
105
+ tokenizer = AutoTokenizer.from_pretrained(path)
106
+ model = AutoModelForCausalLM.from_pretrained(path, torch_dtype=torch.bfloat16, device_map=device, trust_remote_code=True)
107
+
108
+ # User can directly use the chat interface
109
+ # responds, history = model.chat(tokenizer, "Write an article about Artificial Intelligence.", temperature=0.7, top_p=0.7)
110
+ # print(responds)
111
+
112
+ # User can also use the generate interface
113
+ messages = [
114
+ {"role": "user", "content": "Write an article about Artificial Intelligence."},
115
+ ]
116
+ prompt_text = tokenizer.apply_chat_template(
117
+ messages,
118
+ tokenize=False,
119
+ add_generation_prompt=True,
120
+ )
121
+ model_inputs = tokenizer([prompt_text], return_tensors="pt").to(device)
122
+
123
+ model_outputs = model.generate(
124
+ **model_inputs,
125
+ max_new_tokens=32768,
126
+ top_p=0.95,
127
+ temperature=0.6
128
+ )
129
+ output_token_ids = [
130
+ model_outputs[i][len(model_inputs[i]):] for i in range(len(model_inputs['input_ids']))
131
+ ]
132
+
133
+ responses = tokenizer.batch_decode(output_token_ids, skip_special_tokens=True)[0]
134
+ print(responses)
135
+ ```
136
+
137
+ - **Inference with Sparse Attention**
138
+ MiniCPM4.1-8B supports `InfLLM v2`, a sparse attention mechanism designed for efficient long-sequence inference. It requires the [infllmv2_cuda_impl](https://github.com/OpenBMB/infllmv2_cuda_impl) library.
139
+
140
+ You can install it by running the following command:
141
+ ```bash
142
+ git clone -b feature_infer https://github.com/OpenBMB/infllmv2_cuda_impl.git
143
+ cd infllmv2_cuda_impl
144
+ git submodule update --init --recursive
145
+ pip install -e . # or python setup.py install
146
+ ```
147
+
148
+ To enable InfLLM v2, you need to add the `sparse_config` field in `config.json`:
149
+ ```json
150
+ {
151
+ ...,
152
+ "sparse_config": {
153
+ "kernel_size": 32,
154
+ "kernel_stride": 16,
155
+ "init_blocks": 1,
156
+ "block_size": 64,
157
+ "window_size": 2048,
158
+ "topk": 64,
159
+ "use_nope": false,
160
+ "dense_len": 8192
161
+ }
162
+ }
163
+ ```
164
+
165
+ These parameters control the behavior of InfLLM v2:
166
+ * `kernel_size` (default: 32): The size of semantic kernels.
167
+ * `kernel_stride` (default: 16): The stride between adjacent kernels.
168
+ * `init_blocks` (default: 1): The number of initial blocks that every query token attends to. This ensures attention to the beginning of the sequence.
169
+ * `block_size` (default: 64): The block size for key-value blocks.
170
+ * `window_size` (default: 2048): The size of the local sliding window.
171
+ * `topk` (default: 64): The specifies that each token computes attention with only the top-k most relevant key-value blocks.
172
+ * `use_nope` (default: false): Whether to use the NOPE technique in block selection for improved performance.
173
+ * `dense_len` (default: 8192): Since Sparse Attention offers limited benefits for short sequences, the model can use standard (dense) attention for shorter texts. The model will use dense attention for sequences with a token length below `dense_len` and switch to sparse attention for sequences exceeding this length. Set this to `-1` to always use sparse attention regardless of sequence length.
174
+
175
+ - **Long Context Extension**
176
+ MiniCPM4.1 natively supports context lengths of up to 65,536(64k) tokens. For conversations where the total length (including both input and output) significantly exceeds this limit, we recommend using RoPE scaling techniques for effective handling of long texts. We have validated the model's performance on context lengths of up to 131,072 tokens by modifying the LongRoPE factor.
177
+
178
+ You can apply the LongRoPE factor modification by modifying the model files. Specifically, in the `config.json` file, adjust the `rope_scaling` fields.
179
+ ```json
180
+ {
181
+ ...,
182
+ "rope_scaling": {
183
+ "rope_type": "longrope",
184
+ "long_factor": [0.9982316082870437, 1.033048153422584, 1.0749920956484724, 1.1255096879436193, 1.1863348602111476, 1.259543828902579, 1.3476188888731149, 1.4535223827776373, 1.5807816745852985, 1.7335856049489526, 1.9168922912975785, 2.1365471404135326, 2.3994084200118646, 2.713475511863602, 3.0880118452194134, 3.533650295140154, 4.062463396503134, 4.687974098908333, 5.425075306704039, 6.289818967956352, 7.29902962722721, 8.6357018163639, 10.210822723989212, 12.053807765671676, 14.193944598909404, 16.65780676784363, 19.463620727694074, 22.628311203524586, 26.150106147261315, 30.02526691405111, 34.23183327975347, 38.73811934094828, 43.502489489729555, 48.47627117965394, 53.61139491762471, 58.857366522037935, 64.16798299215064, 69.51359464319125, 74.86555458220285, 80.21497790341579, 85.55322183307433, 90.89611806932027, 96.26245306514224, 101.68269304046481, 107.18619510219668, 112.82253283014026, 118.63764063163615, 119.88866203644656, 120.9462882391725, 121.837565139014, 122.58663780572562, 123.2147719894291, 123.74049454862576, 124.17980424685767, 124.54641761955492, 124.85202548028222, 125.10654406389756, 125.31835105170659, 125.49450117164764, 125.64091910903052, 125.76256945356558, 125.86360463815589, 125.94749252260765, 126.01712561287873],
185
+ "short_factor": [0.9982316082870437, 1.033048153422584, 1.0749920956484724, 1.1255096879436193, 1.1863348602111476, 1.259543828902579, 1.3476188888731149, 1.4535223827776373, 1.5807816745852985, 1.7335856049489526, 1.9168922912975785, 2.1365471404135326, 2.3994084200118646, 2.713475511863602, 3.0880118452194134, 3.533650295140154, 4.062463396503134, 4.687974098908333, 5.425075306704039, 6.289818967956352, 7.29902962722721, 8.6357018163639, 10.210822723989212, 12.053807765671676, 14.193944598909404, 16.65780676784363, 19.463620727694074, 22.628311203524586, 26.150106147261315, 30.02526691405111, 34.23183327975347, 38.73811934094828, 43.502489489729555, 48.47627117965394, 53.61139491762471, 58.857366522037935, 64.16798299215064, 69.51359464319125, 74.86555458220285, 80.21497790341579, 85.55322183307433, 90.89611806932027, 96.26245306514224, 101.68269304046481, 107.18619510219668, 112.82253283014026, 118.63764063163615, 119.88866203644656, 120.9462882391725, 121.837565139014, 122.58663780572562, 123.2147719894291, 123.74049454862576, 124.17980424685767, 124.54641761955492, 124.85202548028222, 125.10654406389756, 125.31835105170659, 125.49450117164764, 125.64091910903052, 125.76256945356558, 125.86360463815589, 125.94749252260765, 126.01712561287873],
186
+ "original_max_position_embeddings": 65536
187
+ }
188
+ }
189
+ ```
190
+
191
+ ### Inference with [SGLang](https://github.com/sgl-project/sglang)
192
+
193
+ You can inference with SGLang using the standard mode and speculative decoding mode.
194
+
195
+ #### Speculative Decoding
196
+
197
+ For accelerated inference with speculative decoding, follow these steps:
198
+
199
+ ##### 1. Download MiniCPM4.1 Draft Model
200
+
201
+ First, download the MiniCPM4.1 draft model:
202
+
203
+ ```bash
204
+ cd /your_path
205
+ git clone https://huggingface.co/openbmb/MiniCPM4.1-8B-Eagle3
206
+ ```
207
+
208
+ ##### 2. Install EAGLE3-Compatible SGLang
209
+
210
+ The EAGLE3 adaptation PR has been submitted. For now, use our repository for installation:
211
+
212
+ ```bash
213
+ git clone https://github.com/LDLINGLINGLING/sglang.git
214
+ cd sglang
215
+ pip install -e "python[all]"
216
+ ```
217
+
218
+ ##### 3. Launch SGLang Server with Speculative Decoding
219
+
220
+ Start the SGLang server with speculative decoding enabled:
221
+
222
+ ```bash
223
+ python -m sglang.launch_server \
224
+ --model-path "openbmb/MiniCPM4.1-8B" \
225
+ --host "127.0.0.1" \
226
+ --port 30002 \
227
+ --mem-fraction-static 0.9 \
228
+ --speculative-algorithm EAGLE3 \
229
+ --speculative-draft-model-path "your/path/MiniCPM4_1-8B-Eagle3-bf16" \
230
+ --speculative-num-steps 3 \
231
+ --speculative-eagle-topk 1 \
232
+ --speculative-num-draft-tokens 32 \
233
+ --temperature 0.7
234
+ ```
235
+
236
+ ##### 4. Client Usage
237
+
238
+ The client usage remains the same for both standard and speculative decoding:
239
+
240
+ ```python
241
+ import openai
242
+
243
+ client = openai.Client(base_url=f"http://localhost:30002/v1", api_key="None")
244
+
245
+ response = client.chat.completions.create(
246
+ model="openbmb/MiniCPM4.1-8B",
247
+ messages=[
248
+ {"role": "user", "content": "Write an article about Artificial Intelligence."},
249
+ ],
250
+ temperature=0.6,
251
+ max_tokens=32768,
252
+ )
253
+
254
+ print(response.choices[0].message.content)
255
+ ```
256
+
257
+ Note: Make sure to update the port number in the client code to match the server port (30002 in the speculative decoding example).
258
+
259
+ ##### Configuration Parameters
260
+
261
+ - `--speculative-algorithm EAGLE3`: Enables EAGLE3 speculative decoding
262
+ - `--speculative-draft-model-path`: Path to the draft model for speculation
263
+ - `--speculative-num-steps`: Number of speculative steps (default: 3)
264
+ - `--speculative-eagle-topk`: Top-k parameter for EAGLE (default: 1)
265
+ - `--speculative-num-draft-tokens`: Number of draft tokens (default: 32)
266
+ - `--mem-fraction-static`: Memory fraction for static allocation (default: 0.9)
267
+
268
+ #### Standard Inference (Without Speculative Decoding)
269
+
270
+ For now, you need to install our forked version of SGLang.
271
+
272
+ ```bash
273
+ git clone -b openbmb https://github.com/OpenBMB/sglang.git
274
+ cd sglang
275
+
276
+ pip install --upgrade pip
277
+ pip install -e "python[all]"
278
+ ```
279
+
280
+ You can start the inference server by running the following command:
281
+
282
+ ```bash
283
+ python -m sglang.launch_server --model openbmb/MiniCPM4.1-8B --trust-remote-code --port 30000 --chat-template chatml
284
+ ```
285
+
286
+ Then you can use the chat interface by running the following command:
287
+
288
+ ```python
289
+ import openai
290
+
291
+ client = openai.Client(base_url=f"http://localhost:30000/v1", api_key="None")
292
+
293
+ response = client.chat.completions.create(
294
+ model="openbmb/MiniCPM4.1-8B",
295
+ messages=[
296
+ {"role": "user", "content": "Write an article about Artificial Intelligence."},
297
+ ],
298
+ temperature=0.6,
299
+ max_tokens=32768,
300
+ )
301
+
302
+ print(response.choices[0].message.content)
303
+ ```
304
+
305
+ ### Inference with [vLLM](https://github.com/vllm-project/vllm)
306
+ You can inference with vLLM using the standard mode and speculative decoding mode.
307
+
308
+ #### Speculative Decoding
309
+
310
+ For accelerated inference with speculative decoding using vLLM, follow these steps:
311
+
312
+ ##### 1. Download MiniCPM4.1 Draft Model
313
+
314
+ First, download the MiniCPM4.1 draft model and change the `architectures` in config.json as `LlamaForCausalLM`.
315
+
316
+ ```bash
317
+ cd /your_path
318
+ git clone https://huggingface.co/openbmb/MiniCPM4.1-8B-Eagle3
319
+ ```
320
+
321
+ ##### 2. Install EAGLE3-Compatible vLLM
322
+
323
+ The EAGLE3 vLLM PR has been submitted. For now, use our repository for installation:
324
+
325
+ ```bash
326
+ git clone https://github.com/LDLINGLINGLING/vllm.git
327
+ cd vllm
328
+ pip install -e .
329
+ ```
330
+
331
+ ##### 3. Launch vLLM Server with Speculative Decoding
332
+
333
+ Start the vLLM inference server with speculative decoding enabled. Make sure to update the model path in the speculative-config to point to your downloaded MiniCPM4_1-8B-Eagle3-bf16 folder:
334
+
335
+ ```bash
336
+ VLLM_USE_V1=1 \
337
+ vllm serve openbmb/MiniCPM4.1-8B \
338
+ --seed 42 \
339
+ --trust-remote-code \
340
+ --speculative-config '{
341
+ "model": "your/path/MiniCPM4_1-8B-Eagle3-bf16",
342
+ "num_speculative_tokens": 3,
343
+ "method": "eagle3",
344
+ "draft_tensor_parallel_size": 1
345
+ }'
346
+ ```
347
+
348
+ ##### 4. Client Usage Example
349
+
350
+ The client usage remains the same for both standard and speculative decoding:
351
+
352
+ ```python
353
+ import openai
354
+
355
+ client = openai.Client(base_url="http://localhost:8000/v1", api_key="EMPTY")
356
+
357
+ response = client.chat.completions.create(
358
+ model="openbmb/MiniCPM4.1-8B",
359
+ messages=[
360
+ {"role": "user", "content": "Write an article about Artificial Intelligence."},
361
+ ],
362
+ temperature=0.6,
363
+ max_tokens=32768,
364
+ extra_body=dict(add_special_tokens=True), # Ensures special tokens are added for chat template
365
+ )
366
+
367
+ print(response.choices[0].message.content)
368
+ ```
369
+
370
+ ##### vLLM Configuration Parameters
371
+
372
+ - `VLLM_USE_V1=1`: Enables vLLM v1 API
373
+ - `--speculative-config`: JSON configuration for speculative decoding
374
+ - `model`: Path to the draft model for speculation
375
+ - `num_speculative_tokens`: Number of speculative tokens (default: 3)
376
+ - `method`: Speculative decoding method (eagle3)
377
+ - `draft_tensor_parallel_size`: Tensor parallel size for draft model (default: 1)
378
+ - `--seed`: Random seed for reproducibility
379
+ - `--trust-remote-code`: Allow execution of remote code for custom models
380
+
381
+ #### Standard Inference (Without Speculative Decoding)
382
+
383
+ For now, you need to install the latest version of vLLM.
384
+
385
+ ```bash
386
+ pip install -U vllm \
387
+ --pre \
388
+ --extra-index-url https://wheels.vllm.ai/nightly
389
+ ```
390
+
391
+ Then you can inference MiniCPM4.1-8B with vLLM:
392
+ ```python
393
+ from transformers import AutoTokenizer
394
+ from vllm import LLM, SamplingParams
395
+
396
+ model_name = "openbmb/MiniCPM4.1-8B"
397
+ prompt = [{"role": "user", "content": "Write an article about Artificial Intelligence."}]
398
+
399
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
400
+ input_text = tokenizer.apply_chat_template(prompt, tokenize=False, add_generation_prompt=True)
401
+
402
+ llm = LLM(
403
+ model=model_name,
404
+ trust_remote_code=True,
405
+ max_num_batched_tokens=65536,
406
+ dtype="bfloat16",
407
+ gpu_memory_utilization=0.8,
408
+ )
409
+ sampling_params = SamplingParams(top_p=0.95, temperature=0.6, max_tokens=32768)
410
+
411
+ outputs = llm.generate(prompts=input_text, sampling_params=sampling_params)
412
+
413
+ print(outputs[0].outputs[0].text)
414
+ ```
415
+
416
+ Also, you can start the inference server by running the following command:
417
+ > **Note**: In vLLM's chat API, `add_special_tokens` is `False` by default. This means important special tokens—such as the beginning-of-sequence (BOS) token—will not be added automatically. To ensure the input prompt is correctly formatted for the model, you should explicitly set `extra_body={"add_special_tokens": True}`.
418
+
419
+ ```bash
420
+ vllm serve openbmb/MiniCPM4.1-8B --trust-remote-code
421
+ ```
422
+
423
+ Then you can use the chat interface by running the following code:
424
+
425
+ ```python
426
+ import openai
427
+
428
+ client = openai.Client(base_url="http://localhost:8000/v1", api_key="EMPTY")
429
+
430
+ response = client.chat.completions.create(
431
+ model="openbmb/MiniCPM4.1-8B",
432
+ messages=[
433
+ {"role": "user", "content": "Write an article about Artificial Intelligence."},
434
+ ],
435
+ temperature=0.6,
436
+ max_tokens=32768,
437
+ extra_body=dict(add_special_tokens=True), # Ensures special tokens are added for chat template
438
+
439
+ )
440
+
441
+ print(response.choices[0].message.content)
442
+ ```
443
+
444
+
445
+ ### Inference with [CPM.cu](https://github.com/OpenBMB/cpm.cu)
446
+
447
+ We recommend using [CPM.cu](https://github.com/OpenBMB/cpm.cu) for the inference of MiniCPM4 and MiniCPM4.1. CPM.cu is a CUDA inference framework developed by OpenBMB, which integrates efficient sparse, speculative sampling, and quantization techniques, fully leveraging the efficiency advantages of MiniCPM4 and MiniCPM4.1.
448
+
449
+ You can install CPM.cu by running the following command:
450
+
451
+ ```bash
452
+ git clone https://github.com/OpenBMB/cpm.cu.git --recursive
453
+ cd cpm.cu
454
+ python3 setup.py install
455
+ ```
456
+
457
+ MiniCPM4.1 natively supports context lengths of up to 65,536(64k) tokens. To reproduce the long-text acceleration effect in the paper, we recommend using the LongRoPE factors that have been validated. Change the `rope_scaling` field in the `config.json` file as the following to enable LongRoPE.
458
+ ```json
459
+ {
460
+ ...,
461
+ "rope_scaling": {
462
+ "rope_type": "longrope",
463
+ "long_factor": [0.9982316082870437, 1.033048153422584, 1.0749920956484724, 1.1255096879436193, 1.1863348602111476, 1.259543828902579, 1.3476188888731149, 1.4535223827776373, 1.5807816745852985, 1.7335856049489526, 1.9168922912975785, 2.1365471404135326, 2.3994084200118646, 2.713475511863602, 3.0880118452194134, 3.533650295140154, 4.062463396503134, 4.687974098908333, 5.425075306704039, 6.289818967956352, 7.29902962722721, 8.6357018163639, 10.210822723989212, 12.053807765671676, 14.193944598909404, 16.65780676784363, 19.463620727694074, 22.628311203524586, 26.150106147261315, 30.02526691405111, 34.23183327975347, 38.73811934094828, 43.502489489729555, 48.47627117965394, 53.61139491762471, 58.857366522037935, 64.16798299215064, 69.51359464319125, 74.86555458220285, 80.21497790341579, 85.55322183307433, 90.89611806932027, 96.26245306514224, 101.68269304046481, 107.18619510219668, 112.82253283014026, 118.63764063163615, 119.88866203644656, 120.9462882391725, 121.837565139014, 122.58663780572562, 123.2147719894291, 123.74049454862576, 124.17980424685767, 124.54641761955492, 124.85202548028222, 125.10654406389756, 125.31835105170659, 125.49450117164764, 125.64091910903052, 125.76256945356558, 125.86360463815589, 125.94749252260765, 126.01712561287873],
464
+ "short_factor": [0.9982316082870437, 1.033048153422584, 1.0749920956484724, 1.1255096879436193, 1.1863348602111476, 1.259543828902579, 1.3476188888731149, 1.4535223827776373, 1.5807816745852985, 1.7335856049489526, 1.9168922912975785, 2.1365471404135326, 2.3994084200118646, 2.713475511863602, 3.0880118452194134, 3.533650295140154, 4.062463396503134, 4.687974098908333, 5.425075306704039, 6.289818967956352, 7.29902962722721, 8.6357018163639, 10.210822723989212, 12.053807765671676, 14.193944598909404, 16.65780676784363, 19.463620727694074, 22.628311203524586, 26.150106147261315, 30.02526691405111, 34.23183327975347, 38.73811934094828, 43.502489489729555, 48.47627117965394, 53.61139491762471, 58.857366522037935, 64.16798299215064, 69.51359464319125, 74.86555458220285, 80.21497790341579, 85.55322183307433, 90.89611806932027, 96.26245306514224, 101.68269304046481, 107.18619510219668, 112.82253283014026, 118.63764063163615, 119.88866203644656, 120.9462882391725, 121.837565139014, 122.58663780572562, 123.2147719894291, 123.74049454862576, 124.17980424685767, 124.54641761955492, 124.85202548028222, 125.10654406389756, 125.31835105170659, 125.49450117164764, 125.64091910903052, 125.76256945356558, 125.86360463815589, 125.94749252260765, 126.01712561287873],
465
+ "original_max_position_embeddings": 65536
466
+ }
467
+ }
468
+ ```
469
+
470
+ After modification, you can run the following command to reproduce the long-context acceleration effect (the script will automatically download the model weights from HuggingFace)
471
+ ```bash
472
+ python3 tests/test_generate.py
473
+ ```
474
+
475
+ You can run the following command to infer with EAGLE3 speculative decoding algorithm.
476
+
477
+ ```bash
478
+ python3 -m cpmcu.cli \
479
+ --model-path $BASE_MODEL_PATH \
480
+ --draft-model-path $EAGLE3_DRAFT_MODEL_PATH \
481
+ --prompt-text "Write an article about Artificial Intelligence." \
482
+ --use-eagle3 true
483
+ ```
484
+
485
+ For more details about CPM.cu, please refer to [the repo CPM.cu](https://github.com/OpenBMB/cpm.cu).
486
+
487
+ ### Inference with llama.cpp and Ollama
488
+
489
+ We also support inference with [llama.cpp](https://github.com/ggml-org/llama.cpp) and [Ollama](https://ollama.com/).
490
+
491
+ ##### llama.cpp
492
+
493
+ You can download the GGUF format of MiniCPM4.1-8B model from [huggingface](https://huggingface.co/openbmb/MiniCPM4.1-8B-GGUF) and run it with llama.cpp for efficient CPU or GPU inference.
494
+ ```
495
+ # case 1: main-cli
496
+ ./build/bin/llama-cli -m MiniCPM4.1-8B-Q4_K_M.gguf -p "Write an article about Artificial Intelligence." -n 1500
497
+
498
+ # case 2: server
499
+ ## launch server
500
+ ./build/bin/llama-server -m MiniCPM4.1-8B-Q4_K_M.gguf --host 127.0.0.1 --port 8080 -c 4096 -fa on &
501
+
502
+ ## send request
503
+ curl -X POST http://127.0.0.1:8080/v1/chat/completions \
504
+ -H "Content-Type: application/json" \
505
+ -d '{
506
+ "model": "gpt-3.5-turbo",
507
+ "messages": [{"role": "user", "content": "Write an article about Artificial Intelligence."}],
508
+ "max_tokens": 1500
509
+ }'
510
+ ```
511
+
512
+ ##### Ollama
513
+ Please refer to [model hub](https://ollama.com/openbmb/minicpm4.1) for model download. After installing ollama package, you can use MiniCPM4.1 with following commands:
514
+ ```
515
+ ollama run openbmb/minicpm4.1
516
+ ```
517
+
518
+ ### Hybird Reasoning Mode
519
+
520
+ MiniCPM4.1 supports hybrid reasoning mode, which can be used in both deep reasoning mode and non-reasoning mode. To enable hybrid reasoning mode. User can set `enable_thinking=True` in `tokenizer.apply_chat_template` to enable hybrid reasoning mode, and set `enable_thinking=False` to enable non-reasoning mode. Similarly, user can directly add `/no_think` at the end of the query to enable non-reasoning mode. If not add any special token or add `/think` at the end of the query, the model will enable reasoning mode.
521
+
522
+ ```python
523
+ # Enable reasoning mode
524
+ prompt_text = tokenizer.apply_chat_template(
525
+ messages,
526
+ tokenize=False,
527
+ add_generation_prompt=True,
528
+ enable_thinking=True
529
+ )
530
+ # Enable non-reasoning mode
531
+ prompt_text = tokenizer.apply_chat_template(
532
+ messages,
533
+ tokenize=False,
534
+ add_generation_prompt=True,
535
+ enable_thinking=False
536
+ )
537
+ ```
538
+
539
+ ## Statement
540
+ - As a language model, MiniCPM generates content by learning from a vast amount of text.
541
+ - However, it does not possess the ability to comprehend or express personal opinions or value judgments.
542
+ - Any content generated by MiniCPM does not represent the viewpoints or positions of the model developers.
543
+ - Therefore, when using content generated by MiniCPM, users should take full responsibility for evaluating and verifying it on their own.
544
+
545
+ ## LICENSE
546
+ - This repository and MiniCPM models are released under the [Apache-2.0](https://github.com/OpenBMB/MiniCPM/blob/main/LICENSE) License.
547
+
548
+ ## Citation
549
+ - Please cite our [paper](https://arxiv.org/abs/2506.07900) if you find our work valuable.
550
+
551
+ ```bibtex
552
+ @article{minicpm4,
553
+ title={Minicpm4: Ultra-efficient llms on end devices},
554
+ author={MiniCPM, Team},
555
+ journal={arXiv preprint arXiv:2506.07900},
556
+ year={2025}
557
+ }
558
+ ```
added_tokens.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<|execute_end|>": 73444,
3
+ "<|execute_start|>": 73443,
4
+ "<|fim_middle|>": 73446,
5
+ "<|fim_prefix|>": 73445,
6
+ "<|fim_suffix|>": 73447,
7
+ "<|im_end|>": 73440,
8
+ "<|im_start|>": 73441,
9
+ "<|tool_call|>": 73442
10
+ }
config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "openbmb/MiniCPM4.1-8B",
3
+ "architectures": [
4
+ "MiniCPMForCausalLM"
5
+ ],
6
+ "auto_map": {
7
+ "AutoConfig": "configuration_minicpm.MiniCPMConfig",
8
+ "AutoModel": "modeling_minicpm.MiniCPMModel",
9
+ "AutoModelForCausalLM": "modeling_minicpm.MiniCPMForCausalLM",
10
+ "AutoModelForSeq2SeqLM": "modeling_minicpm.MiniCPMForCausalLM",
11
+ "AutoModelForSequenceClassification": "modeling_minicpm.MiniCPMForSequenceClassification"
12
+ },
13
+ "bos_token_id": 1,
14
+ "eos_token_id": [2, 73440],
15
+ "pad_token_id": 2,
16
+ "hidden_act": "silu",
17
+ "hidden_size": 4096,
18
+ "initializer_range": 0.1,
19
+ "intermediate_size": 16384,
20
+ "max_position_embeddings": 65536,
21
+ "model_type": "minicpm",
22
+ "num_attention_heads": 32,
23
+ "num_hidden_layers": 32,
24
+ "num_key_value_heads": 2,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_scaling": {
27
+ "rope_type": "longrope",
28
+ "long_factor": [0.9982316082870437, 1.033048153422584, 1.0749920956484724, 1.1255096879436193, 1.1863348602111476, 1.259543828902579, 1.3476188888731149, 1.4535223827776373, 1.5807816745852985, 1.7335856049489526, 1.9168922912975785, 2.1365471404135326, 2.3994084200118646, 2.713475511863602, 3.0880118452194134, 3.533650295140154, 4.062463396503134, 4.687974098908333, 5.425075306704039, 6.289818967956352, 7.29902962722721, 8.469695779093664, 9.81809877306655, 11.358657902065282, 13.102505860712087, 15.055862949967128, 17.218348131364184, 19.581439255386453, 22.127353314656723, 24.828633849376587, 27.6486820771775, 30.54334096108829, 33.46345345363812, 36.358112337548896, 39.17816056534983, 41.879441100069684, 44.425355159339965, 46.78844628336223, 48.95093146475928, 50.90428855401433, 52.648136512661125, 54.18869564165987, 55.537098635632745, 56.7077647874992, 57.71697544677006, 58.58171910802236, 59.31882031581807, 59.94433101822328, 60.47314411958625, 60.918782569507, 61.29331890286281, 61.60738599471455, 61.87024727431288, 62.089902123428836, 62.27320880977746, 62.42601274014111, 62.55327203194878, 62.65917552585329, 62.74725058582382, 62.82045955451526, 62.88128472678279, 62.931802319077946, 62.97374626130382, 63.008562806439365],
29
+ "short_factor": [0.9982316082870437, 1.033048153422584, 1.0749920956484724, 1.1255096879436193, 1.1863348602111476, 1.259543828902579, 1.3476188888731149, 1.4535223827776373, 1.5807816745852985, 1.7335856049489526, 1.9168922912975785, 2.1365471404135326, 2.3994084200118646, 2.713475511863602, 3.0880118452194134, 3.533650295140154, 4.062463396503134, 4.687974098908333, 5.425075306704039, 6.289818967956352, 7.29902962722721, 8.469695779093664, 9.81809877306655, 11.358657902065282, 13.102505860712087, 15.055862949967128, 17.218348131364184, 19.581439255386453, 22.127353314656723, 24.828633849376587, 27.6486820771775, 30.54334096108829, 33.46345345363812, 36.358112337548896, 39.17816056534983, 41.879441100069684, 44.425355159339965, 46.78844628336223, 48.95093146475928, 50.90428855401433, 52.648136512661125, 54.18869564165987, 55.537098635632745, 56.7077647874992, 57.71697544677006, 58.58171910802236, 59.31882031581807, 59.94433101822328, 60.47314411958625, 60.918782569507, 61.29331890286281, 61.60738599471455, 61.87024727431288, 62.089902123428836, 62.27320880977746, 62.42601274014111, 62.55327203194878, 62.65917552585329, 62.74725058582382, 62.82045955451526, 62.88128472678279, 62.931802319077946, 62.97374626130382, 63.008562806439365],
30
+ "original_max_position_embeddings": 65536
31
+ },
32
+ "torch_dtype": "bfloat16",
33
+ "transformers_version": "4.56.1",
34
+ "use_cache": true,
35
+ "vocab_size": 73448,
36
+ "rope_theta": 10000.0,
37
+ "scale_emb": 12,
38
+ "scale_depth": 1.4,
39
+ "mup_denominator": 32,
40
+ "dim_model_base": 256,
41
+ "tie_word_embeddings": false
42
+ }
configuration_minicpm.py ADDED
@@ -0,0 +1,203 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2025 The OpenBMB Team. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """ MiniCPM model configuration"""
16
+
17
+ from transformers.configuration_utils import PretrainedConfig
18
+ from transformers.utils import logging
19
+
20
+ logger = logging.get_logger(__name__)
21
+
22
+ MINICPM_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
23
+
24
+
25
+ class MiniCPMConfig(PretrainedConfig):
26
+ r"""
27
+ This is the configuration class to store the configuration of a [`MiniCPMModel`]. It is used to instantiate an MiniCPM
28
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
29
+ defaults will yield a similar configuration to that of the MiniCPM-7B.
30
+
31
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
32
+ documentation from [`PretrainedConfig`] for more information.
33
+
34
+
35
+ Args:
36
+ vocab_size (`int`, *optional*, defaults to 32000):
37
+ Vocabulary size of the MiniCPM model. Defines the number of different tokens that can be represented by the
38
+ `inputs_ids` passed when calling [`MiniCPMModel`]
39
+ hidden_size (`int`, *optional*, defaults to 4096):
40
+ Dimension of the hidden representations.
41
+ intermediate_size (`int`, *optional*, defaults to 11008):
42
+ Dimension of the MLP representations.
43
+ num_hidden_layers (`int`, *optional*, defaults to 32):
44
+ Number of hidden layers in the Transformer decoder.
45
+ num_attention_heads (`int`, *optional*, defaults to 32):
46
+ Number of attention heads for each attention layer in the Transformer decoder.
47
+ num_key_value_heads (`int`, *optional*):
48
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
49
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
50
+ `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
51
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
52
+ by meanpooling all the original heads within that group. For more details checkout [this
53
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
54
+ `num_attention_heads`.
55
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
56
+ The non-linear activation function (function or string) in the decoder.
57
+ max_position_embeddings (`int`, *optional*, defaults to 2048):
58
+ The maximum sequence length that this model might ever be used with. MiniCPM 1 supports up to 2048 tokens,
59
+ MiniCPM 2 up to 4096, CodeMiniCPM up to 16384.
60
+ initializer_range (`float`, *optional*, defaults to 0.02):
61
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
62
+ rms_norm_eps (`float`, *optional*, defaults to 1e-06):
63
+ The epsilon used by the rms normalization layers.
64
+ use_cache (`bool`, *optional*, defaults to `True`):
65
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
66
+ relevant if `config.is_decoder=True`.
67
+ pad_token_id (`int`, *optional*):
68
+ Padding token id.
69
+ bos_token_id (`int`, *optional*, defaults to 1):
70
+ Beginning of stream token id.
71
+ eos_token_id (`int`, *optional*, defaults to 2):
72
+ End of stream token id.
73
+ pretraining_tp (`int`, *optional*, defaults to 1):
74
+ Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
75
+ document](https://huggingface.co/docs/transformers/parallelism) to understand more about it. This value is
76
+ necessary to ensure exact reproducibility of the pretraining results. Please refer to [this
77
+ issue](https://github.com/pytorch/pytorch/issues/76232).
78
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
79
+ Whether to tie weight embeddings
80
+ rope_theta (`float`, *optional*, defaults to 10000.0):
81
+ The base period of the RoPE embeddings.
82
+ rope_scaling (`Dict`, *optional*):
83
+ Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
84
+ strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
85
+ `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
86
+ `max_position_embeddings` to the expected new maximum. See the following thread for more information on how
87
+ these scaling strategies behave:
88
+ https://www.reddit.com/r/LocalMiniCPM/comments/14mrgpr/dynamically_scaled_rope_further_increases/. This is an
89
+ experimental feature, subject to breaking API changes in future versions.
90
+ attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
91
+ Whether to use a bias in the query, key, value and output projection layers during self-attention.
92
+ attention_dropout (`float`, *optional*, defaults to 0.0):
93
+ The dropout ratio for the attention probabilities.
94
+
95
+ ```python
96
+ >>> from transformers import MiniCPMModel, MiniCPMConfig
97
+
98
+ >>> # Initializing a MiniCPM minicpm-7b style configuration
99
+ >>> configuration = MiniCPMConfig()
100
+
101
+ >>> # Initializing a model from the minicpm-7b style configuration
102
+ >>> model = MiniCPMModel(configuration)
103
+
104
+ >>> # Accessing the model configuration
105
+ >>> configuration = model.config
106
+ ```"""
107
+
108
+ model_type = 'minicpm'
109
+ keys_to_ignore_at_inference = ['past_key_values']
110
+
111
+ def __init__(
112
+ self,
113
+ vocab_size=32000,
114
+ hidden_size=4096,
115
+ intermediate_size=11008,
116
+ num_hidden_layers=32,
117
+ num_attention_heads=32,
118
+ num_key_value_heads=None,
119
+ hidden_act='silu',
120
+ max_position_embeddings=2048,
121
+ initializer_range=0.02,
122
+ rms_norm_eps=1e-6,
123
+ use_cache=True,
124
+ pad_token_id=None,
125
+ bos_token_id=1,
126
+ eos_token_id=2,
127
+ pretraining_tp=1,
128
+ tie_word_embeddings=True,
129
+ rope_theta=10000.0,
130
+ rope_scaling=None,
131
+ attention_bias=False,
132
+ attention_dropout=0.0,
133
+ scale_emb=1,
134
+ dim_model_base=1,
135
+ scale_depth=1,
136
+ mup_denominator=32,
137
+ sparse_config=None,
138
+ **kwargs):
139
+
140
+ self.vocab_size = vocab_size
141
+ self.max_position_embeddings = max_position_embeddings
142
+ self.hidden_size = hidden_size
143
+ self.intermediate_size = intermediate_size
144
+ self.num_hidden_layers = num_hidden_layers
145
+ self.num_attention_heads = num_attention_heads
146
+
147
+ # for backward compatibility
148
+ if num_key_value_heads is None:
149
+ num_key_value_heads = num_attention_heads
150
+
151
+ self.num_key_value_heads = num_key_value_heads
152
+ self.hidden_act = hidden_act
153
+ self.initializer_range = initializer_range
154
+ self.rms_norm_eps = rms_norm_eps
155
+ self.pretraining_tp = pretraining_tp
156
+ self.use_cache = use_cache
157
+ self.rope_theta = rope_theta
158
+ self.rope_scaling = rope_scaling
159
+ # self._rope_scaling_validation()
160
+ self.attention_bias = attention_bias
161
+ self.attention_dropout = attention_dropout
162
+ self.scale_emb = scale_emb
163
+ self.dim_model_base = dim_model_base
164
+ self.scale_depth = scale_depth
165
+ # only used for Eagle Head
166
+ self.mup_denominator = mup_denominator
167
+
168
+ # sparse config
169
+ self.sparse_config = sparse_config
170
+
171
+ super().__init__(
172
+ pad_token_id=pad_token_id,
173
+ bos_token_id=bos_token_id,
174
+ eos_token_id=eos_token_id,
175
+ tie_word_embeddings=tie_word_embeddings,
176
+ **kwargs,
177
+ )
178
+ try:
179
+ import flash_attn
180
+ self._attn_implementation = 'flash_attention_2'
181
+ except:
182
+ pass
183
+
184
+ def _rope_scaling_validation(self):
185
+ """
186
+ Validate the `rope_scaling` configuration.
187
+ """
188
+ if self.rope_scaling is None:
189
+ return
190
+
191
+ if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
192
+ raise ValueError(
193
+ '`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, '
194
+ f'got {self.rope_scaling}'
195
+ )
196
+ rope_scaling_type = self.rope_scaling.get('type', None)
197
+ rope_scaling_factor = self.rope_scaling.get('factor', None)
198
+ if rope_scaling_type is None or rope_scaling_type not in ['linear', 'dynamic']:
199
+ raise ValueError(
200
+ f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"
201
+ )
202
+ if rope_scaling_factor is None or not isinstance(rope_scaling_factor, float) or rope_scaling_factor <= 1.0:
203
+ raise ValueError(f"`rope_scaling`'s factor field must be a float > 1, got {rope_scaling_factor}")
generation_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 1,
3
+ "do_sample": true,
4
+ "eos_token_id": [
5
+ 2,
6
+ 73440
7
+ ],
8
+ "pad_token_id": 2,
9
+ "temperature": 0.8,
10
+ "top_p": 0.8,
11
+ "transformers_version": "4.46.1"
12
+ }
model-00001-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d893df3a00be018df333f102251db12de40d614a39438ee9721398b272b0b9c0
3
+ size 4968335472
model-00002-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:064bcba714ff425b660057fd2a09f9e55bbb0db75e52fc5d082ddf3fcf5af11e
3
+ size 4542417008
model-00003-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a23c3ad2c4c1ca4eae8cbd0fd14cbc0391096cd167bb4969dc49b52abfac7a67
3
+ size 4936647824
model-00004-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e3809c32918b84150715c343736a27fb4cf082473fb82505f87566d7179588ea
3
+ size 1923141264
model.safetensors.index.json ADDED
@@ -0,0 +1,299 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_parameters": 8185253888,
4
+ "total_size": 16370507776
5
+ },
6
+ "weight_map": {
7
+ "lm_head.weight": "model-00003-of-00004.safetensors",
8
+ "model.embed_tokens.weight": "model-00002-of-00004.safetensors",
9
+ "model.layers.0.input_layernorm.weight": "model-00004-of-00004.safetensors",
10
+ "model.layers.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
11
+ "model.layers.0.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
12
+ "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
13
+ "model.layers.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
14
+ "model.layers.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
15
+ "model.layers.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
16
+ "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
17
+ "model.layers.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
18
+ "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
19
+ "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
20
+ "model.layers.1.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
21
+ "model.layers.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
22
+ "model.layers.1.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
23
+ "model.layers.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
24
+ "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
25
+ "model.layers.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
26
+ "model.layers.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
27
+ "model.layers.10.input_layernorm.weight": "model-00003-of-00004.safetensors",
28
+ "model.layers.10.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
29
+ "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
30
+ "model.layers.10.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
31
+ "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
32
+ "model.layers.10.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
33
+ "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
34
+ "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
35
+ "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
36
+ "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
37
+ "model.layers.11.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
38
+ "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
39
+ "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
40
+ "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
41
+ "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
42
+ "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
43
+ "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
44
+ "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
45
+ "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
46
+ "model.layers.12.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
47
+ "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
48
+ "model.layers.12.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
49
+ "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
50
+ "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
51
+ "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
52
+ "model.layers.12.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
53
+ "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
54
+ "model.layers.13.input_layernorm.weight": "model-00001-of-00004.safetensors",
55
+ "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
56
+ "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
57
+ "model.layers.13.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
58
+ "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
59
+ "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
60
+ "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
61
+ "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
62
+ "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
63
+ "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
64
+ "model.layers.14.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
65
+ "model.layers.14.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
66
+ "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
67
+ "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
68
+ "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
69
+ "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
70
+ "model.layers.14.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
71
+ "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
72
+ "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
73
+ "model.layers.15.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
74
+ "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
75
+ "model.layers.15.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
76
+ "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
77
+ "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
78
+ "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
79
+ "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
80
+ "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
81
+ "model.layers.16.input_layernorm.weight": "model-00001-of-00004.safetensors",
82
+ "model.layers.16.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
83
+ "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
84
+ "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
85
+ "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
86
+ "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
87
+ "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
88
+ "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
89
+ "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
90
+ "model.layers.17.input_layernorm.weight": "model-00001-of-00004.safetensors",
91
+ "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
92
+ "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
93
+ "model.layers.17.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
94
+ "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
95
+ "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
96
+ "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
97
+ "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
98
+ "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
99
+ "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
100
+ "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
101
+ "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
102
+ "model.layers.18.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
103
+ "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
104
+ "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
105
+ "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
106
+ "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
107
+ "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
108
+ "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
109
+ "model.layers.19.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
110
+ "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
111
+ "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
112
+ "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
113
+ "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
114
+ "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
115
+ "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
116
+ "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
117
+ "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
118
+ "model.layers.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
119
+ "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
120
+ "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
121
+ "model.layers.2.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
122
+ "model.layers.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
123
+ "model.layers.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
124
+ "model.layers.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
125
+ "model.layers.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
126
+ "model.layers.20.input_layernorm.weight": "model-00001-of-00004.safetensors",
127
+ "model.layers.20.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
128
+ "model.layers.20.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
129
+ "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
130
+ "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
131
+ "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
132
+ "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
133
+ "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
134
+ "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
135
+ "model.layers.21.input_layernorm.weight": "model-00004-of-00004.safetensors",
136
+ "model.layers.21.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
137
+ "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
138
+ "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
139
+ "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
140
+ "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
141
+ "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
142
+ "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
143
+ "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
144
+ "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
145
+ "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
146
+ "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
147
+ "model.layers.22.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
148
+ "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
149
+ "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
150
+ "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
151
+ "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
152
+ "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
153
+ "model.layers.23.input_layernorm.weight": "model-00002-of-00004.safetensors",
154
+ "model.layers.23.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
155
+ "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
156
+ "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
157
+ "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
158
+ "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
159
+ "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
160
+ "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
161
+ "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
162
+ "model.layers.24.input_layernorm.weight": "model-00002-of-00004.safetensors",
163
+ "model.layers.24.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
164
+ "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
165
+ "model.layers.24.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
166
+ "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
167
+ "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
168
+ "model.layers.24.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
169
+ "model.layers.24.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
170
+ "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
171
+ "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
172
+ "model.layers.25.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
173
+ "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
174
+ "model.layers.25.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
175
+ "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
176
+ "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
177
+ "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
178
+ "model.layers.25.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
179
+ "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
180
+ "model.layers.26.input_layernorm.weight": "model-00002-of-00004.safetensors",
181
+ "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
182
+ "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
183
+ "model.layers.26.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
184
+ "model.layers.26.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
185
+ "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
186
+ "model.layers.26.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
187
+ "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
188
+ "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
189
+ "model.layers.27.input_layernorm.weight": "model-00001-of-00004.safetensors",
190
+ "model.layers.27.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
191
+ "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
192
+ "model.layers.27.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
193
+ "model.layers.27.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
194
+ "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
195
+ "model.layers.27.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
196
+ "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
197
+ "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
198
+ "model.layers.28.input_layernorm.weight": "model-00002-of-00004.safetensors",
199
+ "model.layers.28.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
200
+ "model.layers.28.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
201
+ "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
202
+ "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
203
+ "model.layers.28.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
204
+ "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
205
+ "model.layers.28.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
206
+ "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
207
+ "model.layers.29.input_layernorm.weight": "model-00001-of-00004.safetensors",
208
+ "model.layers.29.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
209
+ "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
210
+ "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
211
+ "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
212
+ "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
213
+ "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
214
+ "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
215
+ "model.layers.29.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
216
+ "model.layers.3.input_layernorm.weight": "model-00003-of-00004.safetensors",
217
+ "model.layers.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
218
+ "model.layers.3.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
219
+ "model.layers.3.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
220
+ "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
221
+ "model.layers.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
222
+ "model.layers.3.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
223
+ "model.layers.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
224
+ "model.layers.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
225
+ "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
226
+ "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
227
+ "model.layers.30.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
228
+ "model.layers.30.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
229
+ "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
230
+ "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
231
+ "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
232
+ "model.layers.30.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
233
+ "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
234
+ "model.layers.31.input_layernorm.weight": "model-00001-of-00004.safetensors",
235
+ "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
236
+ "model.layers.31.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
237
+ "model.layers.31.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
238
+ "model.layers.31.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
239
+ "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
240
+ "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
241
+ "model.layers.31.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
242
+ "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
243
+ "model.layers.4.input_layernorm.weight": "model-00003-of-00004.safetensors",
244
+ "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
245
+ "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
246
+ "model.layers.4.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
247
+ "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
248
+ "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
249
+ "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
250
+ "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
251
+ "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
252
+ "model.layers.5.input_layernorm.weight": "model-00002-of-00004.safetensors",
253
+ "model.layers.5.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
254
+ "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
255
+ "model.layers.5.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
256
+ "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
257
+ "model.layers.5.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
258
+ "model.layers.5.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
259
+ "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
260
+ "model.layers.5.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
261
+ "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
262
+ "model.layers.6.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
263
+ "model.layers.6.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
264
+ "model.layers.6.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
265
+ "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
266
+ "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
267
+ "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
268
+ "model.layers.6.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
269
+ "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
270
+ "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
271
+ "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
272
+ "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
273
+ "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
274
+ "model.layers.7.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
275
+ "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
276
+ "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
277
+ "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
278
+ "model.layers.7.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
279
+ "model.layers.8.input_layernorm.weight": "model-00003-of-00004.safetensors",
280
+ "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
281
+ "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
282
+ "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
283
+ "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
284
+ "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
285
+ "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
286
+ "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
287
+ "model.layers.8.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
288
+ "model.layers.9.input_layernorm.weight": "model-00003-of-00004.safetensors",
289
+ "model.layers.9.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
290
+ "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
291
+ "model.layers.9.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
292
+ "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
293
+ "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
294
+ "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
295
+ "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
296
+ "model.layers.9.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
297
+ "model.norm.weight": "model-00003-of-00004.safetensors"
298
+ }
299
+ }
modeling_minicpm.py ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|im_end|>",
4
+ "<|im_start|>",
5
+ "<|tool_call|>",
6
+ "<|execute_start|>",
7
+ "<|execute_end|>",
8
+ "<|fim_prefix|>",
9
+ "<|fim_middle|>",
10
+ "<|fim_suffix|>"
11
+ ],
12
+ "bos_token": {
13
+ "content": "<s>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false
18
+ },
19
+ "eos_token": {
20
+ "content": "<|im_end|>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false
25
+ },
26
+ "unk_token": {
27
+ "content": "<unk>",
28
+ "lstrip": false,
29
+ "normalized": false,
30
+ "rstrip": false,
31
+ "single_word": false
32
+ }
33
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bb74d51116831c3bf65db812c553f94ab0c88dcf97a5bbb37e3504f6d359c530
3
+ size 1181204
tokenizer_config.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": null,
5
+ "added_tokens_decoder": {
6
+ "0": {
7
+ "content": "<unk>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "1": {
15
+ "content": "<s>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "2": {
23
+ "content": "</s>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": false,
27
+ "single_word": false,
28
+ "special": true
29
+ },
30
+ "73440": {
31
+ "content": "<|im_end|>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false,
36
+ "special": true
37
+ },
38
+ "73441": {
39
+ "content": "<|im_start|>",
40
+ "lstrip": false,
41
+ "normalized": false,
42
+ "rstrip": false,
43
+ "single_word": false,
44
+ "special": true
45
+ },
46
+ "73442": {
47
+ "content": "<|tool_call|>",
48
+ "lstrip": false,
49
+ "normalized": false,
50
+ "rstrip": false,
51
+ "single_word": false,
52
+ "special": true
53
+ },
54
+ "73443": {
55
+ "content": "<|execute_start|>",
56
+ "lstrip": false,
57
+ "normalized": false,
58
+ "rstrip": false,
59
+ "single_word": false,
60
+ "special": true
61
+ },
62
+ "73444": {
63
+ "content": "<|execute_end|>",
64
+ "lstrip": false,
65
+ "normalized": false,
66
+ "rstrip": false,
67
+ "single_word": false,
68
+ "special": true
69
+ },
70
+ "73445": {
71
+ "content": "<|fim_prefix|>",
72
+ "lstrip": false,
73
+ "normalized": false,
74
+ "rstrip": false,
75
+ "single_word": false,
76
+ "special": true
77
+ },
78
+ "73446": {
79
+ "content": "<|fim_middle|>",
80
+ "lstrip": false,
81
+ "normalized": false,
82
+ "rstrip": false,
83
+ "single_word": false,
84
+ "special": true
85
+ },
86
+ "73447": {
87
+ "content": "<|fim_suffix|>",
88
+ "lstrip": false,
89
+ "normalized": false,
90
+ "rstrip": false,
91
+ "single_word": false,
92
+ "special": true
93
+ }
94
+ },
95
+ "additional_special_tokens": [
96
+ "<|im_end|>",
97
+ "<|im_start|>",
98
+ "<|tool_call|>",
99
+ "<|execute_start|>",
100
+ "<|execute_end|>",
101
+ "<|fim_prefix|>",
102
+ "<|fim_middle|>",
103
+ "<|fim_suffix|>"
104
+ ],
105
+ "bos_token": "<s>",
106
+ "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% if enable_thinking is defined and enable_thinking is false %}{{ '<think>\n\n</think>\n' }}{% endif %}{% endif %}",
107
+ "clean_up_tokenization_spaces": false,
108
+ "eos_token": "<|im_end|>",
109
+ "legacy": true,
110
+ "model_max_length": 1000000000000000019884624838656,
111
+ "pad_token": null,
112
+ "sp_model_kwargs": {},
113
+ "spaces_between_special_tokens": false,
114
+ "tokenizer_class": "LlamaTokenizer",
115
+ "unk_token": "<unk>",
116
+ "use_default_system_prompt": false
117
+ }