Xenova HF Staff nico-martin HF Staff commited on
Commit
774cc90
·
1 Parent(s): 1f4ea7c

Add response template (#4)

Browse files

- Add response template (ef6fb121506639b89a34bb51c089a2e47884c802)
- Match response template to checkpoint output protocol (20d5926d6de88fdb5ee75228f9a391f82bd6d75f)


Co-authored-by: Nico Martin <nico-martin@users.noreply.huggingface.co>

Files changed (1) hide show
  1. tokenizer_config.json +23 -1
tokenizer_config.json CHANGED
@@ -203,5 +203,27 @@
203
  "pad_token": "<|endoftext|>",
204
  "split_special_tokens": false,
205
  "tokenizer_class": "Qwen2Tokenizer",
206
- "unk_token": null
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
207
  }
 
203
  "pad_token": "<|endoftext|>",
204
  "split_special_tokens": false,
205
  "tokenizer_class": "Qwen2Tokenizer",
206
+ "unk_token": null,
207
+ "response_template": {
208
+ "defaults": {
209
+ "role": "assistant"
210
+ },
211
+ "start_anchor": "<|im_start|>assistant\n",
212
+ "fields": {
213
+ "tool_calls": {
214
+ "open_pattern": "\\s*<tool_call>",
215
+ "close": "</tool_call>",
216
+ "repeats": true,
217
+ "content": "json",
218
+ "transform": {
219
+ "type": "function",
220
+ "function": "{content}"
221
+ }
222
+ },
223
+ "content": {
224
+ "close_pattern": "\\s*(?:<\\|im_end\\|>|<\\|endoftext\\|>|<\\|eot_id\\|>)",
225
+ "content": "text"
226
+ }
227
+ }
228
+ }
229
  }