miceworks commited on
Commit
0583e7a
·
verified ·
1 Parent(s): dfdedfe

Synchronize tokenizer_config.json with GGUF binary single source of truth

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +27 -2
tokenizer_config.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "vocab_size": 4311,
3
  "bos_token_id": 1,
4
  "eos_token_id": 2,
5
  "special_tokens": {
@@ -58,5 +58,30 @@
58
  "ar",
59
  "hi",
60
  "zh-Hans"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
61
  ]
62
- }
 
1
  {
2
+ "vocab_size": 49152,
3
  "bos_token_id": 1,
4
  "eos_token_id": 2,
5
  "special_tokens": {
 
58
  "ar",
59
  "hi",
60
  "zh-Hans"
61
+ ],
62
+ "bos_token": "<|bos|>",
63
+ "eos_token": "<|eos|>",
64
+ "pad_token": "<|pad|>",
65
+ "pad_token_id": 0,
66
+ "model_max_length": 4096,
67
+ "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}<|bos|>System: {{ message['content'] }}<|eos|>\n{% elif message['role'] == 'user' %}<|bos|>User: {{ message['content'] }}<|eos|>\n{% elif message['role'] == 'assistant' %}<|bos|>Assistant: {% if message['content'] %}{{ message['content'] }}<|eos|>\n{% endif %}{% endif %}{% endfor %}{% if add_generation_prompt %}<|bos|>Assistant:\n<think>\n{% endif %}",
68
+ "additional_special_tokens": [
69
+ "<think>",
70
+ "</think>",
71
+ "[RETRIEVE]",
72
+ "[/RETRIEVE]",
73
+ "[EVALUATE]",
74
+ "[ACCEPT]",
75
+ "[REJECT]",
76
+ "<|vision_start|>",
77
+ "<|vision_patch|>",
78
+ "<|vision_end|>",
79
+ "<|voice_stream|>",
80
+ "<|rag_context|>",
81
+ "<|/rag_context|>",
82
+ "<|mcp_call|>",
83
+ "<|mcp_response|>",
84
+ "<|citation|>",
85
+ "<|/citation|>"
86
  ]
87
+ }