File size: 483 Bytes
546acd4
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
{
  "tokenizer_class": "GabByteTokenizer",
  "auto_map": {
    "AutoTokenizer": [
      "tokenization_gabgpt.GabByteTokenizer",
      null
    ]
  },
  "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}<|u|>{{ message['content'] }}<|e|>{% elif message['role'] == 'assistant' %}<|a|>{{ message['content'] }}<|e|>{% endif %}{% endfor %}{% if add_generation_prompt %}<|a|>{% endif %}",
  "model_max_length": 512,
  "clean_up_tokenization_spaces": false
}