Gab10M / tokenizer_config.json
gszauer's picture
Gab 10M: chat model, Transformers port, reference.js
04036d0 verified
Raw History Blame Contribute Delete
484 Bytes
{
"tokenizer_class": "GabByteTokenizer",
"auto_map": {
"AutoTokenizer": [
"tokenization_gabgpt.GabByteTokenizer",
null
]
},
"chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}<|u|>{{ message['content'] }}<|e|>{% elif message['role'] == 'assistant' %}<|a|>{{ message['content'] }}<|e|>{% endif %}{% endfor %}{% if add_generation_prompt %}<|a|>{% endif %}",
"model_max_length": 2048,
"clean_up_tokenization_spaces": false
}