tyrian-1b / tokenizer_config.json
redptam's picture
Upload Tyrian 1B (DPO)
ff5eca0 verified
Raw History Blame Contribute Delete
400 Bytes
{
"tokenizer_class": "PreTrainedTokenizerFast",
"model_max_length": 8192,
"padding_side": "right",
"bos_token": "<bos>",
"eos_token": "<eos>",
"unk_token": "<unk>",
"pad_token": "<pad>",
"chat_template": "{% for message in messages %}<|im_start|>{{ message['role'] }}\n{{ message['content'] }}<|im_end|>\n{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
}