File size: 672 Bytes
07976a1 | 1 2 3 4 5 6 7 8 9 10 | {
"bos_token": "<|bos|>",
"eos_token": "<|assistant_end|>",
"pad_token": "<|assistant_end|>",
"clean_up_tokenization_spaces": false,
"model_max_length": 2048,
"add_bos_token": true,
"tokenizer_class": "PreTrainedTokenizerFast",
"chat_template": "{{ bos_token }}{% set sys = namespace(t='') %}{% for m in messages %}{% if m['role'] == 'system' %}{% set sys.t = m['content'] + '\n\n' %}{% elif m['role'] == 'user' %}<|user_start|>{{ sys.t }}{{ m['content'] }}<|user_end|>{% set sys.t = '' %}{% elif m['role'] == 'assistant' %}<|assistant_start|>{{ m['content'] }}<|assistant_end|>{% endif %}{% endfor %}{% if add_generation_prompt %}<|assistant_start|>{% endif %}"
} |