| { | |
| "backend": "tokenizers", | |
| "bos_token": "<|endoftext|>", | |
| "chat_template": "{% set ns = namespace(has_sys=false) %}{% for m in messages %}{% if m['role'] == 'system' %}{% set ns.has_sys = true %}{% endif %}{% endfor %}{% if not ns.has_sys %}<|system|>你是 HakuChat，一个由本地团队训练的中文代码助手。写代码时先写「思考：」，再写「答案：」，代码放在 python 代码块中。{% endif %}{% for message in messages %}{% if message['role'] == 'system' %}<|system|>{{ message['content'] }}{% elif message['role'] == 'user' %}<|user|>{{ message['content'] }}{% elif message['role'] == 'assistant' %}<|assistant|>{{ message['content'] }}{% endif %}{% endfor %}{% if add_generation_prompt %}<|assistant|>{% endif %}", | |
| "clean_up_tokenization_spaces": false, | |
| "eos_token": "<|endoftext|>", | |
| "extra_special_tokens": [ | |
| "<|system|>", | |
| "<|user|>", | |
| "<|assistant|>", | |
| "<|tool|>", | |
| "<|observation|>" | |
| ], | |
| "is_local": true, | |
| "local_files_only": false, | |
| "model_max_length": 4096, | |
| "pad_token": "<|endoftext|>", | |
| "tokenizer_class": "TokenizersBackend", | |
| "unk_token": null | |
| } | |