File size: 827 Bytes
4e4a7d8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
{
 "add_prefix_space": false,
 "backend": "tokenizers",
 "bos_token": "<|endoftext|>",
 "clean_up_tokenization_spaces": true,
 "eos_token": "<|endoftext|>",
 "is_local": true,
 "local_files_only": false,
 "max_length": 200,
 "model_max_length": 2048,
 "pad_to_multiple_of": null,
 "pad_token": "<|endoftext|>",
 "pad_token_type_id": 0,
 "padding_side": "left",
 "return_token_type_ids": false,
 "stride": 0,
 "tokenizer_class": "CodeGenTokenizer",
 "truncation_side": "right",
 "truncation_strategy": "longest_first",
 "unk_token": "<|endoftext|>",
 "chat_template": "{{ bos_token }}{% for m in messages %}{% if m['role'] == 'system' %}[SYS] {{ m['content'] }} [/SYS]\n{% elif m['role'] == 'user' %}[INST] {{ m['content'] }} [/INST]{% elif m['role'] == 'assistant' %} {{ m['content'] }}{{ eos_token }}{% endif %}{% endfor %}"
}