alta-tokenizer / tokenizer_config.json
Nschadrack's picture
Upload 2 files
c77e9fb verified
Raw History Blame Contribute Delete
635 Bytes
{
"vocab_size": 65536,
"model_type": "BPE",
"byte_fallback": true,
"tokenizer_class": "PreTrainedTokenizerFast",
"special_tokens": [
"<|pad|>",
"<|unk|>",
"<|bos|>",
"<|eos|>",
"<|im_start|>",
"<|im_end|>",
"<|begin_of_text|>",
"<|start_header_id|>",
"<|end_header_id|>",
"<|eot_id|>",
"<|thought_start|>",
"<|thought_end|>",
"<|tool_start|>",
"<|tool_end|>",
"<|tool_call|>",
"<|tool_result|>",
"<|fim_prefix|>",
"<|fim_middle|>",
"<|fim_suffix|>",
"<|fim_pad|>"
],
"language": "Kinyarwanda",
"apostrophe_handling": "preserved_in_words"
}