BanglaFast-500M-Instruct / tokenizer_config.json
HFShishir's picture
Release BanglaFast-500M-Instruct v1.0
367cc32 verified
Raw History Blame Contribute Delete
1.08 kB
{
"backend": "tokenizers",
"bos_token": "<s>",
"clean_up_tokenization_spaces": true,
"eos_token": "</s>",
"extra_special_tokens": [
"<|endoftext|>",
"<s>",
"</s>",
"<unk>",
"<pad>",
"<mask>",
"<think>",
"</think>",
"<|im_start|>",
"<|im_end|>",
"<|system|>",
"<|user|>",
"<|assistant|>",
"<|tool_call|>",
"<|tool_result|>",
"<|code|>",
"<|/code|>",
"<|math|>",
"<|/math|>"
],
"is_local": false,
"local_files_only": false,
"mask_token": "<mask>",
"model_max_length": 1000000000000000019884624838656,
"model_type": "bpe",
"pad_token": "<pad>",
"special_tokens": [
"<|endoftext|>",
"<s>",
"</s>",
"<unk>",
"<pad>",
"<mask>",
"<think>",
"</think>",
"<|im_start|>",
"<|im_end|>",
"<|system|>",
"<|user|>",
"<|assistant|>",
"<|tool_call|>",
"<|tool_result|>",
"<|code|>",
"<|/code|>",
"<|math|>",
"<|/math|>"
],
"tokenizer_class": "TokenizersBackend",
"unk_token": "<unk>",
"vocab_size": 48000
}