pac3-code-nano / tokenizer_config.json
PAC3VISION's picture
Upload 3 files
dfc5d68 verified
Raw History Blame Contribute Delete
609 Bytes
{
"model_type": "mamba",
"bos_token": "<|system|>",
"eos_token": "<|end_of_turn|>",
"pad_token": "<|endoftext|>",
"unk_token": "<|unk|>",
"sep_token": "<|end_of_turn|>",
"mask_token": "<|mask|>",
"tokenizer_class": "MambaTokenizer",
"chat_template": null,
"clean_up_tokenization_spaces": false,
"special_tokens_map": {
"bos_token": "<|system|>",
"eos_token": "<|end_of_turn|>",
"pad_token": "<|endoftext|>",
"unk_token": "<|unk|>",
"sep_token": "<|end_of_turn|>",
"mask_token": "<|mask|>"
}
}