Download tokenizer_config.json from yalilabs/alta-tokenizer: direct link, hf CLI and curl.
- Browser
- Download file 635 Bytes
-
https://huggingface.co/yalilabs/alta-tokenizer/resolve/main/tokenizer_config.json
- Command line
-
hf download hf://yalilabs/alta-tokenizer/tokenizer_config.json
-
curl -L -o tokenizer_config.json https://huggingface.co/yalilabs/alta-tokenizer/resolve/main/tokenizer_config.json
635 Bytes
| { | |
| "vocab_size": 65536, | |
| "model_type": "BPE", | |
| "byte_fallback": true, | |
| "tokenizer_class": "PreTrainedTokenizerFast", | |
| "special_tokens": [ | |
| "<|pad|>", | |
| "<|unk|>", | |
| "<|bos|>", | |
| "<|eos|>", | |
| "<|im_start|>", | |
| "<|im_end|>", | |
| "<|begin_of_text|>", | |
| "<|start_header_id|>", | |
| "<|end_header_id|>", | |
| "<|eot_id|>", | |
| "<|thought_start|>", | |
| "<|thought_end|>", | |
| "<|tool_start|>", | |
| "<|tool_end|>", | |
| "<|tool_call|>", | |
| "<|tool_result|>", | |
| "<|fim_prefix|>", | |
| "<|fim_middle|>", | |
| "<|fim_suffix|>", | |
| "<|fim_pad|>" | |
| ], | |
| "language": "Kinyarwanda", | |
| "apostrophe_handling": "preserved_in_words" | |
| } |