File size: 196 Bytes
4b750ba | 1 2 3 4 5 6 7 8 | {
"bos_token": "<|bos|>",
"eos_token": "<|eos|>",
"name_or_path": "ntokenizer-hf-export",
"pad_token": "<|pad|>",
"tokenizer_class": "PreTrainedTokenizerFast",
"unk_token": "<|unk|>"
} |
4b750ba | 1 2 3 4 5 6 7 8 | {
"bos_token": "<|bos|>",
"eos_token": "<|eos|>",
"name_or_path": "ntokenizer-hf-export",
"pad_token": "<|pad|>",
"tokenizer_class": "PreTrainedTokenizerFast",
"unk_token": "<|unk|>"
} |