File size: 577 Bytes
b8782ab | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | {
"tokenizer_class": "PreTrainedTokenizerFast",
"model_max_length": 128,
"bos_token": "<S>",
"eos_token": "<E>",
"unk_token": "<unk>",
"pad_token": "<pad>",
"mask_token": "<mask>",
"additional_special_tokens": [
"<U>",
"<A>",
"<Y>",
"<C>",
"<H>"
],
"bos_token_id": 0,
"eos_token_id": 2,
"unk_token_id": 9,
"pad_token_id": 7,
"mask_token_id": 8,
"clean_up_tokenization_spaces": false,
"model_type_internal": "BPE",
"decoder": "ByteLevel",
"pre_tokenizer": "ByteLevel",
"normalizer": null,
"has_post_processor": true
}
|