tokenizer_config.json

{"add_prefix_space": false, "added_tokens_decoder": {"0": {"content": "<s>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true}, "1": {"content": "<pad>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true}, "2": {"content": "</s>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true}, "3": {"content": "<unk>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true}, "50264": {"content": "<mask>", "lstrip": true, "normalized": true, "rstrip": false, "single_word": false, "special": true}, "50265": {"content": "Ғ", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}, "50266": {"content": "ғ", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}, "50267": {"content": "Ҙ", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}, "50268": {"content": "ҙ", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}, "50269": {"content": "Ҡ", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}, "50270": {"content": "ҡ", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}, "50271": {"content": "Ң", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}, "50272": {"content": "ң", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}, "50273": {"content": "Ө", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}, "50274": {"content": "ө", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}, "50275": {"content": "Ҫ", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}, "50276": {"content": "ҫ", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}, "50277": {"content": "Ү", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}, "50278": {"content": "ү", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}, "50279": {"content": "Һ", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}, "50280": {"content": "һ", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}, "50281": {"content": "Ә", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}, "50282": {"content": "ә", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false}}, "backend": "tokenizers", "bos_token": "<s>", "clean_up_tokenization_spaces": false, "cls_token": "<s>", "eos_token": "</s>", "errors": "replace", "is_local": false, "local_files_only": false, "mask_token": "<mask>", "model_max_length": 512, "pad_token": "<pad>", "processor_class": "TrOCRProcessor", "sep_token": "</s>", "tokenizer_class": "RobertaTokenizer", "trim_offsets": true, "unk_token": "<unk>"}