{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "post_processor": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "": 0, "": 1, "": 2, "": 3, ".": 4, "T": 5, "a": 6, "d": 7, "e": 8, "g": 9, "h": 10, "i": 11, "l": 12, "m": 13, "n": 14, "o": 15, "p": 16, "r": 17, "s": 18, "t": 19, "v": 20, "x": 21, "»": 22, "¿": 23, "ï": 24, "Ċ": 25, "č": 26, "Ġ": 27, "is": 28, "Ġt": 29, "Th": 30, "ex": 31, "in": 32, "»¿": 33, "": 34, "čĊ": 35, "Ġis": 36, "Ġ": 37, "Ġtex": 38, "This": 39, "Ġtext": 40 }, "merges": [ [ "i", "s" ], [ "Ġ", "t" ], [ "T", "h" ], [ "e", "x" ], [ "i", "n" ], [ "»", "¿" ], [ "ï", "»¿" ], [ "č", "Ċ" ], [ "Ġ", "is" ], [ "Ġ", "" ], [ "Ġt", "ex" ], [ "Th", "is" ], [ "Ġtex", "t" ] ] } }