{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": { "type": "Lowercase" }, "pre_tokenizer": { "type": "Split", "pattern": { "Regex": "." }, "behavior": "Isolated", "invert": false }, "post_processor": null, "decoder": null, "model": { "type": "WordLevel", "vocab": { "\n": 1, " ": 2, "!": 3, "\"": 4, "#": 5, "%": 6, "'": 7, "(": 8, ")": 9, "*": 10, "+": 11, ",": 12, "-": 13, ".": 14, "/": 15, "0": 16, "1": 17, "2": 18, "3": 19, "4": 20, "5": 21, "6": 22, "7": 23, "8": 24, "9": 25, ":": 26, ";": 27, "<": 28, "=": 29, ">": 30, "?": 31, "A": 32, "B": 33, "C": 34, "D": 35, "E": 36, "F": 37, "G": 38, "H": 39, "I": 40, "J": 41, "K": 42, "L": 43, "M": 44, "N": 45, "O": 46, "P": 47, "Q": 48, "R": 49, "S": 50, "T": 51, "U": 52, "V": 53, "W": 54, "X": 55, "Y": 56, "Z": 57, "[": 58, "\\": 59, "]": 60, "_": 61, "a": 62, "b": 63, "c": 64, "d": 65, "e": 66, "f": 67, "g": 68, "h": 69, "i": 70, "j": 71, "k": 72, "l": 73, "m": 74, "n": 75, "o": 76, "p": 77, "q": 78, "r": 79, "s": 80, "t": 81, "u": 82, "v": 83, "w": 84, "x": 85, "y": 86, "z": 87, "{": 88, "}": 89, "": 0 }, "unk_token": "" } }