diff --git "a/tokenizer.json" "b/tokenizer.json" new file mode 100644--- /dev/null +++ "b/tokenizer.json" @@ -0,0 +1,19543 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "<|eot|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1, + "content": "<|pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 5, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 6, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 7, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 8, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": { + "type": "Sequence", + "normalizers": [ + { + "type": "NFC" + } + ] + }, + "pre_tokenizer": { + "type": "ByteLevel", + "add_prefix_space": false, + "trim_offsets": true, + "use_regex": true + }, + "post_processor": null, + "decoder": { + "type": "ByteLevel", + "add_prefix_space": true, + "trim_offsets": true, + "use_regex": true + }, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": null, + "continuing_subword_prefix": null, + "end_of_word_suffix": null, + "fuse_unk": false, + "byte_fallback": false, + "ignore_merges": false, + "vocab": { + "<|eot|>": 0, + "<|pad|>": 1, + "": 2, + "": 3, + "": 4, + "": 5, + "": 6, + "": 7, + "": 8, + "!": 9, + "\"": 10, + "#": 11, + "$": 12, + "%": 13, + "&": 14, + "'": 15, + "(": 16, + ")": 17, + "*": 18, + "+": 19, + ",": 20, + "-": 21, + ".": 22, + "/": 23, + "0": 24, + "1": 25, + "2": 26, + "3": 27, + "4": 28, + "5": 29, + "6": 30, + "7": 31, + "8": 32, + "9": 33, + ":": 34, + ";": 35, + "<": 36, + "=": 37, + ">": 38, + "?": 39, + "@": 40, + "A": 41, + "B": 42, + "C": 43, + "D": 44, + "E": 45, + "F": 46, + "G": 47, + "H": 48, + "I": 49, + "J": 50, + "K": 51, + "L": 52, + "M": 53, + "N": 54, + "O": 55, + "P": 56, + "Q": 57, + "R": 58, + "S": 59, + "T": 60, + "U": 61, + "V": 62, + "W": 63, + "X": 64, + "Y": 65, + "Z": 66, + "[": 67, + "\\": 68, + "]": 69, + "^": 70, + "_": 71, + "`": 72, + "a": 73, + "b": 74, + "c": 75, + "d": 76, + "e": 77, + "f": 78, + "g": 79, + "h": 80, + "i": 81, + "j": 82, + "k": 83, + "l": 84, + "m": 85, + "n": 86, + "o": 87, + "p": 88, + "q": 89, + "r": 90, + "s": 91, + "t": 92, + "u": 93, + "v": 94, + "w": 95, + "x": 96, + "y": 97, + "z": 98, + "{": 99, + "|": 100, + "}": 101, + "~": 102, + "¡": 103, + "¢": 104, + "£": 105, + "¤": 106, + "¥": 107, + "¦": 108, + "§": 109, + "¨": 110, + "©": 111, + "ª": 112, + "«": 113, + "¬": 114, + "®": 115, + "¯": 116, + "°": 117, + "±": 118, + "²": 119, + "³": 120, + "´": 121, + "µ": 122, + "¶": 123, + "·": 124, + "¸": 125, + "¹": 126, + "º": 127, + "»": 128, + "¼": 129, + "½": 130, + "¾": 131, + "¿": 132, + "À": 133, + "Á": 134, + "Â": 135, + "Ã": 136, + "Ä": 137, + "Å": 138, + "Æ": 139, + "Ç": 140, + "È": 141, + "É": 142, + "Ê": 143, + "Ë": 144, + "Ì": 145, + "Í": 146, + "Î": 147, + "Ï": 148, + "Ð": 149, + "Ñ": 150, + "Ò": 151, + "Ó": 152, + "Ô": 153, + "Õ": 154, + "Ö": 155, + "×": 156, + "Ø": 157, + "Ù": 158, + "Ú": 159, + "Û": 160, + "Ü": 161, + "Ý": 162, + "Þ": 163, + "ß": 164, + "à": 165, + "á": 166, + "â": 167, + "ã": 168, + "ä": 169, + "å": 170, + "æ": 171, + "ç": 172, + "è": 173, + "é": 174, + "ê": 175, + "ë": 176, + "ì": 177, + "í": 178, + "î": 179, + "ï": 180, + "ð": 181, + "ñ": 182, + "ò": 183, + "ó": 184, + "ô": 185, + "õ": 186, + "ö": 187, + "÷": 188, + "ø": 189, + "ù": 190, + "ú": 191, + "û": 192, + "ü": 193, + "ý": 194, + "þ": 195, + "ÿ": 196, + "Ā": 197, + "ā": 198, + "Ă": 199, + "ă": 200, + "Ą": 201, + "ą": 202, + "Ć": 203, + "ć": 204, + "Ĉ": 205, + "ĉ": 206, + "Ċ": 207, + "ċ": 208, + "Č": 209, + "č": 210, + "Ď": 211, + "ď": 212, + "Đ": 213, + "đ": 214, + "Ē": 215, + "ē": 216, + "Ĕ": 217, + "ĕ": 218, + "Ė": 219, + "ė": 220, + "Ę": 221, + "ę": 222, + "Ě": 223, + "ě": 224, + "Ĝ": 225, + "ĝ": 226, + "Ğ": 227, + "ğ": 228, + "Ġ": 229, + "ġ": 230, + "Ģ": 231, + "ģ": 232, + "Ĥ": 233, + "ĥ": 234, + "Ħ": 235, + "ħ": 236, + "Ĩ": 237, + "ĩ": 238, + "Ī": 239, + "ī": 240, + "Ĭ": 241, + "ĭ": 242, + "Į": 243, + "į": 244, + "İ": 245, + "ı": 246, + "IJ": 247, + "ij": 248, + "Ĵ": 249, + "ĵ": 250, + "Ķ": 251, + "ķ": 252, + "ĸ": 253, + "Ĺ": 254, + "ĺ": 255, + "Ļ": 256, + "ļ": 257, + "Ľ": 258, + "ľ": 259, + "Ŀ": 260, + "ŀ": 261, + "Ł": 262, + "ł": 263, + "Ń": 264, + "Ġt": 265, + "er": 266, + "oo": 267, + "ot": 268, + "he": 269, + "in": 270, + "eot": 271, + "<|": 272, + "|>": 273, + "Ġa": 274, + "at": 275, + "loo": 276, + "on": 277, + "re": 278, + "Ġthe": 279, + "an": 280, + "us": 281, + "Ġs": 282, + "Ġw": 283, + "Ġo": 284, + "Ġc": 285, + "ed": 286, + "en": 287, + "loom": 288, + "es": 289, + "hat": 290, + "is": 291, + "it": 292, + "up": 293, + "Ġf": 294, + "or": 295, + "Ġof": 296, + "Ġm": 297, + "al": 298, + "look": 299, + "user": 300, + "ar": 301, + "Ġp": 302, + "Ġin": 303, + "Ġb": 304, + "om": 305, + "Ġd": 306, + "ing": 307, + "lookup": 308, + ".<|": 309, + "ic": 310, + "as": 311, + "nd": 312, + "ou": 313, + "ion": 314, + "Ġto": 315, + "le": 316, + "ls": 317, + "ul": 318, + "Ġand": 319, + "res": 320, + "ent": 321, + "am": 322, + "Ġh": 323, + "Ġn": 324, + "ools": 325, + "ow": 326, + "tools": 327, + "Ġl": 328, + "il": 329, + "id": 330, + "ĠI": 331, + "Ġis": 332, + "ult": 333, + "What": 334, + "ĠS": 335, + "Ġthat": 336, + "el": 337, + "<|": 339, + "rom": 340, + "ac": 341, + "result": 342, + "Ġre": 343, + "Ġe": 344, + "her": 345, + "ro": 346, + "ĠT": 347, + "ĠC": 348, + "ad": 349, + "st": 350, + "Ġg": 351, + "ĠA": 352, + "Ġfrom": 353, + "Ġbe": 354, + "ol": 355, + "Ġwas": 356, + "ur": 357, + "Ġon": 358, + "Ġth": 359, + "ation": 360, + "ame": 361, + "iv": 362, + "?" + ], + [ + "Ġ", + "a" + ], + [ + "a", + "t" + ], + [ + "l", + "oo" + ], + [ + "o", + "n" + ], + [ + "r", + "e" + ], + [ + "Ġt", + "he" + ], + [ + "a", + "n" + ], + [ + "u", + "s" + ], + [ + "Ġ", + "s" + ], + [ + "Ġ", + "w" + ], + [ + "Ġ", + "o" + ], + [ + "Ġ", + "c" + ], + [ + "e", + "d" + ], + [ + "e", + "n" + ], + [ + "loo", + "m" + ], + [ + "e", + "s" + ], + [ + "h", + "at" + ], + [ + "i", + "s" + ], + [ + "i", + "t" + ], + [ + "u", + "p" + ], + [ + "Ġ", + "f" + ], + [ + "o", + "r" + ], + [ + "Ġo", + "f" + ], + [ + "Ġ", + "m" + ], + [ + "a", + "l" + ], + [ + "loo", + "k" + ], + [ + "us", + "er" + ], + [ + "a", + "r" + ], + [ + "Ġ", + "p" + ], + [ + "Ġ", + "in" + ], + [ + "Ġ", + "b" + ], + [ + "o", + "m" + ], + [ + "Ġ", + "d" + ], + [ + "in", + "g" + ], + [ + "look", + "up" + ], + [ + ".", + "<|" + ], + [ + "i", + "c" + ], + [ + "a", + "s" + ], + [ + "n", + "d" + ], + [ + "o", + "u" + ], + [ + "i", + "on" + ], + [ + "Ġt", + "o" + ], + [ + "l", + "e" + ], + [ + "l", + "s" + ], + [ + "u", + "l" + ], + [ + "Ġa", + "nd" + ], + [ + "re", + "s" + ], + [ + "en", + "t" + ], + [ + "a", + "m" + ], + [ + "Ġ", + "h" + ], + [ + "Ġ", + "n" + ], + [ + "oo", + "ls" + ], + [ + "o", + "w" + ], + [ + "t", + "ools" + ], + [ + "Ġ", + "l" + ], + [ + "i", + "l" + ], + [ + "i", + "d" + ], + [ + "Ġ", + "I" + ], + [ + "Ġ", + "is" + ], + [ + "ul", + "t" + ], + [ + "W", + "hat" + ], + [ + "Ġ", + "S" + ], + [ + "Ġt", + "hat" + ], + [ + "e", + "l" + ], + [ + "<", + "/" + ], + [ + ">", + "<|" + ], + [ + "r", + "om" + ], + [ + "a", + "c" + ], + [ + "res", + "ult" + ], + [ + "Ġ", + "re" + ], + [ + "Ġ", + "e" + ], + [ + "h", + "er" + ], + [ + "r", + "o" + ], + [ + "Ġ", + "T" + ], + [ + "Ġ", + "C" + ], + [ + "a", + "d" + ], + [ + "s", + "t" + ], + [ + "Ġ", + "g" + ], + [ + "Ġ", + "A" + ], + [ + "Ġf", + "rom" + ], + [ + "Ġb", + "e" + ], + [ + "o", + "l" + ], + [ + "Ġw", + "as" + ], + [ + "u", + "r" + ], + [ + "Ġ", + "on" + ], + [ + "Ġt", + "h" + ], + [ + "at", + "ion" + ], + [ + "am", + "e" + ], + [ + "i", + "v" + ], + [ + "?", + "