{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 216, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 217, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 218, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 219, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "Split", "pattern": { "Regex": "[\\s\\S]" }, "behavior": "Isolated", "invert": false }, "post_processor": { "type": "TemplateProcessing", "single": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "", "type_id": 0 } } ], "pair": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } }, { "SpecialToken": { "id": "", "type_id": 1 } } ], "special_tokens": { "": { "id": "", "ids": [ 218 ], "tokens": [ "" ] } } }, "decoder": { "type": "Sequence", "decoders": [ { "type": "Replace", "pattern": { "String": "" }, "content": "" }, { "type": "Fuse" } ] }, "model": { "type": "WordLevel", "vocab": { "": 0, "": 1, "": 2, "": 3, " ": 4, "!": 5, "\"": 6, "#": 7, "$": 8, "%": 9, "&": 10, "'": 11, "(": 12, ")": 13, "*": 14, "+": 15, ",": 16, "-": 17, ".": 18, "/": 19, "0": 20, "1": 21, "2": 22, "3": 23, "4": 24, "5": 25, "6": 26, "7": 27, "8": 28, "9": 29, ":": 30, ";": 31, "<": 32, "=": 33, ">": 34, "?": 35, "@": 36, "A": 37, "B": 38, "C": 39, "D": 40, "E": 41, "F": 42, "G": 43, "H": 44, "I": 45, "J": 46, "K": 47, "L": 48, "M": 49, "N": 50, "O": 51, "P": 52, "Q": 53, "R": 54, "S": 55, "T": 56, "U": 57, "V": 58, "W": 59, "X": 60, "Y": 61, "Z": 62, "[": 63, "]": 64, "^": 65, "_": 66, "a": 67, "b": 68, "c": 69, "d": 70, "e": 71, "f": 72, "g": 73, "h": 74, "i": 75, "j": 76, "k": 77, "l": 78, "m": 79, "n": 80, "o": 81, "p": 82, "q": 83, "r": 84, "s": 85, "t": 86, "u": 87, "v": 88, "w": 89, "x": 90, "y": 91, "z": 92, "|": 93, "~": 94, "¡": 95, "«": 96, "®": 97, "°": 98, "·": 99, "»": 100, "×": 101, "à": 102, "á": 103, "ç": 104, "è": 105, "é": 106, "í": 107, "ǔ": 108, "ɓ": 109, "ə": 110, "ก": 111, "ท": 112, "น": 113, "ภ": 114, "ย": 115, "ร": 116, "ษ": 117, "า": 118, "ิ": 119, "เ": 120, "ไ": 121, "๒": 122, "๕": 123, "ລ": 124, "ວ": 125, "ສ": 126, "າ": 127, "ក": 128, "ខ": 129, "គ": 130, "ឃ": 131, "ង": 132, "ច": 133, "ឆ": 134, "ជ": 135, "ឈ": 136, "ញ": 137, "ដ": 138, "ឋ": 139, "ឌ": 140, "ណ": 141, "ត": 142, "ថ": 143, "ទ": 144, "ធ": 145, "ន": 146, "ប": 147, "ផ": 148, "ព": 149, "ភ": 150, "ម": 151, "យ": 152, "រ": 153, "ល": 154, "វ": 155, "ស": 156, "ហ": 157, "ឡ": 158, "អ": 159, "ឧ": 160, "ឫ": 161, "ឬ": 162, "ឯ": 163, "ឲ": 164, "ា": 165, "ិ": 166, "ី": 167, "ឹ": 168, "ឺ": 169, "ុ": 170, "ូ": 171, "ួ": 172, "ើ": 173, "ឿ": 174, "ៀ": 175, "េ": 176, "ែ": 177, "ៃ": 178, "ោ": 179, "ៅ": 180, "ំ": 181, "ះ": 182, "ៈ": 183, "៉": 184, "៊": 185, "់": 186, "៌": 187, "៍": 188, "៏": 189, "័": 190, "្": 191, "។": 192, "៕": 193, "៖": 194, "ៗ": 195, "៘": 196, "៛": 197, "‑": 198, "–": 199, "—": 200, "‘": 201, "’": 202, "“": 203, "”": 204, "•": 205, "…": 206, "‹": 207, "›": 208, "→": 209, "−": 210, "●": 211, "�": 212, "😀": 213, "😛": 214, "🙂": 215, "": 216, "": 217, "": 218, "": 219, " ": 220, "!": 221, "\"": 222, "#": 223, "$": 224, "%": 225, "&": 226, "'": 227, "(": 228, ")": 229, "*": 230, "+": 231, ",": 232, "-": 233, ".": 234, "/": 235, "0": 236, "1": 237, "2": 238, "3": 239, "4": 240, "5": 241, "6": 242, "7": 243, "8": 244, "9": 245, ":": 246, ";": 247, "<": 248, "=": 249, ">": 250, "?": 251, "@": 252, "A": 253, "B": 254, "C": 255, "D": 256, "E": 257, "F": 258, "G": 259, "H": 260, "I": 261, "J": 262, "K": 263, "L": 264, "M": 265, "N": 266, "O": 267, "P": 268, "Q": 269, "R": 270, "S": 271, "T": 272, "U": 273, "V": 274, "W": 275, "X": 276, "Y": 277, "Z": 278, "[": 279, "]": 280, "^": 281, "_": 282, "a": 283, "b": 284, "c": 285, "d": 286, "e": 287, "f": 288, "g": 289, "h": 290, "i": 291, "j": 292, "k": 293, "l": 294, "m": 295, "n": 296, "o": 297, "p": 298, "q": 299, "r": 300, "s": 301, "t": 302, "u": 303, "v": 304, "w": 305, "x": 306, "y": 307, "z": 308, "|": 309, "~": 310, "¡": 311, "«": 312, "®": 313, "°": 314, "·": 315, "»": 316, "×": 317, "à": 318, "á": 319, "ç": 320, "è": 321, "é": 322, "í": 323, "ǔ": 324, "ก": 325, "ท": 326, "น": 327, "ภ": 328, "ย": 329, "ร": 330, "ษ": 331, "า": 332, "ิ": 333, "เ": 334, "ไ": 335, "๒": 336, "๕": 337, "ລ": 338, "ວ": 339, "ສ": 340, "າ": 341, "ក": 342, "ខ": 343, "គ": 344, "ឃ": 345, "ង": 346, "ច": 347, "ឆ": 348, "ជ": 349, "ឈ": 350, "ញ": 351, "ដ": 352, "ឋ": 353, "ឌ": 354, "ឍ": 355, "ណ": 356, "ត": 357, "ថ": 358, "ទ": 359, "ធ": 360, "ន": 361, "ប": 362, "ផ": 363, "ព": 364, "ភ": 365, "ម": 366, "យ": 367, "រ": 368, "ល": 369, "វ": 370, "ឝ": 371, "ឞ": 372, "ស": 373, "ហ": 374, "ឡ": 375, "អ": 376, "ឥ": 377, "ឦ": 378, "ឧ": 379, "ឩ": 380, "ឪ": 381, "ឫ": 382, "ឬ": 383, "ឭ": 384, "ឮ": 385, "ឯ": 386, "ឰ": 387, "ឱ": 388, "ឲ": 389, "ឳ": 390, "ា": 391, "ិ": 392, "ី": 393, "ឹ": 394, "ឺ": 395, "ុ": 396, "ូ": 397, "ួ": 398, "ើ": 399, "ឿ": 400, "ៀ": 401, "េ": 402, "ែ": 403, "ៃ": 404, "ោ": 405, "ៅ": 406, "ំ": 407, "ះ": 408, "ៈ": 409, "៉": 410, "៊": 411, "់": 412, "៌": 413, "៍": 414, "៎": 415, "៏": 416, "័": 417, "៑": 418, "្": 419, "។": 420, "៕": 421, "៖": 422, "ៗ": 423, "៘": 424, "៛": 425, "‑": 426, "–": 427, "—": 428, "‘": 429, "’": 430, "“": 431, "”": 432, "•": 433, "…": 434, "‹": 435, "›": 436, "→": 437, "−": 438, "●": 439, "�": 440, "😀": 441, "😛": 442, "🙂": 443 }, "unk_token": "" } }