{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 228, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 229, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 230, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 231, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "Split", "pattern": { "Regex": "[\\s\\S]" }, "behavior": "Isolated", "invert": false }, "post_processor": { "type": "TemplateProcessing", "single": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "", "type_id": 0 } } ], "pair": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } }, { "SpecialToken": { "id": "", "type_id": 1 } } ], "special_tokens": { "": { "id": "", "ids": [ 230 ], "tokens": [ "" ] } } }, "decoder": { "type": "Sequence", "decoders": [ { "type": "Replace", "pattern": { "String": "" }, "content": "" }, { "type": "Fuse" } ] }, "model": { "type": "WordLevel", "vocab": { "": 0, "": 1, "": 2, "": 3, " ": 4, "!": 5, "\"": 6, "#": 7, "$": 8, "%": 9, "&": 10, "'": 11, "(": 12, ")": 13, "*": 14, "+": 15, ",": 16, "-": 17, ".": 18, "/": 19, "0": 20, "1": 21, "2": 22, "3": 23, "4": 24, "5": 25, "6": 26, "7": 27, "8": 28, "9": 29, ":": 30, ";": 31, "<": 32, "=": 33, ">": 34, "?": 35, "@": 36, "A": 37, "B": 38, "C": 39, "D": 40, "E": 41, "F": 42, "G": 43, "H": 44, "I": 45, "J": 46, "K": 47, "L": 48, "M": 49, "N": 50, "O": 51, "P": 52, "Q": 53, "R": 54, "S": 55, "T": 56, "U": 57, "V": 58, "W": 59, "X": 60, "Y": 61, "Z": 62, "[": 63, "]": 64, "^": 65, "_": 66, "a": 67, "b": 68, "c": 69, "d": 70, "e": 71, "f": 72, "g": 73, "h": 74, "i": 75, "j": 76, "k": 77, "l": 78, "m": 79, "n": 80, "o": 81, "p": 82, "q": 83, "r": 84, "s": 85, "t": 86, "u": 87, "v": 88, "w": 89, "x": 90, "y": 91, "z": 92, "|": 93, "~": 94, "¡": 95, "«": 96, "®": 97, "°": 98, "·": 99, "»": 100, "×": 101, "à": 102, "á": 103, "ç": 104, "è": 105, "é": 106, "í": 107, "ǔ": 108, "ก": 109, "ท": 110, "น": 111, "ภ": 112, "ย": 113, "ร": 114, "ษ": 115, "า": 116, "ิ": 117, "เ": 118, "ไ": 119, "๒": 120, "๕": 121, "ລ": 122, "ວ": 123, "ສ": 124, "າ": 125, "ក": 126, "ខ": 127, "គ": 128, "ឃ": 129, "ង": 130, "ច": 131, "ឆ": 132, "ជ": 133, "ឈ": 134, "ញ": 135, "ដ": 136, "ឋ": 137, "ឌ": 138, "ឍ": 139, "ណ": 140, "ត": 141, "ថ": 142, "ទ": 143, "ធ": 144, "ន": 145, "ប": 146, "ផ": 147, "ព": 148, "ភ": 149, "ម": 150, "យ": 151, "រ": 152, "ល": 153, "វ": 154, "ឝ": 155, "ឞ": 156, "ស": 157, "ហ": 158, "ឡ": 159, "អ": 160, "ឥ": 161, "ឦ": 162, "ឧ": 163, "ឩ": 164, "ឪ": 165, "ឫ": 166, "ឬ": 167, "ឭ": 168, "ឮ": 169, "ឯ": 170, "ឰ": 171, "ឱ": 172, "ឲ": 173, "ឳ": 174, "ា": 175, "ិ": 176, "ី": 177, "ឹ": 178, "ឺ": 179, "ុ": 180, "ូ": 181, "ួ": 182, "ើ": 183, "ឿ": 184, "ៀ": 185, "េ": 186, "ែ": 187, "ៃ": 188, "ោ": 189, "ៅ": 190, "ំ": 191, "ះ": 192, "ៈ": 193, "៉": 194, "៊": 195, "់": 196, "៌": 197, "៍": 198, "៎": 199, "៏": 200, "័": 201, "៑": 202, "្": 203, "។": 204, "៕": 205, "៖": 206, "ៗ": 207, "៘": 208, "៛": 209, "‑": 210, "–": 211, "—": 212, "‘": 213, "’": 214, "“": 215, "”": 216, "•": 217, "…": 218, "‹": 219, "›": 220, "→": 221, "−": 222, "●": 223, "�": 224, "😀": 225, "😛": 226, "🙂": 227, "": 228, "": 229, "": 230, "": 231, " ": 232, "!": 233, "\"": 234, "#": 235, "$": 236, "%": 237, "&": 238, "'": 239, "(": 240, ")": 241, "*": 242, "+": 243, ",": 244, "-": 245, ".": 246, "/": 247, "0": 248, "1": 249, "2": 250, "3": 251, "4": 252, "5": 253, "6": 254, "7": 255, "8": 256, "9": 257, ":": 258, ";": 259, "<": 260, "=": 261, ">": 262, "?": 263, "@": 264, "A": 265, "B": 266, "C": 267, "D": 268, "E": 269, "F": 270, "G": 271, "H": 272, "I": 273, "J": 274, "K": 275, "L": 276, "M": 277, "N": 278, "O": 279, "P": 280, "Q": 281, "R": 282, "S": 283, "T": 284, "U": 285, "V": 286, "W": 287, "X": 288, "Y": 289, "Z": 290, "[": 291, "]": 292, "^": 293, "_": 294, "a": 295, "b": 296, "c": 297, "d": 298, "e": 299, "f": 300, "g": 301, "h": 302, "i": 303, "j": 304, "k": 305, "l": 306, "m": 307, "n": 308, "o": 309, "p": 310, "q": 311, "r": 312, "s": 313, "t": 314, "u": 315, "v": 316, "w": 317, "x": 318, "y": 319, "z": 320, "|": 321, "~": 322, "¡": 323, "«": 324, "®": 325, "°": 326, "·": 327, "»": 328, "×": 329, "à": 330, "á": 331, "ç": 332, "è": 333, "é": 334, "í": 335, "ǔ": 336, "ɓ": 337, "ə": 338, "ก": 339, "ท": 340, "น": 341, "ภ": 342, "ย": 343, "ร": 344, "ษ": 345, "า": 346, "ิ": 347, "เ": 348, "ไ": 349, "๒": 350, "๕": 351, "ລ": 352, "ວ": 353, "ສ": 354, "າ": 355, "ក": 356, "ខ": 357, "គ": 358, "ឃ": 359, "ង": 360, "ច": 361, "ឆ": 362, "ជ": 363, "ឈ": 364, "ញ": 365, "ដ": 366, "ឋ": 367, "ឌ": 368, "ណ": 369, "ត": 370, "ថ": 371, "ទ": 372, "ធ": 373, "ន": 374, "ប": 375, "ផ": 376, "ព": 377, "ភ": 378, "ម": 379, "យ": 380, "រ": 381, "ល": 382, "វ": 383, "ស": 384, "ហ": 385, "ឡ": 386, "អ": 387, "ឧ": 388, "ឫ": 389, "ឬ": 390, "ឯ": 391, "ឲ": 392, "ា": 393, "ិ": 394, "ី": 395, "ឹ": 396, "ឺ": 397, "ុ": 398, "ូ": 399, "ួ": 400, "ើ": 401, "ឿ": 402, "ៀ": 403, "េ": 404, "ែ": 405, "ៃ": 406, "ោ": 407, "ៅ": 408, "ំ": 409, "ះ": 410, "ៈ": 411, "៉": 412, "៊": 413, "់": 414, "៌": 415, "៍": 416, "៏": 417, "័": 418, "្": 419, "។": 420, "៕": 421, "៖": 422, "ៗ": 423, "៘": 424, "៛": 425, "‑": 426, "–": 427, "—": 428, "‘": 429, "’": 430, "“": 431, "”": 432, "•": 433, "…": 434, "‹": 435, "›": 436, "→": 437, "−": 438, "●": 439, "�": 440, "😀": 441, "😛": 442, "🙂": 443 }, "unk_token": "" } }