{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "<|im_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "<|im_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "<|userprompt|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "<|endofuserprompt|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 5, "content": "<|response|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 6, "content": "<|endofresponse|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 7, "content": "<|system|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 8, "content": "<|endofsystem|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 9, "content": "<|observation|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 10, "content": "<|endofobservation|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 11, "content": "<|execution|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 12, "content": "<|endofexecution|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 13, "content": "<|reject-unknown|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 14, "content": "<|sec-cot|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 15, "content": "<|sec-end-cot|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 16, "content": "<|im_end>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 17, "content": "<|imgpad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 18, "content": "<|img|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 19, "content": "<|endofimg|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 20, "content": "<|systemprompt|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 21, "content": "<|endofsystemprompt|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 22, "content": "<|user|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 23, "content": "<|endofuser|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 24, "content": "<|assistant|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 25, "content": "<|endofassistant|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 26, "content": "<|box_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 27, "content": "<|box_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 28, "content": "<|ref_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 29, "content": "<|ref_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 30, "content": "[SEP]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 31, "content": "<|pic|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 32, "content": "<|text|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 33, "content": "<|pictotext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 34, "content": "<|slice|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 35, "content": "<|endofslice|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 36, "content": "<|imgrowend|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 37, "content": "<|video_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 38, "content": "<|polygon_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 39, "content": "<|polygon_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 40, "content": "<|task:image en-captioning|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 41, "content": "<|task:image zh-captioning|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 42, "content": "<|task:object detection|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 43, "content": "<|task:ocr|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 44, "content": "<|task:image en-dense-captioning|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 45, "content": "<|task:image zh-dense-captioning|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 46, "content": "<|task:image en-concept|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 47, "content": "<|task:image zh-concept|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 48, "content": "<|task:image grounding|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 49, "content": "<|task:image orientation|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 50, "content": "<|task:ocr with format latex|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 51, "content": "<|task:ocr with format json|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 52, "content": "<|task:ocr of molecule with format molgrapher|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 53, "content": "<|task:ocr of staff with format ekern|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 54, "content": "<|task:ocr of chart with format matplotlib|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 55, "content": "<|task:ocr of chart with format mermaid|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 56, "content": "<|task:ocr-detection and recognition|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 57, "content": "<|task:doc layout with format json|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 58, "content": "<|task:image captioning|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 59, "content": "<|task:table captioning|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 60, "content": "<|task:ocr with format image captioning|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 61, "content": "<|task:ocr with nodes,lines and circle|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 62, "content": "<|task:ocr with format html|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 63, "content": "<|task:video captioning|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 64, "content": "<|url|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 65, "content": "<|endofurl|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 66, "content": "<|pdfmeta|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 67, "content": "<|endofpdfmeta|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 68, "content": "<|alttext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 69, "content": "<|discarddensecaption|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 70, "content": "<|endofalttext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 71, "content": "<|to_think|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 72, "content": "<|audio_gen_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 73, "content": "<|audio_gen_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 74, "content": "<|audio_gen_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 75, "content": "<|audio_comp_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 76, "content": "<|audio_comp_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 77, "content": "<|audio_comp_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 78, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 79, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 80, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 81, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 82, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 83, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 84, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false } ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false } ] }, "post_processor": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false }, "decoder": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false }, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": "", "end_of_word_suffix": "", "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "<|endoftext|>": 0, "<|im_start|>": 1, "<|im_end|>": 2, "<|userprompt|>": 3, "<|endofuserprompt|>": 4, "<|response|>": 5, "<|endofresponse|>": 6, "<|system|>": 7, "<|endofsystem|>": 8, "<|observation|>": 9, "<|endofobservation|>": 10, "<|execution|>": 11, "<|endofexecution|>": 12, "<|reject-unknown|>": 13, "<|sec-cot|>": 14, "<|sec-end-cot|>": 15, "<|im_end>": 16, "<|imgpad|>": 17, "<|img|>": 18, "<|endofimg|>": 19, "<|systemprompt|>": 20, "<|endofsystemprompt|>": 21, "<|user|>": 22, "<|endofuser|>": 23, "<|assistant|>": 24, "<|endofassistant|>": 25, "<|box_start|>": 26, "<|box_end|>": 27, "<|ref_start|>": 28, "<|ref_end|>": 29, "[SEP]": 30, "<|pic|>": 31, "<|text|>": 32, "<|pictotext|>": 33, "<|slice|>": 34, "<|endofslice|>": 35, "<|imgrowend|>": 36, "<|video_pad|>": 37, "<|polygon_start|>": 38, "<|polygon_end|>": 39, "<|task:image en-captioning|>": 40, "<|task:image zh-captioning|>": 41, "<|task:object detection|>": 42, "<|task:ocr|>": 43, "<|task:image en-dense-captioning|>": 44, "<|task:image zh-dense-captioning|>": 45, "<|task:image en-concept|>": 46, "<|task:image zh-concept|>": 47, "<|task:image grounding|>": 48, "<|task:image orientation|>": 49, "<|task:ocr with format latex|>": 50, "<|task:ocr with format json|>": 51, "<|task:ocr of molecule with format molgrapher|>": 52, "<|task:ocr of staff with format ekern|>": 53, "<|task:ocr of chart with format matplotlib|>": 54, "<|task:ocr of chart with format mermaid|>": 55, "<|task:ocr-detection and recognition|>": 56, "<|task:doc layout with format json|>": 57, "<|task:image captioning|>": 58, "<|task:table captioning|>": 59, "<|task:ocr with format image captioning|>": 60, "<|task:ocr with nodes,lines and circle|>": 61, "<|task:ocr with format html|>": 62, "<|task:video captioning|>": 63, "<|url|>": 64, "<|endofurl|>": 65, "<|pdfmeta|>": 66, "<|endofpdfmeta|>": 67, "<|alttext|>": 68, "<|discarddensecaption|>": 69, "<|endofalttext|>": 70, "<|to_think|>": 71, "<|audio_gen_start|>": 72, "<|audio_gen_end|>": 73, "<|audio_gen_pad|>": 74, "<|audio_comp_start|>": 75, "<|audio_comp_end|>": 76, "<|audio_comp_pad|>": 77, "": 78, "": 79, "": 80, "": 81, "": 82, "": 83, "": 84, "!": 85, "\"": 86, "#": 87, "$": 88, "%": 89, "&": 90, "'": 91, "(": 92, ")": 93, "*": 94, "+": 95, ",": 96, "-": 97, ".": 98, "/": 99, "0": 100, "1": 101, "2": 102, "3": 103, "4": 104, "5": 105, "6": 106, "7": 107, "8": 108, "9": 109, ":": 110, ";": 111, "<": 112, "=": 113, ">": 114, "?": 115, "@": 116, "A": 117, "B": 118, "C": 119, "D": 120, "E": 121, "F": 122, "G": 123, "H": 124, "I": 125, "J": 126, "K": 127, "L": 128, "M": 129, "N": 130, "O": 131, "P": 132, "Q": 133, "R": 134, "S": 135, "T": 136, "U": 137, "V": 138, "W": 139, "X": 140, "Y": 141, "Z": 142, "[": 143, "\\": 144, "]": 145, "^": 146, "_": 147, "`": 148, "a": 149, "b": 150, "c": 151, "d": 152, "e": 153, "f": 154, "g": 155, "h": 156, "i": 157, "j": 158, "k": 159, "l": 160, "m": 161, "n": 162, "o": 163, "p": 164, "q": 165, "r": 166, "s": 167, "t": 168, "u": 169, "v": 170, "w": 171, "x": 172, "y": 173, "z": 174, "{": 175, "|": 176, "}": 177, "~": 178, "¡": 179, "¢": 180, "£": 181, "¤": 182, "¥": 183, "¦": 184, "§": 185, "¨": 186, "©": 187, "ª": 188, "«": 189, "¬": 190, "®": 191, "¯": 192, "°": 193, "±": 194, "²": 195, "³": 196, "´": 197, "µ": 198, "¶": 199, "·": 200, "¸": 201, "¹": 202, "º": 203, "»": 204, "¼": 205, "½": 206, "¾": 207, "¿": 208, "À": 209, "Á": 210, "Â": 211, "Ã": 212, "Ä": 213, "Å": 214, "Æ": 215, "Ç": 216, "È": 217, "É": 218, "Ê": 219, "Ë": 220, "Ì": 221, "Í": 222, "Î": 223, "Ï": 224, "Ð": 225, "Ñ": 226, "Ò": 227, "Ó": 228, "Ô": 229, "Õ": 230, "Ö": 231, "×": 232, "Ø": 233, "Ù": 234, "Ú": 235, "Û": 236, "Ü": 237, "Ý": 238, "Þ": 239, "ß": 240, "à": 241, "á": 242, "â": 243, "ã": 244, "ä": 245, "å": 246, "æ": 247, "ç": 248, "è": 249, "é": 250, "ê": 251, "ë": 252, "ì": 253, "í": 254, "î": 255, "ï": 256, "ð": 257, "ñ": 258, "ò": 259, "ó": 260, "ô": 261, "õ": 262, "ö": 263, "÷": 264, "ø": 265, "ù": 266, "ú": 267, "û": 268, "ü": 269, "ý": 270, "þ": 271, "ÿ": 272, "Ā": 273, "ā": 274, "Ă": 275, "ă": 276, "Ą": 277, "ą": 278, "Ć": 279, "ć": 280, "Ĉ": 281, "ĉ": 282, "Ċ": 283, "ċ": 284, "Č": 285, "č": 286, "Ď": 287, "ď": 288, "Đ": 289, "đ": 290, "Ē": 291, "ē": 292, "Ĕ": 293, "ĕ": 294, "Ė": 295, "ė": 296, "Ę": 297, "ę": 298, "Ě": 299, "ě": 300, "Ĝ": 301, "ĝ": 302, "Ğ": 303, "ğ": 304, "Ġ": 305, "ġ": 306, "Ģ": 307, "ģ": 308, "Ĥ": 309, "ĥ": 310, "Ħ": 311, "ħ": 312, "Ĩ": 313, "ĩ": 314, "Ī": 315, "ī": 316, "Ĭ": 317, "ĭ": 318, "Į": 319, "į": 320, "İ": 321, "ı": 322, "IJ": 323, "ij": 324, "Ĵ": 325, "ĵ": 326, "Ķ": 327, "ķ": 328, "ĸ": 329, "Ĺ": 330, "ĺ": 331, "Ļ": 332, "ļ": 333, "Ľ": 334, "ľ": 335, "Ŀ": 336, "ŀ": 337, "Ł": 338, "ł": 339, "Ń": 340 }, "merges": [] } }