{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<|im_start|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<|im_end|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "<|userprompt|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 4,
"content": "<|endofuserprompt|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 5,
"content": "<|response|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 6,
"content": "<|endofresponse|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 7,
"content": "<|system|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 8,
"content": "<|endofsystem|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 9,
"content": "<|observation|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 10,
"content": "<|endofobservation|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 11,
"content": "<|execution|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 12,
"content": "<|endofexecution|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 13,
"content": "<|reject-unknown|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 14,
"content": "<|sec-cot|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 15,
"content": "<|sec-end-cot|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 16,
"content": "<|im_end>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 17,
"content": "<|imgpad|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 18,
"content": "<|img|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 19,
"content": "<|endofimg|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 20,
"content": "<|systemprompt|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 21,
"content": "<|endofsystemprompt|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 22,
"content": "<|user|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 23,
"content": "<|endofuser|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 24,
"content": "<|assistant|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 25,
"content": "<|endofassistant|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 26,
"content": "<|box_start|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 27,
"content": "<|box_end|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 28,
"content": "<|ref_start|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 29,
"content": "<|ref_end|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 30,
"content": "[SEP]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 31,
"content": "<|pic|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 32,
"content": "<|text|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 33,
"content": "<|pictotext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 34,
"content": "<|slice|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 35,
"content": "<|endofslice|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 36,
"content": "<|imgrowend|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 37,
"content": "<|video_pad|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 38,
"content": "<|polygon_start|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 39,
"content": "<|polygon_end|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 40,
"content": "<|task:image en-captioning|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 41,
"content": "<|task:image zh-captioning|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 42,
"content": "<|task:object detection|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 43,
"content": "<|task:ocr|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 44,
"content": "<|task:image en-dense-captioning|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 45,
"content": "<|task:image zh-dense-captioning|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 46,
"content": "<|task:image en-concept|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 47,
"content": "<|task:image zh-concept|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 48,
"content": "<|task:image grounding|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 49,
"content": "<|task:image orientation|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 50,
"content": "<|task:ocr with format latex|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 51,
"content": "<|task:ocr with format json|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 52,
"content": "<|task:ocr of molecule with format molgrapher|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 53,
"content": "<|task:ocr of staff with format ekern|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 54,
"content": "<|task:ocr of chart with format matplotlib|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 55,
"content": "<|task:ocr of chart with format mermaid|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 56,
"content": "<|task:ocr-detection and recognition|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 57,
"content": "<|task:doc layout with format json|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 58,
"content": "<|task:image captioning|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 59,
"content": "<|task:table captioning|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 60,
"content": "<|task:ocr with format image captioning|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 61,
"content": "<|task:ocr with nodes,lines and circle|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 62,
"content": "<|task:ocr with format html|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 63,
"content": "<|task:video captioning|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 64,
"content": "<|url|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 65,
"content": "<|endofurl|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 66,
"content": "<|pdfmeta|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 67,
"content": "<|endofpdfmeta|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 68,
"content": "<|alttext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 69,
"content": "<|discarddensecaption|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 70,
"content": "<|endofalttext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 71,
"content": "<|to_think|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 72,
"content": "<|audio_gen_start|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 73,
"content": "<|audio_gen_end|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 74,
"content": "<|audio_gen_pad|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 75,
"content": "<|audio_comp_start|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 76,
"content": "<|audio_comp_end|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 77,
"content": "<|audio_comp_pad|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 78,
"content": "",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
},
{
"id": 79,
"content": "",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
},
{
"id": 80,
"content": "",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
},
{
"id": 81,
"content": "",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
},
{
"id": 82,
"content": "",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
},
{
"id": 83,
"content": "",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
},
{
"id": 84,
"content": "",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
}
],
"normalizer": {
"type": "NFC"
},
"pre_tokenizer": {
"type": "Sequence",
"pretokenizers": [
{
"type": "Split",
"pattern": {
"Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
},
"behavior": "Isolated",
"invert": false
},
{
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": false,
"use_regex": false
}
]
},
"post_processor": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": false,
"use_regex": false
},
"decoder": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": false,
"use_regex": false
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": "",
"end_of_word_suffix": "",
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"<|endoftext|>": 0,
"<|im_start|>": 1,
"<|im_end|>": 2,
"<|userprompt|>": 3,
"<|endofuserprompt|>": 4,
"<|response|>": 5,
"<|endofresponse|>": 6,
"<|system|>": 7,
"<|endofsystem|>": 8,
"<|observation|>": 9,
"<|endofobservation|>": 10,
"<|execution|>": 11,
"<|endofexecution|>": 12,
"<|reject-unknown|>": 13,
"<|sec-cot|>": 14,
"<|sec-end-cot|>": 15,
"<|im_end>": 16,
"<|imgpad|>": 17,
"<|img|>": 18,
"<|endofimg|>": 19,
"<|systemprompt|>": 20,
"<|endofsystemprompt|>": 21,
"<|user|>": 22,
"<|endofuser|>": 23,
"<|assistant|>": 24,
"<|endofassistant|>": 25,
"<|box_start|>": 26,
"<|box_end|>": 27,
"<|ref_start|>": 28,
"<|ref_end|>": 29,
"[SEP]": 30,
"<|pic|>": 31,
"<|text|>": 32,
"<|pictotext|>": 33,
"<|slice|>": 34,
"<|endofslice|>": 35,
"<|imgrowend|>": 36,
"<|video_pad|>": 37,
"<|polygon_start|>": 38,
"<|polygon_end|>": 39,
"<|task:image en-captioning|>": 40,
"<|task:image zh-captioning|>": 41,
"<|task:object detection|>": 42,
"<|task:ocr|>": 43,
"<|task:image en-dense-captioning|>": 44,
"<|task:image zh-dense-captioning|>": 45,
"<|task:image en-concept|>": 46,
"<|task:image zh-concept|>": 47,
"<|task:image grounding|>": 48,
"<|task:image orientation|>": 49,
"<|task:ocr with format latex|>": 50,
"<|task:ocr with format json|>": 51,
"<|task:ocr of molecule with format molgrapher|>": 52,
"<|task:ocr of staff with format ekern|>": 53,
"<|task:ocr of chart with format matplotlib|>": 54,
"<|task:ocr of chart with format mermaid|>": 55,
"<|task:ocr-detection and recognition|>": 56,
"<|task:doc layout with format json|>": 57,
"<|task:image captioning|>": 58,
"<|task:table captioning|>": 59,
"<|task:ocr with format image captioning|>": 60,
"<|task:ocr with nodes,lines and circle|>": 61,
"<|task:ocr with format html|>": 62,
"<|task:video captioning|>": 63,
"<|url|>": 64,
"<|endofurl|>": 65,
"<|pdfmeta|>": 66,
"<|endofpdfmeta|>": 67,
"<|alttext|>": 68,
"<|discarddensecaption|>": 69,
"<|endofalttext|>": 70,
"<|to_think|>": 71,
"<|audio_gen_start|>": 72,
"<|audio_gen_end|>": 73,
"<|audio_gen_pad|>": 74,
"<|audio_comp_start|>": 75,
"<|audio_comp_end|>": 76,
"<|audio_comp_pad|>": 77,
"": 78,
"": 79,
"": 80,
"": 81,
"": 82,
"": 83,
"": 84,
"!": 85,
"\"": 86,
"#": 87,
"$": 88,
"%": 89,
"&": 90,
"'": 91,
"(": 92,
")": 93,
"*": 94,
"+": 95,
",": 96,
"-": 97,
".": 98,
"/": 99,
"0": 100,
"1": 101,
"2": 102,
"3": 103,
"4": 104,
"5": 105,
"6": 106,
"7": 107,
"8": 108,
"9": 109,
":": 110,
";": 111,
"<": 112,
"=": 113,
">": 114,
"?": 115,
"@": 116,
"A": 117,
"B": 118,
"C": 119,
"D": 120,
"E": 121,
"F": 122,
"G": 123,
"H": 124,
"I": 125,
"J": 126,
"K": 127,
"L": 128,
"M": 129,
"N": 130,
"O": 131,
"P": 132,
"Q": 133,
"R": 134,
"S": 135,
"T": 136,
"U": 137,
"V": 138,
"W": 139,
"X": 140,
"Y": 141,
"Z": 142,
"[": 143,
"\\": 144,
"]": 145,
"^": 146,
"_": 147,
"`": 148,
"a": 149,
"b": 150,
"c": 151,
"d": 152,
"e": 153,
"f": 154,
"g": 155,
"h": 156,
"i": 157,
"j": 158,
"k": 159,
"l": 160,
"m": 161,
"n": 162,
"o": 163,
"p": 164,
"q": 165,
"r": 166,
"s": 167,
"t": 168,
"u": 169,
"v": 170,
"w": 171,
"x": 172,
"y": 173,
"z": 174,
"{": 175,
"|": 176,
"}": 177,
"~": 178,
"¡": 179,
"¢": 180,
"£": 181,
"¤": 182,
"¥": 183,
"¦": 184,
"§": 185,
"¨": 186,
"©": 187,
"ª": 188,
"«": 189,
"¬": 190,
"®": 191,
"¯": 192,
"°": 193,
"±": 194,
"²": 195,
"³": 196,
"´": 197,
"µ": 198,
"¶": 199,
"·": 200,
"¸": 201,
"¹": 202,
"º": 203,
"»": 204,
"¼": 205,
"½": 206,
"¾": 207,
"¿": 208,
"À": 209,
"Á": 210,
"Â": 211,
"Ã": 212,
"Ä": 213,
"Å": 214,
"Æ": 215,
"Ç": 216,
"È": 217,
"É": 218,
"Ê": 219,
"Ë": 220,
"Ì": 221,
"Í": 222,
"Î": 223,
"Ï": 224,
"Ð": 225,
"Ñ": 226,
"Ò": 227,
"Ó": 228,
"Ô": 229,
"Õ": 230,
"Ö": 231,
"×": 232,
"Ø": 233,
"Ù": 234,
"Ú": 235,
"Û": 236,
"Ü": 237,
"Ý": 238,
"Þ": 239,
"ß": 240,
"à": 241,
"á": 242,
"â": 243,
"ã": 244,
"ä": 245,
"å": 246,
"æ": 247,
"ç": 248,
"è": 249,
"é": 250,
"ê": 251,
"ë": 252,
"ì": 253,
"í": 254,
"î": 255,
"ï": 256,
"ð": 257,
"ñ": 258,
"ò": 259,
"ó": 260,
"ô": 261,
"õ": 262,
"ö": 263,
"÷": 264,
"ø": 265,
"ù": 266,
"ú": 267,
"û": 268,
"ü": 269,
"ý": 270,
"þ": 271,
"ÿ": 272,
"Ā": 273,
"ā": 274,
"Ă": 275,
"ă": 276,
"Ą": 277,
"ą": 278,
"Ć": 279,
"ć": 280,
"Ĉ": 281,
"ĉ": 282,
"Ċ": 283,
"ċ": 284,
"Č": 285,
"č": 286,
"Ď": 287,
"ď": 288,
"Đ": 289,
"đ": 290,
"Ē": 291,
"ē": 292,
"Ĕ": 293,
"ĕ": 294,
"Ė": 295,
"ė": 296,
"Ę": 297,
"ę": 298,
"Ě": 299,
"ě": 300,
"Ĝ": 301,
"ĝ": 302,
"Ğ": 303,
"ğ": 304,
"Ġ": 305,
"ġ": 306,
"Ģ": 307,
"ģ": 308,
"Ĥ": 309,
"ĥ": 310,
"Ħ": 311,
"ħ": 312,
"Ĩ": 313,
"ĩ": 314,
"Ī": 315,
"ī": 316,
"Ĭ": 317,
"ĭ": 318,
"Į": 319,
"į": 320,
"İ": 321,
"ı": 322,
"IJ": 323,
"ij": 324,
"Ĵ": 325,
"ĵ": 326,
"Ķ": 327,
"ķ": 328,
"ĸ": 329,
"Ĺ": 330,
"ĺ": 331,
"Ļ": 332,
"ļ": 333,
"Ľ": 334,
"ľ": 335,
"Ŀ": 336,
"ŀ": 337,
"Ł": 338,
"ł": 339,
"Ń": 340
},
"merges": []
}
}