Add tiny processor fixture for testing

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+'}}{% if message['content'] is string %}{{ message['content'] }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' %}{{ '<IMG_CONTEXT>
+' }}{% elif content['type'] == 'video' %}{{ '<video>
+' }}{% elif content['type'] == 'text' %}{{ content['text'] }}{% endif %}{% endfor %}{% endif %}{{'<|im_end|>
+'}}{% endfor %}{% if add_generation_prompt %}{{'<|im_start|>assistant
+' }}{% endif %}

config.json

@@ -0,0 +1,80 @@
+{
+  "architectures": [
+    "InternVLForConditionalGeneration"
+  ],
+  "downsample_ratio": 0.5,
+  "image_seq_length": 256,
+  "image_token_id": 151667,
+  "model_type": "internvl",
+  "projector_hidden_act": "gelu",
+  "text_config": {
+    "architectures": [
+      "Qwen2ForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bos_token_id": 151643,
+    "eos_token_id": 151645,
+    "hidden_act": "silu",
+    "hidden_size": 896,
+    "initializer_range": 0.02,
+    "intermediate_size": 4864,
+    "max_position_embeddings": 32768,
+    "max_window_layers": 70,
+    "model_type": "qwen2",
+    "num_attention_heads": 14,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 2,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": {
+      "factor": 2.0,
+      "rope_type": "dynamic",
+      "type": "dynamic"
+    },
+    "rope_theta": 1000000.0,
+    "sliding_window": null,
+    "torch_dtype": "bfloat16",
+    "use_cache": true,
+    "use_sliding_window": false,
+    "vocab_size": 151674
+  },
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.52.0.dev0",
+  "vision_config": {
+    "architectures": [
+      "InternVisionModel"
+    ],
+    "attention_bias": true,
+    "attention_dropout": 0.0,
+    "dropout": 0.0,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.0,
+    "hidden_size": 1024,
+    "image_size": [
+      448,
+      448
+    ],
+    "initializer_factor": 0.1,
+    "initializer_range": 1e-10,
+    "intermediate_size": 4096,
+    "layer_norm_eps": 1e-06,
+    "layer_scale_init_value": 0.1,
+    "model_type": "internvl_vision",
+    "norm_type": "layer_norm",
+    "num_attention_heads": 16,
+    "num_channels": 3,
+    "num_hidden_layers": 24,
+    "patch_size": [
+      14,
+      14
+    ],
+    "projection_dropout": 0.0,
+    "torch_dtype": "bfloat16",
+    "use_absolute_position_embeddings": true,
+    "use_mask_token": false,
+    "use_mean_pooling": true,
+    "use_qk_norm": false
+  },
+  "vision_feature_layer": -1,
+  "vision_feature_select_strategy": "default",
+  "vocab_size": 500
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "transformers_version": "4.52.0.dev0"
+}

preprocessor_config.json

@@ -0,0 +1,34 @@
+{
+  "crop_size": null,
+  "crop_to_patches": false,
+  "data_format": "channels_first",
+  "default_to_square": true,
+  "device": null,
+  "do_center_crop": null,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.485,
+    0.456,
+    0.406
+  ],
+  "image_processor_type": "GotOcr2ImageProcessorFast",
+  "image_std": [
+    0.229,
+    0.224,
+    0.225
+  ],
+  "input_data_format": null,
+  "max_patches": 12,
+  "min_patches": 1,
+  "processor_class": "InternVLProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "return_tensors": null,
+  "size": {
+    "height": 448,
+    "width": 448
+  }
+}

processor_config.json

@@ -0,0 +1,4 @@
+{
+  "image_seq_length": 256,
+  "processor_class": "InternVLProcessor"
+}

tokenizer.json

@@ -0,0 +1,1819 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 500,
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 501,
+      "content": "<|im_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 502,
+      "content": "<|im_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 503,
+      "content": "<|object_ref_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 504,
+      "content": "<|object_ref_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 505,
+      "content": "<|box_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 506,
+      "content": "<|box_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 507,
+      "content": "<|quad_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 508,
+      "content": "<|quad_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 509,
+      "content": "<|vision_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 510,
+      "content": "<|vision_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 511,
+      "content": "<|vision_pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 512,
+      "content": "<|image_pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 513,
+      "content": "<|video_pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 514,
+      "content": "<tool_call>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 515,
+      "content": "</tool_call>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 516,
+      "content": "<|fim_prefix|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 517,
+      "content": "<|fim_middle|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 518,
+      "content": "<|fim_suffix|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 519,
+      "content": "<|fim_pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 520,
+      "content": "<|repo_name|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 521,
+      "content": "<|file_sep|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 522,
+      "content": "<img>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 523,
+      "content": "</img>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 524,
+      "content": "<IMG_CONTEXT>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 525,
+      "content": "<quad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 526,
+      "content": "</quad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 527,
+      "content": "<ref>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 528,
+      "content": "</ref>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 529,
+      "content": "<box>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 530,
+      "content": "</box>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 531,
+      "content": "<video>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "NFC"
+  },
+  "pre_tokenizer": {
+    "type": "Sequence",
+    "pretokenizers": [
+      {
+        "type": "Split",
+        "pattern": {
+          "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
+        },
+        "behavior": "Isolated",
+        "invert": false
+      },
+      {
+        "type": "ByteLevel",
+        "add_prefix_space": false,
+        "trim_offsets": false,
+        "use_regex": false
+      }
+    ]
+  },
+  "post_processor": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": false,
+    "use_regex": false
+  },
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": false,
+    "use_regex": false
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": "",
+    "end_of_word_suffix": "",
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "!": 0,
+      "\"": 1,
+      "#": 2,
+      "$": 3,
+      "%": 4,
+      "&": 5,
+      "'": 6,
+      "(": 7,
+      ")": 8,
+      "*": 9,
+      "+": 10,
+      ",": 11,
+      "-": 12,
+      ".": 13,
+      "/": 14,
+      "0": 15,
+      "1": 16,
+      "2": 17,
+      "3": 18,
+      "4": 19,
+      "5": 20,
+      "6": 21,
+      "7": 22,
+      "8": 23,
+      "9": 24,
+      ":": 25,
+      ";": 26,
+      "<": 27,
+      "=": 28,
+      ">": 29,
+      "?": 30,
+      "@": 31,
+      "A": 32,
+      "B": 33,
+      "C": 34,
+      "D": 35,
+      "E": 36,
+      "F": 37,
+      "G": 38,
+      "H": 39,
+      "I": 40,
+      "J": 41,
+      "K": 42,
+      "L": 43,
+      "M": 44,
+      "N": 45,
+      "O": 46,
+      "P": 47,
+      "Q": 48,
+      "R": 49,
+      "S": 50,
+      "T": 51,
+      "U": 52,
+      "V": 53,
+      "W": 54,
+      "X": 55,
+      "Y": 56,
+      "Z": 57,
+      "[": 58,
+      "\\": 59,
+      "]": 60,
+      "^": 61,
+      "_": 62,
+      "`": 63,
+      "a": 64,
+      "b": 65,
+      "c": 66,
+      "d": 67,
+      "e": 68,
+      "f": 69,
+      "g": 70,
+      "h": 71,
+      "i": 72,
+      "j": 73,
+      "k": 74,
+      "l": 75,
+      "m": 76,
+      "n": 77,
+      "o": 78,
+      "p": 79,
+      "q": 80,
+      "r": 81,
+      "s": 82,
+      "t": 83,
+      "u": 84,
+      "v": 85,
+      "w": 86,
+      "x": 87,
+      "y": 88,
+      "z": 89,
+      "{": 90,
+      "|": 91,
+      "}": 92,
+      "~": 93,
+      "¡": 94,
+      "¢": 95,
+      "£": 96,
+      "¤": 97,
+      "¥": 98,
+      "¦": 99,
+      "§": 100,
+      "¨": 101,
+      "©": 102,
+      "ª": 103,
+      "«": 104,
+      "¬": 105,
+      "®": 106,
+      "¯": 107,
+      "°": 108,
+      "±": 109,
+      "²": 110,
+      "³": 111,
+      "´": 112,
+      "µ": 113,
+      "¶": 114,
+      "·": 115,
+      "¸": 116,
+      "¹": 117,
+      "º": 118,
+      "»": 119,
+      "¼": 120,
+      "½": 121,
+      "¾": 122,
+      "¿": 123,
+      "À": 124,
+      "Á": 125,
+      "Â": 126,
+      "Ã": 127,
+      "Ä": 128,
+      "Å": 129,
+      "Æ": 130,
+      "Ç": 131,
+      "È": 132,
+      "É": 133,
+      "Ê": 134,
+      "Ë": 135,
+      "Ì": 136,
+      "Í": 137,
+      "Î": 138,
+      "Ï": 139,
+      "Ð": 140,
+      "Ñ": 141,
+      "Ò": 142,
+      "Ó": 143,
+      "Ô": 144,
+      "Õ": 145,
+      "Ö": 146,
+      "×": 147,
+      "Ø": 148,
+      "Ù": 149,
+      "Ú": 150,
+      "Û": 151,
+      "Ü": 152,
+      "Ý": 153,
+      "Þ": 154,
+      "ß": 155,
+      "à": 156,
+      "á": 157,
+      "â": 158,
+      "ã": 159,
+      "ä": 160,
+      "å": 161,
+      "æ": 162,
+      "ç": 163,
+      "è": 164,
+      "é": 165,
+      "ê": 166,
+      "ë": 167,
+      "ì": 168,
+      "í": 169,
+      "î": 170,
+      "ï": 171,
+      "ð": 172,
+      "ñ": 173,
+      "ò": 174,
+      "ó": 175,
+      "ô": 176,
+      "õ": 177,
+      "ö": 178,
+      "÷": 179,
+      "ø": 180,
+      "ù": 181,
+      "ú": 182,
+      "û": 183,
+      "ü": 184,
+      "ý": 185,
+      "þ": 186,
+      "ÿ": 187,
+      "Ā": 188,
+      "ā": 189,
+      "Ă": 190,
+      "ă": 191,
+      "Ą": 192,
+      "ą": 193,
+      "Ć": 194,
+      "ć": 195,
+      "Ĉ": 196,
+      "ĉ": 197,
+      "Ċ": 198,
+      "ċ": 199,
+      "Č": 200,
+      "č": 201,
+      "Ď": 202,
+      "ď": 203,
+      "Đ": 204,
+      "đ": 205,
+      "Ē": 206,
+      "ē": 207,
+      "Ĕ": 208,
+      "ĕ": 209,
+      "Ė": 210,
+      "ė": 211,
+      "Ę": 212,
+      "ę": 213,
+      "Ě": 214,
+      "ě": 215,
+      "Ĝ": 216,
+      "ĝ": 217,
+      "Ğ": 218,
+      "ğ": 219,
+      "Ġ": 220,
+      "ġ": 221,
+      "Ģ": 222,
+      "ģ": 223,
+      "Ĥ": 224,
+      "ĥ": 225,
+      "Ħ": 226,
+      "ħ": 227,
+      "Ĩ": 228,
+      "ĩ": 229,
+      "Ī": 230,
+      "ī": 231,
+      "Ĭ": 232,
+      "ĭ": 233,
+      "Į": 234,
+      "į": 235,
+      "İ": 236,
+      "ı": 237,
+      "IJ": 238,
+      "ij": 239,
+      "Ĵ": 240,
+      "ĵ": 241,
+      "Ķ": 242,
+      "ķ": 243,
+      "ĸ": 244,
+      "Ĺ": 245,
+      "ĺ": 246,
+      "Ļ": 247,
+      "ļ": 248,
+      "Ľ": 249,
+      "ľ": 250,
+      "Ŀ": 251,
+      "ŀ": 252,
+      "Ł": 253,
+      "ł": 254,
+      "Ń": 255,
+      "ĠĠ": 256,
+      "ĠĠĠĠ": 257,
+      "in": 258,
+      "Ġt": 259,
+      "ĠĠĠĠĠĠĠĠ": 260,
+      "er": 261,
+      "ĠĠĠ": 262,
+      "on": 263,
+      "Ġa": 264,
+      "re": 265,
+      "at": 266,
+      "st": 267,
+      "en": 268,
+      "or": 269,
+      "Ġth": 270,
+      "ĊĊ": 271,
+      "Ġc": 272,
+      "le": 273,
+      "Ġs": 274,
+      "it": 275,
+      "an": 276,
+      "ar": 277,
+      "al": 278,
+      "Ġthe": 279,
+      ";Ċ": 280,
+      "Ġp": 281,
+      "Ġf": 282,
+      "ou": 283,
+      "Ġ=": 284,
+      "is": 285,
+      "ĠĠĠĠĠĠĠ": 286,
+      "ing": 287,
+      "es": 288,
+      "Ġw": 289,
+      "ion": 290,
+      "ed": 291,
+      "ic": 292,
+      "Ġb": 293,
+      "Ġd": 294,
+      "et": 295,
+      "Ġm": 296,
+      "Ġo": 297,
+      "ĉĉ": 298,
+      "ro": 299,
+      "as": 300,
+      "el": 301,
+      "ct": 302,
+      "nd": 303,
+      "Ġin": 304,
+      "Ġh": 305,
+      "ent": 306,
+      "id": 307,
+      "Ġn": 308,
+      "am": 309,
+      "ĠĠĠĠĠĠĠĠĠĠĠ": 310,
+      "Ġto": 311,
+      "Ġre": 312,
+      "--": 313,
+      "Ġ{": 314,
+      "Ġof": 315,
+      "om": 316,
+      ");Ċ": 317,
+      "im": 318,
+      "čĊ": 319,
+      "Ġ(": 320,
+      "il": 321,
+      "//": 322,
+      "Ġand": 323,
+      "ur": 324,
+      "se": 325,
+      "Ġl": 326,
+      "ex": 327,
+      "ĠS": 328,
+      "ad": 329,
+      "Ġ\"": 330,
+      "ch": 331,
+      "ut": 332,
+      "if": 333,
+      "**": 334,
+      "Ġ}": 335,
+      "em": 336,
+      "ol": 337,
+      "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 338,
+      "th": 339,
+      ")Ċ": 340,
+      "Ġ{Ċ": 341,
+      "Ġg": 342,
+      "ig": 343,
+      "iv": 344,
+      ",Ċ": 345,
+      "ce": 346,
+      "od": 347,
+      "Ġv": 348,
+      "ate": 349,
+      "ĠT": 350,
+      "ag": 351,
+      "ay": 352,
+      "Ġ*": 353,
+      "ot": 354,
+      "us": 355,
+      "ĠC": 356,
+      "Ġst": 357,
+      "ĠI": 358,
+      "un": 359,
+      "ul": 360,
+      "ue": 361,
+      "ĠA": 362,
+      "ow": 363,
+      "Ġ'": 364,
+      "ew": 365,
+      "Ġ<": 366,
+      "ation": 367,
+      "()": 368,
+      "Ġfor": 369,
+      "ab": 370,
+      "ort": 371,
+      "um": 372,
+      "ame": 373,
+      "Ġis": 374,
+      "pe": 375,
+      "tr": 376,
+      "ck": 377,
+      "âĢ": 378,
+      "Ġy": 379,
+      "ist": 380,
+      "----": 381,
+      ".ĊĊ": 382,
+      "he": 383,
+      "Ġe": 384,
+      "lo": 385,
+      "ĠM": 386,
+      "Ġbe": 387,
+      "ers": 388,
+      "Ġon": 389,
+      "Ġcon": 390,
+      "ap": 391,
+      "ub": 392,
+      "ĠP": 393,
+      "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 394,
+      "ass": 395,
+      "int": 396,
+      ">Ċ": 397,
+      "ly": 398,
+      "urn": 399,
+      "Ġ$": 400,
+      ";ĊĊ": 401,
+      "av": 402,
+      "port": 403,
+      "ir": 404,
+      "->": 405,
+      "nt": 406,
+      "ction": 407,
+      "end": 408,
+      "Ġde": 409,
+      "ith": 410,
+      "out": 411,
+      "turn": 412,
+      "our": 413,
+      "ĠĠĠĠĠ": 414,
+      "lic": 415,
+      "res": 416,
+      "pt": 417,
+      "==": 418,
+      "Ġthis": 419,
+      "Ġwh": 420,
+      "Ġif": 421,
+      "ĠD": 422,
+      "ver": 423,
+      "age": 424,
+      "ĠB": 425,
+      "ht": 426,
+      "ext": 427,
+      "=\"": 428,
+      "Ġthat": 429,
+      "****": 430,
+      "ĠR": 431,
+      "Ġit": 432,
+      "ess": 433,
+      "ĠF": 434,
+      "Ġr": 435,
+      "os": 436,
+      "and": 437,
+      "Ġas": 438,
+      "ect": 439,
+      "ke": 440,
+      "rom": 441,
+      "Ġ//": 442,
+      "con": 443,
+      "ĠL": 444,
+      "(\"": 445,
+      "qu": 446,
+      "lass": 447,
+      "Ġwith": 448,
+      "iz": 449,
+      "de": 450,
+      "ĠN": 451,
+      "Ġal": 452,
+      "op": 453,
+      "up": 454,
+      "get": 455,
+      "Ġ}Ċ": 456,
+      "ile": 457,
+      "Ġan": 458,
+      "ata": 459,
+      "ore": 460,
+      "ri": 461,
+      "Ġpro": 462,
+      ";čĊ": 463,
+      "ĉĉĉĉ": 464,
+      "ter": 465,
+      "ain": 466,
+      "ĠW": 467,
+      "ĠE": 468,
+      "Ġcom": 469,
+      "Ġreturn": 470,
+      "art": 471,
+      "ĠH": 472,
+      "ack": 473,
+      "import": 474,
+      "ublic": 475,
+      "Ġor": 476,
+      "est": 477,
+      "ment": 478,
+      "ĠG": 479,
+      "able": 480,
+      "Ġ-": 481,
+      "ine": 482,
+      "ill": 483,
+      "ind": 484,
+      "ere": 485,
+      "::": 486,
+      "ity": 487,
+      "Ġ+": 488,
+      "Ġtr": 489,
+      "elf": 490,
+      "ight": 491,
+      "('": 492,
+      "orm": 493,
+      "ult": 494,
+      "str": 495,
+      "..": 496,
+      "\",": 497,
+      "Ġyou": 498,
+      "ype": 499
+    },
+    "merges": [
+      [
+        "Ġ",
+        "Ġ"
+      ],
+      [
+        "ĠĠ",
+        "ĠĠ"
+      ],
+      [
+        "i",
+        "n"
+      ],
+      [
+        "Ġ",
+        "t"
+      ],
+      [
+        "ĠĠĠĠ",
+        "ĠĠĠĠ"
+      ],
+      [
+        "e",
+        "r"
+      ],
+      [
+        "ĠĠ",
+        "Ġ"
+      ],
+      [
+        "o",
+        "n"
+      ],
+      [
+        "Ġ",
+        "a"
+      ],
+      [
+        "r",
+        "e"
+      ],
+      [
+        "a",
+        "t"
+      ],
+      [
+        "s",
+        "t"
+      ],
+      [
+        "e",
+        "n"
+      ],
+      [
+        "o",
+        "r"
+      ],
+      [
+        "Ġt",
+        "h"
+      ],
+      [
+        "Ċ",
+        "Ċ"
+      ],
+      [
+        "Ġ",
+        "c"
+      ],
+      [
+        "l",
+        "e"
+      ],
+      [
+        "Ġ",
+        "s"
+      ],
+      [
+        "i",
+        "t"
+      ],
+      [
+        "a",
+        "n"
+      ],
+      [
+        "a",
+        "r"
+      ],
+      [
+        "a",
+        "l"
+      ],
+      [
+        "Ġth",
+        "e"
+      ],
+      [
+        ";",
+        "Ċ"
+      ],
+      [
+        "Ġ",
+        "p"
+      ],
+      [
+        "Ġ",
+        "f"
+      ],
+      [
+        "o",
+        "u"
+      ],
+      [
+        "Ġ",
+        "="
+      ],
+      [
+        "i",
+        "s"
+      ],
+      [
+        "ĠĠĠĠ",
+        "ĠĠĠ"
+      ],
+      [
+        "in",
+        "g"
+      ],
+      [
+        "e",
+        "s"
+      ],
+      [
+        "Ġ",
+        "w"
+      ],
+      [
+        "i",
+        "on"
+      ],
+      [
+        "e",
+        "d"
+      ],
+      [
+        "i",
+        "c"
+      ],
+      [
+        "Ġ",
+        "b"
+      ],
+      [
+        "Ġ",
+        "d"
+      ],
+      [
+        "e",
+        "t"
+      ],
+      [
+        "Ġ",
+        "m"
+      ],
+      [
+        "Ġ",
+        "o"
+      ],
+      [
+        "ĉ",
+        "ĉ"
+      ],
+      [
+        "r",
+        "o"
+      ],
+      [
+        "a",
+        "s"
+      ],
+      [
+        "e",
+        "l"
+      ],
+      [
+        "c",
+        "t"
+      ],
+      [
+        "n",
+        "d"
+      ],
+      [
+        "Ġ",
+        "in"
+      ],
+      [
+        "Ġ",
+        "h"
+      ],
+      [
+        "en",
+        "t"
+      ],
+      [
+        "i",
+        "d"
+      ],
+      [
+        "Ġ",
+        "n"
+      ],
+      [
+        "a",
+        "m"
+      ],
+      [
+        "ĠĠĠĠĠĠĠĠ",
+        "ĠĠĠ"
+      ],
+      [
+        "Ġt",
+        "o"
+      ],
+      [
+        "Ġ",
+        "re"
+      ],
+      [
+        "-",
+        "-"
+      ],
+      [
+        "Ġ",
+        "{"
+      ],
+      [
+        "Ġo",
+        "f"
+      ],
+      [
+        "o",
+        "m"
+      ],
+      [
+        ")",
+        ";Ċ"
+      ],
+      [
+        "i",
+        "m"
+      ],
+      [
+        "č",
+        "Ċ"
+      ],
+      [
+        "Ġ",
+        "("
+      ],
+      [
+        "i",
+        "l"
+      ],
+      [
+        "/",
+        "/"
+      ],
+      [
+        "Ġa",
+        "nd"
+      ],
+      [
+        "u",
+        "r"
+      ],
+      [
+        "s",
+        "e"
+      ],
+      [
+        "Ġ",
+        "l"
+      ],
+      [
+        "e",
+        "x"
+      ],
+      [
+        "Ġ",
+        "S"
+      ],
+      [
+        "a",
+        "d"
+      ],
+      [
+        "Ġ",
+        "\""
+      ],
+      [
+        "c",
+        "h"
+      ],
+      [
+        "u",
+        "t"
+      ],
+      [
+        "i",
+        "f"
+      ],
+      [
+        "*",
+        "*"
+      ],
+      [
+        "Ġ",
+        "}"
+      ],
+      [
+        "e",
+        "m"
+      ],
+      [
+        "o",
+        "l"
+      ],
+      [
+        "ĠĠĠĠĠĠĠĠ",
+        "ĠĠĠĠĠĠĠĠ"
+      ],
+      [
+        "t",
+        "h"
+      ],
+      [
+        ")",
+        "Ċ"
+      ],
+      [
+        "Ġ{",
+        "Ċ"
+      ],
+      [
+        "Ġ",
+        "g"
+      ],
+      [
+        "i",
+        "g"
+      ],
+      [
+        "i",
+        "v"
+      ],
+      [
+        ",",
+        "Ċ"
+      ],
+      [
+        "c",
+        "e"
+      ],
+      [
+        "o",
+        "d"
+      ],
+      [
+        "Ġ",
+        "v"
+      ],
+      [
+        "at",
+        "e"
+      ],
+      [
+        "Ġ",
+        "T"
+      ],
+      [
+        "a",
+        "g"
+      ],
+      [
+        "a",
+        "y"
+      ],
+      [
+        "Ġ",
+        "*"
+      ],
+      [
+        "o",
+        "t"
+      ],
+      [
+        "u",
+        "s"
+      ],
+      [
+        "Ġ",
+        "C"
+      ],
+      [
+        "Ġ",
+        "st"
+      ],
+      [
+        "Ġ",
+        "I"
+      ],
+      [
+        "u",
+        "n"
+      ],
+      [
+        "u",
+        "l"
+      ],
+      [
+        "u",
+        "e"
+      ],
+      [
+        "Ġ",
+        "A"
+      ],
+      [
+        "o",
+        "w"
+      ],
+      [
+        "Ġ",
+        "'"
+      ],
+      [
+        "e",
+        "w"
+      ],
+      [
+        "Ġ",
+        "<"
+      ],
+      [
+        "at",
+        "ion"
+      ],
+      [
+        "(",
+        ")"
+      ],
+      [
+        "Ġf",
+        "or"
+      ],
+      [
+        "a",
+        "b"
+      ],
+      [
+        "or",
+        "t"
+      ],
+      [
+        "u",
+        "m"
+      ],
+      [
+        "am",
+        "e"
+      ],
+      [
+        "Ġ",
+        "is"
+      ],
+      [
+        "p",
+        "e"
+      ],
+      [
+        "t",
+        "r"
+      ],
+      [
+        "c",
+        "k"
+      ],
+      [
+        "â",
+        "Ģ"
+      ],
+      [
+        "Ġ",
+        "y"
+      ],
+      [
+        "i",
+        "st"
+      ],
+      [
+        "--",
+        "--"
+      ],
+      [
+        ".",
+        "ĊĊ"
+      ],
+      [
+        "h",
+        "e"
+      ],
+      [
+        "Ġ",
+        "e"
+      ],
+      [
+        "l",
+        "o"
+      ],
+      [
+        "Ġ",
+        "M"
+      ],
+      [
+        "Ġb",
+        "e"
+      ],
+      [
+        "er",
+        "s"
+      ],
+      [
+        "Ġ",
+        "on"
+      ],
+      [
+        "Ġc",
+        "on"
+      ],
+      [
+        "a",
+        "p"
+      ],
+      [
+        "u",
+        "b"
+      ],
+      [
+        "Ġ",
+        "P"
+      ],
+      [
+        "ĠĠĠĠĠĠĠĠ",
+        "ĠĠĠĠĠĠĠ"
+      ],
+      [
+        "as",
+        "s"
+      ],
+      [
+        "in",
+        "t"
+      ],
+      [
+        ">",
+        "Ċ"
+      ],
+      [
+        "l",
+        "y"
+      ],
+      [
+        "ur",
+        "n"
+      ],
+      [
+        "Ġ",
+        "$"
+      ],
+      [
+        ";",
+        "ĊĊ"
+      ],
+      [
+        "a",
+        "v"
+      ],
+      [
+        "p",
+        "ort"
+      ],
+      [
+        "i",
+        "r"
+      ],
+      [
+        "-",
+        ">"
+      ],
+      [
+        "n",
+        "t"
+      ],
+      [
+        "ct",
+        "ion"
+      ],
+      [
+        "en",
+        "d"
+      ],
+      [
+        "Ġd",
+        "e"
+      ],
+      [
+        "it",
+        "h"
+      ],
+      [
+        "ou",
+        "t"
+      ],
+      [
+        "t",
+        "urn"
+      ],
+      [
+        "ou",
+        "r"
+      ],
+      [
+        "ĠĠĠĠ",
+        "Ġ"
+      ],
+      [
+        "l",
+        "ic"
+      ],
+      [
+        "re",
+        "s"
+      ],
+      [
+        "p",
+        "t"
+      ],
+      [
+        "=",
+        "="
+      ],
+      [
+        "Ġth",
+        "is"
+      ],
+      [
+        "Ġw",
+        "h"
+      ],
+      [
+        "Ġ",
+        "if"
+      ],
+      [
+        "Ġ",
+        "D"
+      ],
+      [
+        "v",
+        "er"
+      ],
+      [
+        "ag",
+        "e"
+      ],
+      [
+        "Ġ",
+        "B"
+      ],
+      [
+        "h",
+        "t"
+      ],
+      [
+        "ex",
+        "t"
+      ],
+      [
+        "=",
+        "\""
+      ],
+      [
+        "Ġth",
+        "at"
+      ],
+      [
+        "**",
+        "**"
+      ],
+      [
+        "Ġ",
+        "R"
+      ],
+      [
+        "Ġ",
+        "it"
+      ],
+      [
+        "es",
+        "s"
+      ],
+      [
+        "Ġ",
+        "F"
+      ],
+      [
+        "Ġ",
+        "r"
+      ],
+      [
+        "o",
+        "s"
+      ],
+      [
+        "an",
+        "d"
+      ],
+      [
+        "Ġa",
+        "s"
+      ],
+      [
+        "e",
+        "ct"
+      ],
+      [
+        "k",
+        "e"
+      ],
+      [
+        "ro",
+        "m"
+      ],
+      [
+        "Ġ",
+        "//"
+      ],
+      [
+        "c",
+        "on"
+      ],
+      [
+        "Ġ",
+        "L"
+      ],
+      [
+        "(",
+        "\""
+      ],
+      [
+        "q",
+        "u"
+      ],
+      [
+        "l",
+        "ass"
+      ],
+      [
+        "Ġw",
+        "ith"
+      ],
+      [
+        "i",
+        "z"
+      ],
+      [
+        "d",
+        "e"
+      ],
+      [
+        "Ġ",
+        "N"
+      ],
+      [
+        "Ġa",
+        "l"
+      ],
+      [
+        "o",
+        "p"
+      ],
+      [
+        "u",
+        "p"
+      ],
+      [
+        "g",
+        "et"
+      ],
+      [
+        "Ġ}",
+        "Ċ"
+      ],
+      [
+        "i",
+        "le"
+      ],
+      [
+        "Ġa",
+        "n"
+      ],
+      [
+        "at",
+        "a"
+      ],
+      [
+        "o",
+        "re"
+      ],
+      [
+        "r",
+        "i"
+      ],
+      [
+        "Ġp",
+        "ro"
+      ],
+      [
+        ";",
+        "čĊ"
+      ],
+      [
+        "ĉĉ",
+        "ĉĉ"
+      ],
+      [
+        "t",
+        "er"
+      ],
+      [
+        "a",
+        "in"
+      ],
+      [
+        "Ġ",
+        "W"
+      ],
+      [
+        "Ġ",
+        "E"
+      ],
+      [
+        "Ġc",
+        "om"
+      ],
+      [
+        "Ġre",
+        "turn"
+      ],
+      [
+        "ar",
+        "t"
+      ],
+      [
+        "Ġ",
+        "H"
+      ],
+      [
+        "a",
+        "ck"
+      ],
+      [
+        "im",
+        "port"
+      ],
+      [
+        "ub",
+        "lic"
+      ],
+      [
+        "Ġ",
+        "or"
+      ],
+      [
+        "e",
+        "st"
+      ],
+      [
+        "m",
+        "ent"
+      ],
+      [
+        "Ġ",
+        "G"
+      ],
+      [
+        "ab",
+        "le"
+      ],
+      [
+        "Ġ",
+        "-"
+      ],
+      [
+        "in",
+        "e"
+      ],
+      [
+        "il",
+        "l"
+      ],
+      [
+        "in",
+        "d"
+      ],
+      [
+        "er",
+        "e"
+      ],
+      [
+        ":",
+        ":"
+      ],
+      [
+        "it",
+        "y"
+      ],
+      [
+        "Ġ",
+        "+"
+      ],
+      [
+        "Ġt",
+        "r"
+      ],
+      [
+        "el",
+        "f"
+      ],
+      [
+        "ig",
+        "ht"
+      ],
+      [
+        "(",
+        "'"
+      ],
+      [
+        "or",
+        "m"
+      ],
+      [
+        "ul",
+        "t"
+      ],
+      [
+        "st",
+        "r"
+      ],
+      [
+        ".",
+        "."
+      ],
+      [
+        "\"",
+        ","
+      ],
+      [
+        "Ġy",
+        "ou"
+      ],
+      [
+        "y",
+        "pe"
+      ]
+    ]
+  }
+}

tokenizer_config.json

@@ -0,0 +1,306 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "</img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "<IMG_CONTEXT>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "<quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151669": {
+      "content": "</quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "</ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "</box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "<video>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>",
+    "<img>",
+    "</img>",
+    "<IMG_CONTEXT>",
+    "<quad>",
+    "</quad>",
+    "<ref>",
+    "</ref>",
+    "<box>",
+    "</box>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "context_image_token": "<IMG_CONTEXT>",
+  "end_image_token": "</img>",
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {
+    "context_image_token": "<IMG_CONTEXT>",
+    "end_image_token": "</img>",
+    "start_image_token": "<img>",
+    "video_token": "<video>"
+  },
+  "model_max_length": 8192,
+  "pad_token": "<|endoftext|>",
+  "return_token_type_ids": false,
+  "split_special_tokens": false,
+  "start_image_token": "<img>",
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "video_token": "<video>"
+}