Add tiny processor fixture for testing

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,145 @@
+{%- if enable_thinking is not defined -%}
+    {%- set enable_thinking = false -%}
+{%- endif -%}
+{%- macro render_content(content, is_system_content=false) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content is iterable and content is not mapping -%}
+        {%- set ns = namespace(parts=[]) -%}
+        {%- for item in content -%}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' -%}
+                {%- if is_system_content -%}
+                    {{- raise_exception('System message cannot contain images.') -}}
+                {%- endif -%}
+                {%- set ns.parts = ns.parts + ['<|image_pad|>'] -%}
+            {%- elif 'video' in item or item.type == 'video' -%}
+                {%- if is_system_content -%}
+                    {{- raise_exception('System message cannot contain videos.') -}}
+                {%- endif -%}
+                {%- set ns.parts = ns.parts + ['<|video_pad|>'] -%}
+            {%- elif 'text' in item -%}
+                {%- set ns.parts = ns.parts + [item.text] -%}
+            {%- else -%}
+                {{- raise_exception('Unexpected item type in content.') -}}
+            {%- endif -%}
+        {%- endfor -%}
+        {{- ns.parts | join('\n') -}}
+    {%- elif content is none or content is undefined -%}
+        {{- '' -}}
+    {%- else -%}
+        {{- raise_exception('Unexpected content type.') -}}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- else %}
+        {{- '<think>\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,91 @@
+{
+  "architectures": [
+    "MiniCPMV4_6ForConditionalGeneration"
+  ],
+  "bos_token_id": null,
+  "drop_vision_last_layer": false,
+  "eos_token_id": 500,
+  "image_size": 1120,
+  "model_type": "minicpmv4_6",
+  "pad_token_id": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.7.0",
+  "use_cache": true,
+  "vision_config": {
+    "attention_dropout": 0.0,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "image_size": 980,
+    "intermediate_size": 4304,
+    "layer_norm_eps": 1e-06,
+    "model_type": "minicpmv4_6_vision",
+    "num_attention_heads": 16,
+    "num_channels": 3,
+    "num_hidden_layers": 27,
+    "patch_size": 14
+  },
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_output_gate": true,
+    "full_attention_interval": 4,
+    "head_dim": 256,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3584,
+    "layer_types": [
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention"
+    ],
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 16,
+    "linear_value_head_dim": 128,
+    "mamba_ssm_dtype": "float32",
+    "max_position_embeddings": 262144,
+    "mlp_only_layers": [],
+    "mtp_num_hidden_layers": 1,
+    "mtp_use_dedicated_embeddings": false,
+    "num_attention_heads": 8,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 2,
+    "partial_rotary_factor": 0.25,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "partial_rotary_factor": 0.25,
+      "rope_theta": 10000000,
+      "rope_type": "default"
+    },
+    "vocab_size": 248094,
+    "model_type": "qwen3_5_text",
+    "tie_word_embeddings": true
+  },
+  "insert_layer_id": 6,
+  "image_token_id": 248056,
+  "video_token_id": 248057,
+  "vocab_size": 500
+}

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 248045,
+  "do_sample": true,
+  "eos_token_id": [
+    248044,
+    248046
+  ],
+  "temperature": 0.7,
+  "top_k": 0,
+  "top_p": 1.0,
+  "repetition_penalty": 1.0,
+  "transformers_version": "5.7.0"
+}

preprocessor_config.json

@@ -0,0 +1,19 @@
+{
+  "image_processor_type": "MiniCPMV4_6ImageProcessor",
+  "processor_class": "MiniCPMV4_6Processor",
+  "max_slice_nums": 9,
+  "scale_resolution": 448,
+  "patch_size": 14,
+  "use_image_id": true,
+  "slice_mode": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ]
+}

tokenizer.json

@@ -0,0 +1,1981 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 500,
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 501,
+      "content": "<|im_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 502,
+      "content": "<|im_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 503,
+      "content": "<|object_ref_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 504,
+      "content": "<|object_ref_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 505,
+      "content": "<|box_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 506,
+      "content": "<|box_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 507,
+      "content": "<|quad_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 508,
+      "content": "<|quad_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 509,
+      "content": "<|vision_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 510,
+      "content": "<|vision_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 511,
+      "content": "<|vision_pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 512,
+      "content": "<|image_pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 513,
+      "content": "<|video_pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 514,
+      "content": "<tool_call>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 515,
+      "content": "</tool_call>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 516,
+      "content": "<|fim_prefix|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 517,
+      "content": "<|fim_middle|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 518,
+      "content": "<|fim_suffix|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 519,
+      "content": "<|fim_pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 520,
+      "content": "<|repo_name|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 521,
+      "content": "<|file_sep|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 522,
+      "content": "<tool_response>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 523,
+      "content": "</tool_response>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 524,
+      "content": "<think>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 525,
+      "content": "</think>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 526,
+      "content": "<|audio_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 527,
+      "content": "<|audio_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 528,
+      "content": "<tts_pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 529,
+      "content": "<tts_text_bos>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 530,
+      "content": "<tts_text_eod>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 531,
+      "content": "<tts_text_bos_single>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 532,
+      "content": "<|audio_pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 533,
+      "content": "<unk>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 534,
+      "content": "<image>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 535,
+      "content": "</image>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 536,
+      "content": "<ref>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 537,
+      "content": "</ref>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 538,
+      "content": "<box>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 539,
+      "content": "</box>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 540,
+      "content": "<quad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 541,
+      "content": "</quad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 542,
+      "content": "<point>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 543,
+      "content": "</point>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 544,
+      "content": "<slice>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 545,
+      "content": "</slice>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 546,
+      "content": "<image_id>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 547,
+      "content": "</image_id>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 548,
+      "content": "<timestamp>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 549,
+      "content": "</timestamp>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "NFC"
+  },
+  "pre_tokenizer": {
+    "type": "Sequence",
+    "pretokenizers": [
+      {
+        "type": "Split",
+        "pattern": {
+          "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
+        },
+        "behavior": "Isolated",
+        "invert": false
+      },
+      {
+        "type": "ByteLevel",
+        "add_prefix_space": false,
+        "trim_offsets": false,
+        "use_regex": false
+      }
+    ]
+  },
+  "post_processor": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": false,
+    "use_regex": false
+  },
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": false,
+    "use_regex": false
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": "",
+    "end_of_word_suffix": "",
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "!": 0,
+      "\"": 1,
+      "#": 2,
+      "$": 3,
+      "%": 4,
+      "&": 5,
+      "'": 6,
+      "(": 7,
+      ")": 8,
+      "*": 9,
+      "+": 10,
+      ",": 11,
+      "-": 12,
+      ".": 13,
+      "/": 14,
+      "0": 15,
+      "1": 16,
+      "2": 17,
+      "3": 18,
+      "4": 19,
+      "5": 20,
+      "6": 21,
+      "7": 22,
+      "8": 23,
+      "9": 24,
+      ":": 25,
+      ";": 26,
+      "<": 27,
+      "=": 28,
+      ">": 29,
+      "?": 30,
+      "@": 31,
+      "A": 32,
+      "B": 33,
+      "C": 34,
+      "D": 35,
+      "E": 36,
+      "F": 37,
+      "G": 38,
+      "H": 39,
+      "I": 40,
+      "J": 41,
+      "K": 42,
+      "L": 43,
+      "M": 44,
+      "N": 45,
+      "O": 46,
+      "P": 47,
+      "Q": 48,
+      "R": 49,
+      "S": 50,
+      "T": 51,
+      "U": 52,
+      "V": 53,
+      "W": 54,
+      "X": 55,
+      "Y": 56,
+      "Z": 57,
+      "[": 58,
+      "\\": 59,
+      "]": 60,
+      "^": 61,
+      "_": 62,
+      "`": 63,
+      "a": 64,
+      "b": 65,
+      "c": 66,
+      "d": 67,
+      "e": 68,
+      "f": 69,
+      "g": 70,
+      "h": 71,
+      "i": 72,
+      "j": 73,
+      "k": 74,
+      "l": 75,
+      "m": 76,
+      "n": 77,
+      "o": 78,
+      "p": 79,
+      "q": 80,
+      "r": 81,
+      "s": 82,
+      "t": 83,
+      "u": 84,
+      "v": 85,
+      "w": 86,
+      "x": 87,
+      "y": 88,
+      "z": 89,
+      "{": 90,
+      "|": 91,
+      "}": 92,
+      "~": 93,
+      "¡": 94,
+      "¢": 95,
+      "£": 96,
+      "¤": 97,
+      "¥": 98,
+      "¦": 99,
+      "§": 100,
+      "¨": 101,
+      "©": 102,
+      "ª": 103,
+      "«": 104,
+      "¬": 105,
+      "®": 106,
+      "¯": 107,
+      "°": 108,
+      "±": 109,
+      "²": 110,
+      "³": 111,
+      "´": 112,
+      "µ": 113,
+      "¶": 114,
+      "·": 115,
+      "¸": 116,
+      "¹": 117,
+      "º": 118,
+      "»": 119,
+      "¼": 120,
+      "½": 121,
+      "¾": 122,
+      "¿": 123,
+      "À": 124,
+      "Á": 125,
+      "Â": 126,
+      "Ã": 127,
+      "Ä": 128,
+      "Å": 129,
+      "Æ": 130,
+      "Ç": 131,
+      "È": 132,
+      "É": 133,
+      "Ê": 134,
+      "Ë": 135,
+      "Ì": 136,
+      "Í": 137,
+      "Î": 138,
+      "Ï": 139,
+      "Ð": 140,
+      "Ñ": 141,
+      "Ò": 142,
+      "Ó": 143,
+      "Ô": 144,
+      "Õ": 145,
+      "Ö": 146,
+      "×": 147,
+      "Ø": 148,
+      "Ù": 149,
+      "Ú": 150,
+      "Û": 151,
+      "Ü": 152,
+      "Ý": 153,
+      "Þ": 154,
+      "ß": 155,
+      "à": 156,
+      "á": 157,
+      "â": 158,
+      "ã": 159,
+      "ä": 160,
+      "å": 161,
+      "æ": 162,
+      "ç": 163,
+      "è": 164,
+      "é": 165,
+      "ê": 166,
+      "ë": 167,
+      "ì": 168,
+      "í": 169,
+      "î": 170,
+      "ï": 171,
+      "ð": 172,
+      "ñ": 173,
+      "ò": 174,
+      "ó": 175,
+      "ô": 176,
+      "õ": 177,
+      "ö": 178,
+      "÷": 179,
+      "ø": 180,
+      "ù": 181,
+      "ú": 182,
+      "û": 183,
+      "ü": 184,
+      "ý": 185,
+      "þ": 186,
+      "ÿ": 187,
+      "Ā": 188,
+      "ā": 189,
+      "Ă": 190,
+      "ă": 191,
+      "Ą": 192,
+      "ą": 193,
+      "Ć": 194,
+      "ć": 195,
+      "Ĉ": 196,
+      "ĉ": 197,
+      "Ċ": 198,
+      "ċ": 199,
+      "Č": 200,
+      "č": 201,
+      "Ď": 202,
+      "ď": 203,
+      "Đ": 204,
+      "đ": 205,
+      "Ē": 206,
+      "ē": 207,
+      "Ĕ": 208,
+      "ĕ": 209,
+      "Ė": 210,
+      "ė": 211,
+      "Ę": 212,
+      "ę": 213,
+      "Ě": 214,
+      "ě": 215,
+      "Ĝ": 216,
+      "ĝ": 217,
+      "Ğ": 218,
+      "ğ": 219,
+      "Ġ": 220,
+      "ġ": 221,
+      "Ģ": 222,
+      "ģ": 223,
+      "Ĥ": 224,
+      "ĥ": 225,
+      "Ħ": 226,
+      "ħ": 227,
+      "Ĩ": 228,
+      "ĩ": 229,
+      "Ī": 230,
+      "ī": 231,
+      "Ĭ": 232,
+      "ĭ": 233,
+      "Į": 234,
+      "į": 235,
+      "İ": 236,
+      "ı": 237,
+      "IJ": 238,
+      "ij": 239,
+      "Ĵ": 240,
+      "ĵ": 241,
+      "Ķ": 242,
+      "ķ": 243,
+      "ĸ": 244,
+      "Ĺ": 245,
+      "ĺ": 246,
+      "Ļ": 247,
+      "ļ": 248,
+      "Ľ": 249,
+      "ľ": 250,
+      "Ŀ": 251,
+      "ŀ": 252,
+      "Ł": 253,
+      "ł": 254,
+      "Ń": 255,
+      "ĠĠ": 256,
+      "ĠĠĠĠ": 257,
+      "in": 258,
+      "Ġt": 259,
+      "ĠĠĠĠĠĠĠĠ": 260,
+      "er": 261,
+      "ĠĠĠ": 262,
+      "on": 263,
+      "Ġa": 264,
+      "re": 265,
+      "at": 266,
+      "st": 267,
+      "en": 268,
+      "or": 269,
+      "Ġth": 270,
+      "ĊĊ": 271,
+      "Ġc": 272,
+      "le": 273,
+      "Ġs": 274,
+      "it": 275,
+      "an": 276,
+      "ar": 277,
+      "al": 278,
+      "Ġthe": 279,
+      "Ġp": 280,
+      "Ġf": 281,
+      "ou": 282,
+      "Ġ=": 283,
+      "is": 284,
+      "ĠĠĠĠĠĠĠ": 285,
+      "ing": 286,
+      "es": 287,
+      "Ġw": 288,
+      "ion": 289,
+      "ed": 290,
+      "ic": 291,
+      "Ġb": 292,
+      "Ġd": 293,
+      "et": 294,
+      "Ġm": 295,
+      "Ġo": 296,
+      "ĉĉ": 297,
+      "ro": 298,
+      "as": 299,
+      "el": 300,
+      "ct": 301,
+      "nd": 302,
+      "Ġin": 303,
+      "Ġh": 304,
+      "ent": 305,
+      "id": 306,
+      "Ġn": 307,
+      "am": 308,
+      "ĠĠĠĠĠĠĠĠĠĠĠ": 309,
+      "Ġto": 310,
+      "Ġre": 311,
+      "--": 312,
+      "Ġ{": 313,
+      "Ġof": 314,
+      "om": 315,
+      "im": 316,
+      "čĊ": 317,
+      "Ġ(": 318,
+      "il": 319,
+      "//": 320,
+      "Ġand": 321,
+      "ur": 322,
+      "se": 323,
+      "Ġl": 324,
+      "ex": 325,
+      "ĠS": 326,
+      "ad": 327,
+      "Ġ\"": 328,
+      "ch": 329,
+      "ut": 330,
+      "if": 331,
+      "**": 332,
+      "Ġ}": 333,
+      "em": 334,
+      "ol": 335,
+      "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 336,
+      "th": 337,
+      "Ġg": 338,
+      "ig": 339,
+      "iv": 340,
+      "ce": 341,
+      "od": 342,
+      "Ġv": 343,
+      "ate": 344,
+      "ĠT": 345,
+      "ag": 346,
+      "ay": 347,
+      "Ġ*": 348,
+      "ot": 349,
+      "us": 350,
+      "ĠC": 351,
+      "Ġst": 352,
+      "ĠI": 353,
+      "un": 354,
+      "ul": 355,
+      "ue": 356,
+      "ĠA": 357,
+      "ow": 358,
+      "Ġ'": 359,
+      "ew": 360,
+      "Ġ<": 361,
+      "ation": 362,
+      "()": 363,
+      "Ġfor": 364,
+      "ab": 365,
+      "ort": 366,
+      "um": 367,
+      "ame": 368,
+      "Ġis": 369,
+      "pe": 370,
+      "tr": 371,
+      "ck": 372,
+      "âĢ": 373,
+      "Ġy": 374,
+      "ist": 375,
+      "----": 376,
+      "he": 377,
+      "Ġe": 378,
+      "lo": 379,
+      "ĠM": 380,
+      "Ġbe": 381,
+      "ers": 382,
+      "Ġon": 383,
+      "Ġcon": 384,
+      "ap": 385,
+      "ub": 386,
+      "ĠP": 387,
+      "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 388,
+      "ass": 389,
+      "int": 390,
+      "ly": 391,
+      "urn": 392,
+      "Ġ$": 393,
+      "av": 394,
+      "port": 395,
+      "ir": 396,
+      "->": 397,
+      "nt": 398,
+      "ction": 399,
+      "end": 400,
+      "Ġde": 401,
+      "ith": 402,
+      "out": 403,
+      "turn": 404,
+      "our": 405,
+      "ĠĠĠĠĠ": 406,
+      "lic": 407,
+      "res": 408,
+      "pt": 409,
+      "==": 410,
+      "Ġthis": 411,
+      "Ġwh": 412,
+      "Ġif": 413,
+      "ĠD": 414,
+      "ver": 415,
+      "age": 416,
+      "ĠB": 417,
+      "ht": 418,
+      "ext": 419,
+      "=\"": 420,
+      "Ġthat": 421,
+      "****": 422,
+      "ĠR": 423,
+      "Ġit": 424,
+      "ess": 425,
+      "ĠF": 426,
+      "Ġr": 427,
+      "os": 428,
+      "and": 429,
+      "Ġas": 430,
+      "ect": 431,
+      "ke": 432,
+      "rom": 433,
+      "Ġ//": 434,
+      "con": 435,
+      "ĠL": 436,
+      "(\"": 437,
+      "qu": 438,
+      "lass": 439,
+      "Ġwith": 440,
+      "iz": 441,
+      "de": 442,
+      "ĠN": 443,
+      "Ġal": 444,
+      "op": 445,
+      "up": 446,
+      "get": 447,
+      "ile": 448,
+      "Ġan": 449,
+      "ata": 450,
+      "ore": 451,
+      "ri": 452,
+      "Ġpro": 453,
+      "ĉĉĉĉ": 454,
+      "ter": 455,
+      "ain": 456,
+      "ĠW": 457,
+      "ĠE": 458,
+      "Ġcom": 459,
+      "Ġreturn": 460,
+      "art": 461,
+      "ĠH": 462,
+      "ack": 463,
+      "import": 464,
+      "ublic": 465,
+      "Ġor": 466,
+      "est": 467,
+      "ment": 468,
+      "ĠG": 469,
+      "able": 470,
+      "Ġ-": 471,
+      "ine": 472,
+      "ill": 473,
+      "ind": 474,
+      "ere": 475,
+      "::": 476,
+      "ity": 477,
+      "Ġ+": 478,
+      "Ġtr": 479,
+      "elf": 480,
+      "ight": 481,
+      "('": 482,
+      "orm": 483,
+      "ult": 484,
+      "str": 485,
+      "..": 486,
+      "\",": 487,
+      "Ġyou": 488,
+      "ype": 489,
+      "pl": 490,
+      "Ġnew": 491,
+      "Ġj": 492,
+      "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 493,
+      "Ġfrom": 494,
+      "Ġex": 495,
+      "ĠO": 496,
+      "ld": 497,
+      "Ġ[": 498,
+      "oc": 499
+    },
+    "merges": [
+      [
+        "Ġ",
+        "Ġ"
+      ],
+      [
+        "ĠĠ",
+        "ĠĠ"
+      ],
+      [
+        "i",
+        "n"
+      ],
+      [
+        "Ġ",
+        "t"
+      ],
+      [
+        "ĠĠĠĠ",
+        "ĠĠĠĠ"
+      ],
+      [
+        "e",
+        "r"
+      ],
+      [
+        "ĠĠ",
+        "Ġ"
+      ],
+      [
+        "o",
+        "n"
+      ],
+      [
+        "Ġ",
+        "a"
+      ],
+      [
+        "r",
+        "e"
+      ],
+      [
+        "a",
+        "t"
+      ],
+      [
+        "s",
+        "t"
+      ],
+      [
+        "e",
+        "n"
+      ],
+      [
+        "o",
+        "r"
+      ],
+      [
+        "Ġt",
+        "h"
+      ],
+      [
+        "Ċ",
+        "Ċ"
+      ],
+      [
+        "Ġ",
+        "c"
+      ],
+      [
+        "l",
+        "e"
+      ],
+      [
+        "Ġ",
+        "s"
+      ],
+      [
+        "i",
+        "t"
+      ],
+      [
+        "a",
+        "n"
+      ],
+      [
+        "a",
+        "r"
+      ],
+      [
+        "a",
+        "l"
+      ],
+      [
+        "Ġth",
+        "e"
+      ],
+      [
+        "Ġ",
+        "p"
+      ],
+      [
+        "Ġ",
+        "f"
+      ],
+      [
+        "o",
+        "u"
+      ],
+      [
+        "Ġ",
+        "="
+      ],
+      [
+        "i",
+        "s"
+      ],
+      [
+        "ĠĠĠĠ",
+        "ĠĠĠ"
+      ],
+      [
+        "in",
+        "g"
+      ],
+      [
+        "e",
+        "s"
+      ],
+      [
+        "Ġ",
+        "w"
+      ],
+      [
+        "i",
+        "on"
+      ],
+      [
+        "e",
+        "d"
+      ],
+      [
+        "i",
+        "c"
+      ],
+      [
+        "Ġ",
+        "b"
+      ],
+      [
+        "Ġ",
+        "d"
+      ],
+      [
+        "e",
+        "t"
+      ],
+      [
+        "Ġ",
+        "m"
+      ],
+      [
+        "Ġ",
+        "o"
+      ],
+      [
+        "ĉ",
+        "ĉ"
+      ],
+      [
+        "r",
+        "o"
+      ],
+      [
+        "a",
+        "s"
+      ],
+      [
+        "e",
+        "l"
+      ],
+      [
+        "c",
+        "t"
+      ],
+      [
+        "n",
+        "d"
+      ],
+      [
+        "Ġ",
+        "in"
+      ],
+      [
+        "Ġ",
+        "h"
+      ],
+      [
+        "en",
+        "t"
+      ],
+      [
+        "i",
+        "d"
+      ],
+      [
+        "Ġ",
+        "n"
+      ],
+      [
+        "a",
+        "m"
+      ],
+      [
+        "ĠĠĠĠĠĠĠĠ",
+        "ĠĠĠ"
+      ],
+      [
+        "Ġt",
+        "o"
+      ],
+      [
+        "Ġ",
+        "re"
+      ],
+      [
+        "-",
+        "-"
+      ],
+      [
+        "Ġ",
+        "{"
+      ],
+      [
+        "Ġo",
+        "f"
+      ],
+      [
+        "o",
+        "m"
+      ],
+      [
+        "i",
+        "m"
+      ],
+      [
+        "č",
+        "Ċ"
+      ],
+      [
+        "Ġ",
+        "("
+      ],
+      [
+        "i",
+        "l"
+      ],
+      [
+        "/",
+        "/"
+      ],
+      [
+        "Ġa",
+        "nd"
+      ],
+      [
+        "u",
+        "r"
+      ],
+      [
+        "s",
+        "e"
+      ],
+      [
+        "Ġ",
+        "l"
+      ],
+      [
+        "e",
+        "x"
+      ],
+      [
+        "Ġ",
+        "S"
+      ],
+      [
+        "a",
+        "d"
+      ],
+      [
+        "Ġ",
+        "\""
+      ],
+      [
+        "c",
+        "h"
+      ],
+      [
+        "u",
+        "t"
+      ],
+      [
+        "i",
+        "f"
+      ],
+      [
+        "*",
+        "*"
+      ],
+      [
+        "Ġ",
+        "}"
+      ],
+      [
+        "e",
+        "m"
+      ],
+      [
+        "o",
+        "l"
+      ],
+      [
+        "ĠĠĠĠĠĠĠĠ",
+        "ĠĠĠĠĠĠĠĠ"
+      ],
+      [
+        "t",
+        "h"
+      ],
+      [
+        "Ġ",
+        "g"
+      ],
+      [
+        "i",
+        "g"
+      ],
+      [
+        "i",
+        "v"
+      ],
+      [
+        "c",
+        "e"
+      ],
+      [
+        "o",
+        "d"
+      ],
+      [
+        "Ġ",
+        "v"
+      ],
+      [
+        "at",
+        "e"
+      ],
+      [
+        "Ġ",
+        "T"
+      ],
+      [
+        "a",
+        "g"
+      ],
+      [
+        "a",
+        "y"
+      ],
+      [
+        "Ġ",
+        "*"
+      ],
+      [
+        "o",
+        "t"
+      ],
+      [
+        "u",
+        "s"
+      ],
+      [
+        "Ġ",
+        "C"
+      ],
+      [
+        "Ġ",
+        "st"
+      ],
+      [
+        "Ġ",
+        "I"
+      ],
+      [
+        "u",
+        "n"
+      ],
+      [
+        "u",
+        "l"
+      ],
+      [
+        "u",
+        "e"
+      ],
+      [
+        "Ġ",
+        "A"
+      ],
+      [
+        "o",
+        "w"
+      ],
+      [
+        "Ġ",
+        "'"
+      ],
+      [
+        "e",
+        "w"
+      ],
+      [
+        "Ġ",
+        "<"
+      ],
+      [
+        "at",
+        "ion"
+      ],
+      [
+        "(",
+        ")"
+      ],
+      [
+        "Ġf",
+        "or"
+      ],
+      [
+        "a",
+        "b"
+      ],
+      [
+        "or",
+        "t"
+      ],
+      [
+        "u",
+        "m"
+      ],
+      [
+        "am",
+        "e"
+      ],
+      [
+        "Ġ",
+        "is"
+      ],
+      [
+        "p",
+        "e"
+      ],
+      [
+        "t",
+        "r"
+      ],
+      [
+        "c",
+        "k"
+      ],
+      [
+        "â",
+        "Ģ"
+      ],
+      [
+        "Ġ",
+        "y"
+      ],
+      [
+        "i",
+        "st"
+      ],
+      [
+        "--",
+        "--"
+      ],
+      [
+        "h",
+        "e"
+      ],
+      [
+        "Ġ",
+        "e"
+      ],
+      [
+        "l",
+        "o"
+      ],
+      [
+        "Ġ",
+        "M"
+      ],
+      [
+        "Ġb",
+        "e"
+      ],
+      [
+        "er",
+        "s"
+      ],
+      [
+        "Ġ",
+        "on"
+      ],
+      [
+        "Ġc",
+        "on"
+      ],
+      [
+        "a",
+        "p"
+      ],
+      [
+        "u",
+        "b"
+      ],
+      [
+        "Ġ",
+        "P"
+      ],
+      [
+        "ĠĠĠĠĠĠĠĠ",
+        "ĠĠĠĠĠĠĠ"
+      ],
+      [
+        "as",
+        "s"
+      ],
+      [
+        "in",
+        "t"
+      ],
+      [
+        "l",
+        "y"
+      ],
+      [
+        "ur",
+        "n"
+      ],
+      [
+        "Ġ",
+        "$"
+      ],
+      [
+        "a",
+        "v"
+      ],
+      [
+        "p",
+        "ort"
+      ],
+      [
+        "i",
+        "r"
+      ],
+      [
+        "-",
+        ">"
+      ],
+      [
+        "n",
+        "t"
+      ],
+      [
+        "ct",
+        "ion"
+      ],
+      [
+        "en",
+        "d"
+      ],
+      [
+        "Ġd",
+        "e"
+      ],
+      [
+        "it",
+        "h"
+      ],
+      [
+        "ou",
+        "t"
+      ],
+      [
+        "t",
+        "urn"
+      ],
+      [
+        "ou",
+        "r"
+      ],
+      [
+        "ĠĠĠĠ",
+        "Ġ"
+      ],
+      [
+        "l",
+        "ic"
+      ],
+      [
+        "re",
+        "s"
+      ],
+      [
+        "p",
+        "t"
+      ],
+      [
+        "=",
+        "="
+      ],
+      [
+        "Ġth",
+        "is"
+      ],
+      [
+        "Ġw",
+        "h"
+      ],
+      [
+        "Ġ",
+        "if"
+      ],
+      [
+        "Ġ",
+        "D"
+      ],
+      [
+        "v",
+        "er"
+      ],
+      [
+        "ag",
+        "e"
+      ],
+      [
+        "Ġ",
+        "B"
+      ],
+      [
+        "h",
+        "t"
+      ],
+      [
+        "ex",
+        "t"
+      ],
+      [
+        "=",
+        "\""
+      ],
+      [
+        "Ġth",
+        "at"
+      ],
+      [
+        "**",
+        "**"
+      ],
+      [
+        "Ġ",
+        "R"
+      ],
+      [
+        "Ġ",
+        "it"
+      ],
+      [
+        "es",
+        "s"
+      ],
+      [
+        "Ġ",
+        "F"
+      ],
+      [
+        "Ġ",
+        "r"
+      ],
+      [
+        "o",
+        "s"
+      ],
+      [
+        "an",
+        "d"
+      ],
+      [
+        "Ġa",
+        "s"
+      ],
+      [
+        "e",
+        "ct"
+      ],
+      [
+        "k",
+        "e"
+      ],
+      [
+        "ro",
+        "m"
+      ],
+      [
+        "Ġ",
+        "//"
+      ],
+      [
+        "c",
+        "on"
+      ],
+      [
+        "Ġ",
+        "L"
+      ],
+      [
+        "(",
+        "\""
+      ],
+      [
+        "q",
+        "u"
+      ],
+      [
+        "l",
+        "ass"
+      ],
+      [
+        "Ġw",
+        "ith"
+      ],
+      [
+        "i",
+        "z"
+      ],
+      [
+        "d",
+        "e"
+      ],
+      [
+        "Ġ",
+        "N"
+      ],
+      [
+        "Ġa",
+        "l"
+      ],
+      [
+        "o",
+        "p"
+      ],
+      [
+        "u",
+        "p"
+      ],
+      [
+        "g",
+        "et"
+      ],
+      [
+        "i",
+        "le"
+      ],
+      [
+        "Ġa",
+        "n"
+      ],
+      [
+        "at",
+        "a"
+      ],
+      [
+        "o",
+        "re"
+      ],
+      [
+        "r",
+        "i"
+      ],
+      [
+        "Ġp",
+        "ro"
+      ],
+      [
+        "ĉĉ",
+        "ĉĉ"
+      ],
+      [
+        "t",
+        "er"
+      ],
+      [
+        "a",
+        "in"
+      ],
+      [
+        "Ġ",
+        "W"
+      ],
+      [
+        "Ġ",
+        "E"
+      ],
+      [
+        "Ġc",
+        "om"
+      ],
+      [
+        "Ġre",
+        "turn"
+      ],
+      [
+        "ar",
+        "t"
+      ],
+      [
+        "Ġ",
+        "H"
+      ],
+      [
+        "a",
+        "ck"
+      ],
+      [
+        "im",
+        "port"
+      ],
+      [
+        "ub",
+        "lic"
+      ],
+      [
+        "Ġ",
+        "or"
+      ],
+      [
+        "e",
+        "st"
+      ],
+      [
+        "m",
+        "ent"
+      ],
+      [
+        "Ġ",
+        "G"
+      ],
+      [
+        "ab",
+        "le"
+      ],
+      [
+        "Ġ",
+        "-"
+      ],
+      [
+        "in",
+        "e"
+      ],
+      [
+        "il",
+        "l"
+      ],
+      [
+        "in",
+        "d"
+      ],
+      [
+        "er",
+        "e"
+      ],
+      [
+        ":",
+        ":"
+      ],
+      [
+        "it",
+        "y"
+      ],
+      [
+        "Ġ",
+        "+"
+      ],
+      [
+        "Ġt",
+        "r"
+      ],
+      [
+        "el",
+        "f"
+      ],
+      [
+        "ig",
+        "ht"
+      ],
+      [
+        "(",
+        "'"
+      ],
+      [
+        "or",
+        "m"
+      ],
+      [
+        "ul",
+        "t"
+      ],
+      [
+        "st",
+        "r"
+      ],
+      [
+        ".",
+        "."
+      ],
+      [
+        "\"",
+        ","
+      ],
+      [
+        "Ġy",
+        "ou"
+      ],
+      [
+        "y",
+        "pe"
+      ],
+      [
+        "p",
+        "l"
+      ],
+      [
+        "Ġn",
+        "ew"
+      ],
+      [
+        "Ġ",
+        "j"
+      ],
+      [
+        "ĠĠĠĠĠĠĠĠ",
+        "ĠĠĠĠĠĠĠĠĠĠĠ"
+      ],
+      [
+        "Ġf",
+        "rom"
+      ],
+      [
+        "Ġ",
+        "ex"
+      ],
+      [
+        "Ġ",
+        "O"
+      ],
+      [
+        "l",
+        "d"
+      ],
+      [
+        "Ġ",
+        "["
+      ],
+      [
+        "o",
+        "c"
+      ]
+    ]
+  }
+}

tokenizer_config.json

@@ -0,0 +1,42 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "image_start_token": "<image>",
+    "image_end_token": "</image>",
+    "slice_start_token": "<slice>",
+    "slice_end_token": "</slice>",
+    "image_id_start_token": "<image_id>",
+    "image_id_end_token": "</image_id>"
+  },
+  "image_token": "<|image_pad|>",
+  "is_local": true,
+  "model_max_length": 262144,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "split_special_tokens": false,
+  "unk_token": "<unk>",
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>",
+  "chat_template": "{%- if enable_thinking is not defined -%}\n    {%- set enable_thinking = false -%}\n{%- endif -%}\n{%- macro render_content(content, is_system_content=false) -%}\n    {%- if content is string -%}\n        {{- content -}}\n    {%- elif content is iterable and content is not mapping -%}\n        {%- set ns = namespace(parts=[]) -%}\n        {%- for item in content -%}\n            {%- if 'image' in item or 'image_url' in item or item.type == 'image' -%}\n                {%- if is_system_content -%}\n                    {{- raise_exception('System message cannot contain images.') -}}\n                {%- endif -%}\n                {%- set ns.parts = ns.parts + ['<|image_pad|>'] -%}\n            {%- elif 'video' in item or item.type == 'video' -%}\n                {%- if is_system_content -%}\n                    {{- raise_exception('System message cannot contain videos.') -}}\n                {%- endif -%}\n                {%- set ns.parts = ns.parts + ['<|video_pad|>'] -%}\n            {%- elif 'text' in item -%}\n                {%- set ns.parts = ns.parts + [item.text] -%}\n            {%- else -%}\n                {{- raise_exception('Unexpected item type in content.') -}}\n            {%- endif -%}\n        {%- endfor -%}\n        {{- ns.parts | join('\\n') -}}\n    {%- elif content is none or content is undefined -%}\n        {{- '' -}}\n    {%- else -%}\n        {{- raise_exception('Unexpected content type.') -}}\n    {%- endif -%}\n{%- endmacro -%}\n{%- if not messages %}\n    {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n    {{- '<|im_start|>system\\n' }}\n    {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\" }}\n    {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n    {%- if messages[0].role == 'system' %}\n        {%- set content = render_content(messages[0].content, true)|trim %}\n        {%- if content %}\n            {{- '\\n\\n' + content }}\n        {%- endif %}\n    {%- endif %}\n    {{- '<|im_end|>\\n' }}\n{%- else %}\n    {%- if messages[0].role == 'system' %}\n        {%- set content = render_content(messages[0].content, true)|trim %}\n        {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n    {%- set index = (messages|length - 1) - loop.index0 %}\n    {%- if ns.multi_step_tool and message.role == \"user\" %}\n        {%- set content = render_content(message.content)|trim %}\n        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n            {%- set ns.multi_step_tool = false %}\n            {%- set ns.last_query_index = index %}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n    {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n    {%- set content = render_content(message.content)|trim %}\n    {%- if message.role == \"system\" %}\n        {%- if not loop.first %}\n            {{- raise_exception('System message must be at the beginning.') }}\n        {%- endif %}\n    {%- elif message.role == \"user\" %}\n        {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {%- set reasoning_content = '' %}\n        {%- if message.reasoning_content is string %}\n            {%- set reasoning_content = message.reasoning_content %}\n        {%- else %}\n            {%- if '</think>' in content %}\n                {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n                {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n            {%- endif %}\n        {%- endif %}\n        {%- set reasoning_content = reasoning_content|trim %}\n        {%- if loop.index0 > ns.last_query_index %}\n            {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n        {%- else %}\n            {{- '<|im_start|>' + message.role + '\\n' + content }}\n        {%- endif %}\n        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n            {%- for tool_call in message.tool_calls %}\n                {%- if tool_call.function is defined %}\n                    {%- set tool_call = tool_call.function %}\n                {%- endif %}\n                {%- if loop.first %}\n                    {%- if content|trim %}\n                        {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n                    {%- else %}\n                        {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n                    {%- endif %}\n                {%- else %}\n                    {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n                {%- endif %}\n                {%- if tool_call.arguments is defined %}\n                    {%- for args_name, args_value in tool_call.arguments|items %}\n                        {{- '<parameter=' + args_name + '>\\n' }}\n                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n                        {{- args_value }}\n                        {{- '\\n</parameter>\\n' }}\n                    {%- endfor %}\n                {%- endif %}\n                {{- '</function>\\n</tool_call>' }}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- content }}\n        {{- '\\n</tool_response>' }}\n        {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n            {{- '<|im_end|>\\n' }}\n        {%- elif loop.last %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- else %}\n        {{- raise_exception('Unexpected message role.') }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n    {%- if enable_thinking is defined and enable_thinking is false %}\n        {{- '<think>\\n\\n</think>\\n\\n' }}\n    {%- else %}\n        {{- '<think>\\n' }}\n    {%- endif %}\n{%- endif %}\n",
+  "tokenizer_class": "Qwen2Tokenizer"
+}