Hugging Face's logo

izblue
/
mj_brain-v2

Image-Text-to-Text
Transformers
English
gemma4
text-generation-inference
unsloth
conversational
Model card Files
xet
 Community
izblue commited on
Commit
29aeb35
·
verified ·
1 Parent(s): 72dcdb0

(Trained with Unsloth)

Browse files
Files changed (6) hide show
  1. chat_template.jinja +70 -93
  2. config.json +183 -28
  3. generation_config.json +8 -8
  4. processor_config.json +75 -0
  5. tokenizer.json +2 -2
  6. tokenizer_config.json +139 -1916
chat_template.jinja CHANGED
@@ -1,93 +1,70 @@
1
- {{- bos_token }}
2
- {%- if custom_tools is defined %}
3
- {%- set tools = custom_tools %}
4
- {%- endif %}
5
- {%- if not tools_in_user_message is defined %}
6
- {%- set tools_in_user_message = true %}
7
- {%- endif %}
8
- {%- if not date_string is defined %}
9
- {%- if strftime_now is defined %}
10
- {%- set date_string = strftime_now("%d %b %Y") %}
11
- {%- else %}
12
- {%- set date_string = "26 Jul 2024" %}
13
- {%- endif %}
14
- {%- endif %}
15
- {%- if not tools is defined %}
16
- {%- set tools = none %}
17
- {%- endif %}
18
-
19
- {#- This block extracts the system message, so we can slot it into the right place. #}
20
- {%- if messages[0]['role'] == 'system' %}
21
- {%- set system_message = messages[0]['content']|trim %}
22
- {%- set messages = messages[1:] %}
23
- {%- else %}
24
- {%- set system_message = "" %}
25
- {%- endif %}
26
-
27
- {#- System message #}
28
- {{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
29
- {%- if tools is not none %}
30
- {{- "Environment: ipython\n" }}
31
- {%- endif %}
32
- {{- "Cutting Knowledge Date: December 2023\n" }}
33
- {{- "Today Date: " + date_string + "\n\n" }}
34
- {%- if tools is not none and not tools_in_user_message %}
35
- {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
36
- {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
37
- {{- "Do not use variables.\n\n" }}
38
- {%- for t in tools %}
39
- {{- t | tojson(indent=4) }}
40
- {{- "\n\n" }}
41
- {%- endfor %}
42
- {%- endif %}
43
- {{- system_message }}
44
- {{- "<|eot_id|>" }}
45
-
46
- {#- Custom tools are passed in a user message with some extra guidance #}
47
- {%- if tools_in_user_message and not tools is none %}
48
- {#- Extract the first user message so we can plug it in here #}
49
- {%- if messages | length != 0 %}
50
- {%- set first_user_message = messages[0]['content']|trim %}
51
- {%- set messages = messages[1:] %}
52
- {%- else %}
53
- {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
54
- {%- endif %}
55
- {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
56
- {{- "Given the following functions, please respond with a JSON for a function call " }}
57
- {{- "with its proper arguments that best answers the given prompt.\n\n" }}
58
- {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
59
- {{- "Do not use variables.\n\n" }}
60
- {%- for t in tools %}
61
- {{- t | tojson(indent=4) }}
62
- {{- "\n\n" }}
63
- {%- endfor %}
64
- {{- first_user_message + "<|eot_id|>"}}
65
- {%- endif %}
66
-
67
- {%- for message in messages %}
68
- {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
69
- {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
70
- {%- elif 'tool_calls' in message %}
71
- {%- if not message.tool_calls|length == 1 %}
72
- {{- raise_exception("This model only supports single tool-calls at once!") }}
73
- {%- endif %}
74
- {%- set tool_call = message.tool_calls[0].function %}
75
- {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
76
- {{- '{"name": "' + tool_call.name + '", ' }}
77
- {{- '"parameters": ' }}
78
- {{- tool_call.arguments | tojson }}
79
- {{- "}" }}
80
- {{- "<|eot_id|>" }}
81
- {%- elif message.role == "tool" or message.role == "ipython" %}
82
- {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
83
- {%- if message.content is mapping or message.content is iterable %}
84
- {{- message.content | tojson }}
85
- {%- else %}
86
- {{- message.content }}
87
- {%- endif %}
88
- {{- "<|eot_id|>" }}
89
- {%- endif %}
90
- {%- endfor %}
91
- {%- if add_generation_prompt %}
92
- {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
93
- {%- endif %}
 
1
+ {{ bos_token }}{%- macro strip_thinking(text) -%}
2
+ {%- set ns = namespace(result='') -%}
3
+ {%- for part in text.split('<channel|>') -%}
4
+ {%- if '<|channel>' in part -%}
5
+ {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}
6
+ {%- else -%}
7
+ {%- set ns.result = ns.result + part -%}
8
+ {%- endif -%}
9
+ {%- endfor -%}
10
+ {{- ns.result | trim -}}
11
+ {%- endmacro -%}
12
+ {%- set thinking = enable_thinking is defined and enable_thinking -%}
13
+ {%- set loop_messages = messages -%}
14
+ {%- if messages[0]['role'] in ['system', 'developer'] or thinking -%}
15
+ {{ '<|turn>system
16
+ ' }}
17
+ {%- if thinking -%}
18
+ {{ '<|think|>
19
+ ' }}
20
+ {%- endif -%}
21
+ {%- if messages[0]['role'] in ['system', 'developer'] -%}
22
+ {{ messages[0]['content'] | trim }}
23
+ {%- set loop_messages = messages[1:] -%}
24
+ {%- endif -%}
25
+ {{ '<turn|>
26
+ ' }}
27
+ {%- endif -%}
28
+ {%- for message in loop_messages -%}
29
+ {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
30
+ {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
31
+ {%- endif -%}
32
+ {%- if (message['role'] == 'assistant') -%}
33
+ {%- set role = "model" -%}
34
+ {%- else -%}
35
+ {%- set role = message['role'] -%}
36
+ {%- endif -%}
37
+ {{ '<|turn>' + role + '
38
+ ' }}
39
+ {%- if message['content'] is string -%}
40
+ {%- if role == "model" -%}
41
+ {{ strip_thinking(message['content']) }}
42
+ {%- else -%}
43
+ {{ message['content'] | trim }}
44
+ {%- endif -%}
45
+ {%- elif message['content'] is iterable -%}
46
+ {%- for item in message['content'] -%}
47
+ {%- if item['type'] == 'audio' -%}
48
+ {{ '<|audio|>' }}
49
+ {%- elif item['type'] == 'image' -%}
50
+ {{ '<|image|>' }}
51
+ {%- elif item['type'] == 'video' -%}
52
+ {{ '<|video|>' }}
53
+ {%- elif item['type'] == 'text' -%}
54
+ {%- if role == "model" -%}
55
+ {{ strip_thinking(item['text']) }}
56
+ {%- else -%}
57
+ {{ item['text'] | trim }}
58
+ {%- endif -%}
59
+ {%- endif -%}
60
+ {%- endfor -%}
61
+ {%- else -%}
62
+ {{ raise_exception("Invalid content type") }}
63
+ {%- endif -%}
64
+ {{ '<turn|>
65
+ ' }}
66
+ {%- endfor -%}
67
+ {%- if add_generation_prompt -%}
68
+ {{'<|turn>model
69
+ '}}
70
+ {%- endif -%}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
config.json CHANGED
@@ -1,38 +1,193 @@
1
  {
2
  "architectures": [
3
- "LlamaForCausalLM"
4
  ],
5
- "attention_bias": false,
6
- "attention_dropout": 0.0,
7
- "bos_token_id": 128000,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8
  "torch_dtype": "float16",
9
- "eos_token_id": 128009,
10
- "head_dim": 64,
11
- "hidden_act": "silu",
12
- "hidden_size": 2048,
 
13
  "initializer_range": 0.02,
14
- "intermediate_size": 8192,
15
- "max_position_embeddings": 131072,
16
- "mlp_bias": false,
17
- "model_name": "unsloth/llama-3.2-1b-instruct-unsloth-bnb-4bit",
18
- "model_type": "llama",
19
- "num_attention_heads": 32,
20
- "num_hidden_layers": 16,
21
- "num_key_value_heads": 8,
22
- "pad_token_id": 128004,
23
- "pretraining_tp": 1,
24
- "rms_norm_eps": 1e-05,
25
- "rope_parameters": {
26
- "factor": 32.0,
27
- "high_freq_factor": 4.0,
28
- "low_freq_factor": 1.0,
29
- "original_max_position_embeddings": 8192,
30
- "rope_theta": 500000.0,
31
- "rope_type": "llama3"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
32
  },
33
  "tie_word_embeddings": true,
34
  "unsloth_fixed": true,
35
  "unsloth_version": "2026.6.9",
36
- "use_cache": true,
37
- "vocab_size": 128256
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
38
  }
 
1
  {
2
  "architectures": [
3
+ "Gemma4ForConditionalGeneration"
4
  ],
5
+ "audio_config": {
6
+ "_name_or_path": "",
7
+ "architectures": null,
8
+ "attention_chunk_size": 12,
9
+ "attention_context_left": 13,
10
+ "attention_context_right": 0,
11
+ "attention_invalid_logits_value": -1000000000.0,
12
+ "attention_logit_cap": 50.0,
13
+ "chunk_size_feed_forward": 0,
14
+ "conv_kernel_size": 5,
15
+ "torch_dtype": "float16",
16
+ "gradient_clipping": 10000000000.0,
17
+ "hidden_act": "silu",
18
+ "hidden_size": 1024,
19
+ "id2label": {
20
+ "0": "LABEL_0",
21
+ "1": "LABEL_1"
22
+ },
23
+ "initializer_range": 0.02,
24
+ "is_encoder_decoder": false,
25
+ "label2id": {
26
+ "LABEL_0": 0,
27
+ "LABEL_1": 1
28
+ },
29
+ "model_type": "gemma4_audio",
30
+ "num_attention_heads": 8,
31
+ "num_hidden_layers": 12,
32
+ "output_attentions": false,
33
+ "output_hidden_states": false,
34
+ "output_proj_dims": 1536,
35
+ "problem_type": null,
36
+ "residual_weight": 0.5,
37
+ "return_dict": true,
38
+ "rms_norm_eps": 1e-06,
39
+ "subsampling_conv_channels": [
40
+ 128,
41
+ 32
42
+ ],
43
+ "use_clipped_linears": true
44
+ },
45
+ "audio_token_id": 258881,
46
+ "boa_token_id": 256000,
47
+ "boi_token_id": 255999,
48
+ "bos_token_id": 2,
49
  "torch_dtype": "float16",
50
+ "eoa_token_id": 258883,
51
+ "eoa_token_index": 258883,
52
+ "eoi_token_id": 258882,
53
+ "eos_token_id": 1,
54
+ "image_token_id": 258880,
55
  "initializer_range": 0.02,
56
+ "model_name": "unsloth/gemma-4-e2b-it-unsloth-bnb-4bit",
57
+ "model_type": "gemma4",
58
+ "pad_token_id": 0,
59
+ "text_config": {
60
+ "attention_bias": false,
61
+ "attention_dropout": 0.0,
62
+ "attention_k_eq_v": false,
63
+ "bos_token_id": 2,
64
+ "torch_dtype": "float16",
65
+ "enable_moe_block": false,
66
+ "eos_token_id": 1,
67
+ "expert_intermediate_size": null,
68
+ "final_logit_softcapping": 30.0,
69
+ "global_head_dim": 512,
70
+ "head_dim": 256,
71
+ "hidden_activation": "gelu_pytorch_tanh",
72
+ "hidden_size": 1536,
73
+ "hidden_size_per_layer_input": 256,
74
+ "initializer_range": 0.02,
75
+ "intermediate_size": 6144,
76
+ "layer_types": [
77
+ "sliding_attention",
78
+ "sliding_attention",
79
+ "sliding_attention",
80
+ "sliding_attention",
81
+ "full_attention",
82
+ "sliding_attention",
83
+ "sliding_attention",
84
+ "sliding_attention",
85
+ "sliding_attention",
86
+ "full_attention",
87
+ "sliding_attention",
88
+ "sliding_attention",
89
+ "sliding_attention",
90
+ "sliding_attention",
91
+ "full_attention",
92
+ "sliding_attention",
93
+ "sliding_attention",
94
+ "sliding_attention",
95
+ "sliding_attention",
96
+ "full_attention",
97
+ "sliding_attention",
98
+ "sliding_attention",
99
+ "sliding_attention",
100
+ "sliding_attention",
101
+ "full_attention",
102
+ "sliding_attention",
103
+ "sliding_attention",
104
+ "sliding_attention",
105
+ "sliding_attention",
106
+ "full_attention",
107
+ "sliding_attention",
108
+ "sliding_attention",
109
+ "sliding_attention",
110
+ "sliding_attention",
111
+ "full_attention"
112
+ ],
113
+ "max_position_embeddings": 131072,
114
+ "model_type": "gemma4_text",
115
+ "moe_intermediate_size": null,
116
+ "num_attention_heads": 8,
117
+ "num_experts": null,
118
+ "num_global_key_value_heads": null,
119
+ "num_hidden_layers": 35,
120
+ "num_key_value_heads": 1,
121
+ "num_kv_shared_layers": 20,
122
+ "pad_token_id": 0,
123
+ "rms_norm_eps": 1e-06,
124
+ "rope_parameters": {
125
+ "full_attention": {
126
+ "partial_rotary_factor": 0.25,
127
+ "rope_theta": 1000000.0,
128
+ "rope_type": "proportional"
129
+ },
130
+ "sliding_attention": {
131
+ "rope_theta": 10000.0,
132
+ "rope_type": "default"
133
+ }
134
+ },
135
+ "sliding_window": 512,
136
+ "tie_word_embeddings": true,
137
+ "top_k_experts": null,
138
+ "use_bidirectional_attention": null,
139
+ "use_cache": true,
140
+ "use_double_wide_mlp": true,
141
+ "vocab_size": 262144,
142
+ "vocab_size_per_layer_input": 262144
143
  },
144
  "tie_word_embeddings": true,
145
  "unsloth_fixed": true,
146
  "unsloth_version": "2026.6.9",
147
+ "use_cache": false,
148
+ "video_token_id": 258884,
149
+ "vision_config": {
150
+ "_name_or_path": "",
151
+ "architectures": null,
152
+ "attention_bias": false,
153
+ "attention_dropout": 0.0,
154
+ "chunk_size_feed_forward": 0,
155
+ "default_output_length": 280,
156
+ "torch_dtype": "float16",
157
+ "global_head_dim": 64,
158
+ "head_dim": 64,
159
+ "hidden_activation": "gelu_pytorch_tanh",
160
+ "hidden_size": 768,
161
+ "id2label": {
162
+ "0": "LABEL_0",
163
+ "1": "LABEL_1"
164
+ },
165
+ "initializer_range": 0.02,
166
+ "intermediate_size": 3072,
167
+ "is_encoder_decoder": false,
168
+ "label2id": {
169
+ "LABEL_0": 0,
170
+ "LABEL_1": 1
171
+ },
172
+ "max_position_embeddings": 131072,
173
+ "model_type": "gemma4_vision",
174
+ "num_attention_heads": 12,
175
+ "num_hidden_layers": 16,
176
+ "num_key_value_heads": 12,
177
+ "output_attentions": false,
178
+ "output_hidden_states": false,
179
+ "patch_size": 16,
180
+ "pooling_kernel_size": 3,
181
+ "position_embedding_size": 10240,
182
+ "problem_type": null,
183
+ "return_dict": true,
184
+ "rms_norm_eps": 1e-06,
185
+ "rope_parameters": {
186
+ "rope_theta": 100.0,
187
+ "rope_type": "default"
188
+ },
189
+ "standardize": false,
190
+ "use_clipped_linears": true
191
+ },
192
+ "vision_soft_tokens_per_image": 280
193
  }
generation_config.json CHANGED
@@ -1,14 +1,14 @@
1
  {
2
- "bos_token_id": 128000,
3
  "do_sample": true,
4
  "eos_token_id": [
5
- 128001,
6
- 128008,
7
- 128009
8
  ],
9
- "max_length": 131072,
10
- "pad_token_id": 128004,
11
- "temperature": 0.6,
12
- "top_p": 0.9,
13
  "transformers_version": "5.5.0"
14
  }
 
1
  {
2
+ "bos_token_id": 2,
3
  "do_sample": true,
4
  "eos_token_id": [
5
+ 1,
6
+ 106,
7
+ 50
8
  ],
9
+ "pad_token_id": 0,
10
+ "temperature": 1.0,
11
+ "top_k": 64,
12
+ "top_p": 0.95,
13
  "transformers_version": "5.5.0"
14
  }
processor_config.json ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "audio_ms_per_token": 40,
3
+ "audio_seq_length": 750,
4
+ "feature_extractor": {
5
+ "dither": 0.0,
6
+ "feature_extractor_type": "Gemma4AudioFeatureExtractor",
7
+ "feature_size": 128,
8
+ "fft_length": 512,
9
+ "fft_overdrive": false,
10
+ "frame_length": 320,
11
+ "hop_length": 160,
12
+ "input_scale_factor": 1.0,
13
+ "max_frequency": 8000.0,
14
+ "mel_floor": 0.001,
15
+ "min_frequency": 0.0,
16
+ "padding_side": "right",
17
+ "padding_value": 0.0,
18
+ "per_bin_mean": null,
19
+ "per_bin_stddev": null,
20
+ "preemphasis": 0.0,
21
+ "preemphasis_htk_flavor": true,
22
+ "return_attention_mask": true,
23
+ "sampling_rate": 16000
24
+ },
25
+ "image_processor": {
26
+ "do_convert_rgb": true,
27
+ "do_normalize": false,
28
+ "do_rescale": true,
29
+ "do_resize": true,
30
+ "image_mean": [
31
+ 0.0,
32
+ 0.0,
33
+ 0.0
34
+ ],
35
+ "image_processor_type": "Gemma4ImageProcessor",
36
+ "image_seq_length": 280,
37
+ "image_std": [
38
+ 1.0,
39
+ 1.0,
40
+ 1.0
41
+ ],
42
+ "max_soft_tokens": 280,
43
+ "patch_size": 16,
44
+ "pooling_kernel_size": 3,
45
+ "resample": 3,
46
+ "rescale_factor": 0.00392156862745098
47
+ },
48
+ "image_seq_length": 280,
49
+ "processor_class": "Gemma4Processor",
50
+ "video_processor": {
51
+ "do_convert_rgb": true,
52
+ "do_normalize": true,
53
+ "do_rescale": true,
54
+ "do_resize": true,
55
+ "do_sample_frames": true,
56
+ "image_mean": [
57
+ 0.0,
58
+ 0.0,
59
+ 0.0
60
+ ],
61
+ "image_std": [
62
+ 1.0,
63
+ 1.0,
64
+ 1.0
65
+ ],
66
+ "max_soft_tokens": 70,
67
+ "num_frames": 32,
68
+ "patch_size": 16,
69
+ "pooling_kernel_size": 3,
70
+ "resample": 3,
71
+ "rescale_factor": 0.00392156862745098,
72
+ "return_metadata": false,
73
+ "video_processor_type": "Gemma4VideoProcessor"
74
+ }
75
+ }
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
3
- size 17209920
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b1d8bced9d66859cd2c3f4dcd8ab427197d4d46af3d9598b72af9fcf80b8392e
3
+ size 32169781
tokenizer_config.json CHANGED
@@ -1,2061 +1,284 @@
1
  {
 
2
  "backend": "tokenizers",
3
- "bos_token": "<|begin_of_text|>",
4
- "clean_up_tokenization_spaces": true,
5
- "eos_token": "<|eot_id|>",
6
- "is_local": false,
7
- "model_input_names": [
8
- "input_ids",
9
- "attention_mask"
 
 
 
 
 
 
 
10
  ],
 
 
 
11
  "model_max_length": 131072,
12
- "pad_token": "<|finetune_right_pad_id|>",
13
- "padding_side": "left",
14
- "tokenizer_class": "TokenizersBackend",
15
- "unk_token": null,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16
  "added_tokens_decoder": {
17
- "128000": {
18
- "content": "<|begin_of_text|>",
19
- "single_word": false,
20
- "lstrip": false,
21
- "rstrip": false,
22
- "normalized": false,
23
- "special": true
24
- },
25
- "128001": {
26
- "content": "<|end_of_text|>",
27
- "single_word": false,
28
- "lstrip": false,
29
- "rstrip": false,
30
- "normalized": false,
31
- "special": true
32
- },
33
- "128002": {
34
- "content": "<|reserved_special_token_0|>",
35
- "single_word": false,
36
- "lstrip": false,
37
- "rstrip": false,
38
- "normalized": false,
39
- "special": true
40
- },
41
- "128003": {
42
- "content": "<|reserved_special_token_1|>",
43
- "single_word": false,
44
- "lstrip": false,
45
- "rstrip": false,
46
- "normalized": false,
47
- "special": true
48
- },
49
- "128004": {
50
- "content": "<|finetune_right_pad_id|>",
51
- "single_word": false,
52
- "lstrip": false,
53
- "rstrip": false,
54
- "normalized": false,
55
- "special": true
56
- },
57
- "128005": {
58
- "content": "<|reserved_special_token_2|>",
59
- "single_word": false,
60
- "lstrip": false,
61
- "rstrip": false,
62
- "normalized": false,
63
- "special": true
64
- },
65
- "128006": {
66
- "content": "<|start_header_id|>",
67
- "single_word": false,
68
- "lstrip": false,
69
- "rstrip": false,
70
- "normalized": false,
71
- "special": true
72
- },
73
- "128007": {
74
- "content": "<|end_header_id|>",
75
- "single_word": false,
76
- "lstrip": false,
77
- "rstrip": false,
78
- "normalized": false,
79
- "special": true
80
- },
81
- "128008": {
82
- "content": "<|eom_id|>",
83
- "single_word": false,
84
- "lstrip": false,
85
- "rstrip": false,
86
- "normalized": false,
87
- "special": true
88
- },
89
- "128009": {
90
- "content": "<|eot_id|>",
91
- "single_word": false,
92
- "lstrip": false,
93
- "rstrip": false,
94
- "normalized": false,
95
- "special": true
96
- },
97
- "128010": {
98
- "content": "<|python_tag|>",
99
- "single_word": false,
100
- "lstrip": false,
101
- "rstrip": false,
102
- "normalized": false,
103
- "special": true
104
- },
105
- "128011": {
106
- "content": "<|reserved_special_token_3|>",
107
- "single_word": false,
108
- "lstrip": false,
109
- "rstrip": false,
110
- "normalized": false,
111
- "special": true
112
- },
113
- "128012": {
114
- "content": "<|reserved_special_token_4|>",
115
- "single_word": false,
116
- "lstrip": false,
117
- "rstrip": false,
118
- "normalized": false,
119
- "special": true
120
- },
121
- "128013": {
122
- "content": "<|reserved_special_token_5|>",
123
- "single_word": false,
124
- "lstrip": false,
125
- "rstrip": false,
126
- "normalized": false,
127
- "special": true
128
- },
129
- "128014": {
130
- "content": "<|reserved_special_token_6|>",
131
- "single_word": false,
132
- "lstrip": false,
133
- "rstrip": false,
134
- "normalized": false,
135
- "special": true
136
- },
137
- "128015": {
138
- "content": "<|reserved_special_token_7|>",
139
- "single_word": false,
140
- "lstrip": false,
141
- "rstrip": false,
142
- "normalized": false,
143
- "special": true
144
- },
145
- "128016": {
146
- "content": "<|reserved_special_token_8|>",
147
- "single_word": false,
148
- "lstrip": false,
149
- "rstrip": false,
150
- "normalized": false,
151
- "special": true
152
- },
153
- "128017": {
154
- "content": "<|reserved_special_token_9|>",
155
- "single_word": false,
156
- "lstrip": false,
157
- "rstrip": false,
158
- "normalized": false,
159
- "special": true
160
- },
161
- "128018": {
162
- "content": "<|reserved_special_token_10|>",
163
- "single_word": false,
164
- "lstrip": false,
165
- "rstrip": false,
166
- "normalized": false,
167
- "special": true
168
- },
169
- "128019": {
170
- "content": "<|reserved_special_token_11|>",
171
- "single_word": false,
172
- "lstrip": false,
173
- "rstrip": false,
174
- "normalized": false,
175
- "special": true
176
- },
177
- "128020": {
178
- "content": "<|reserved_special_token_12|>",
179
- "single_word": false,
180
- "lstrip": false,
181
- "rstrip": false,
182
- "normalized": false,
183
- "special": true
184
- },
185
- "128021": {
186
- "content": "<|reserved_special_token_13|>",
187
- "single_word": false,
188
- "lstrip": false,
189
- "rstrip": false,
190
- "normalized": false,
191
- "special": true
192
- },
193
- "128022": {
194
- "content": "<|reserved_special_token_14|>",
195
- "single_word": false,
196
- "lstrip": false,
197
- "rstrip": false,
198
- "normalized": false,
199
- "special": true
200
- },
201
- "128023": {
202
- "content": "<|reserved_special_token_15|>",
203
- "single_word": false,
204
- "lstrip": false,
205
- "rstrip": false,
206
- "normalized": false,
207
- "special": true
208
- },
209
- "128024": {
210
- "content": "<|reserved_special_token_16|>",
211
- "single_word": false,
212
- "lstrip": false,
213
- "rstrip": false,
214
- "normalized": false,
215
- "special": true
216
- },
217
- "128025": {
218
- "content": "<|reserved_special_token_17|>",
219
- "single_word": false,
220
- "lstrip": false,
221
- "rstrip": false,
222
- "normalized": false,
223
- "special": true
224
- },
225
- "128026": {
226
- "content": "<|reserved_special_token_18|>",
227
- "single_word": false,
228
- "lstrip": false,
229
- "rstrip": false,
230
- "normalized": false,
231
- "special": true
232
- },
233
- "128027": {
234
- "content": "<|reserved_special_token_19|>",
235
- "single_word": false,
236
- "lstrip": false,
237
- "rstrip": false,
238
- "normalized": false,
239
- "special": true
240
- },
241
- "128028": {
242
- "content": "<|reserved_special_token_20|>",
243
- "single_word": false,
244
- "lstrip": false,
245
- "rstrip": false,
246
- "normalized": false,
247
- "special": true
248
- },
249
- "128029": {
250
- "content": "<|reserved_special_token_21|>",
251
- "single_word": false,
252
- "lstrip": false,
253
- "rstrip": false,
254
- "normalized": false,
255
- "special": true
256
- },
257
- "128030": {
258
- "content": "<|reserved_special_token_22|>",
259
- "single_word": false,
260
- "lstrip": false,
261
- "rstrip": false,
262
- "normalized": false,
263
- "special": true
264
- },
265
- "128031": {
266
- "content": "<|reserved_special_token_23|>",
267
- "single_word": false,
268
- "lstrip": false,
269
- "rstrip": false,
270
- "normalized": false,
271
- "special": true
272
- },
273
- "128032": {
274
- "content": "<|reserved_special_token_24|>",
275
- "single_word": false,
276
- "lstrip": false,
277
- "rstrip": false,
278
- "normalized": false,
279
- "special": true
280
- },
281
- "128033": {
282
- "content": "<|reserved_special_token_25|>",
283
- "single_word": false,
284
- "lstrip": false,
285
- "rstrip": false,
286
- "normalized": false,
287
- "special": true
288
- },
289
- "128034": {
290
- "content": "<|reserved_special_token_26|>",
291
- "single_word": false,
292
- "lstrip": false,
293
- "rstrip": false,
294
- "normalized": false,
295
- "special": true
296
- },
297
- "128035": {
298
- "content": "<|reserved_special_token_27|>",
299
- "single_word": false,
300
- "lstrip": false,
301
- "rstrip": false,
302
- "normalized": false,
303
- "special": true
304
- },
305
- "128036": {
306
- "content": "<|reserved_special_token_28|>",
307
- "single_word": false,
308
- "lstrip": false,
309
- "rstrip": false,
310
- "normalized": false,
311
- "special": true
312
- },
313
- "128037": {
314
- "content": "<|reserved_special_token_29|>",
315
- "single_word": false,
316
- "lstrip": false,
317
- "rstrip": false,
318
- "normalized": false,
319
- "special": true
320
- },
321
- "128038": {
322
- "content": "<|reserved_special_token_30|>",
323
- "single_word": false,
324
- "lstrip": false,
325
- "rstrip": false,
326
- "normalized": false,
327
- "special": true
328
- },
329
- "128039": {
330
- "content": "<|reserved_special_token_31|>",
331
- "single_word": false,
332
- "lstrip": false,
333
- "rstrip": false,
334
- "normalized": false,
335
- "special": true
336
- },
337
- "128040": {
338
- "content": "<|reserved_special_token_32|>",
339
- "single_word": false,
340
- "lstrip": false,
341
- "rstrip": false,
342
- "normalized": false,
343
- "special": true
344
- },
345
- "128041": {
346
- "content": "<|reserved_special_token_33|>",
347
- "single_word": false,
348
- "lstrip": false,
349
- "rstrip": false,
350
- "normalized": false,
351
- "special": true
352
- },
353
- "128042": {
354
- "content": "<|reserved_special_token_34|>",
355
- "single_word": false,
356
- "lstrip": false,
357
- "rstrip": false,
358
- "normalized": false,
359
- "special": true
360
- },
361
- "128043": {
362
- "content": "<|reserved_special_token_35|>",
363
- "single_word": false,
364
- "lstrip": false,
365
- "rstrip": false,
366
- "normalized": false,
367
- "special": true
368
- },
369
- "128044": {
370
- "content": "<|reserved_special_token_36|>",
371
- "single_word": false,
372
- "lstrip": false,
373
- "rstrip": false,
374
- "normalized": false,
375
- "special": true
376
- },
377
- "128045": {
378
- "content": "<|reserved_special_token_37|>",
379
- "single_word": false,
380
- "lstrip": false,
381
- "rstrip": false,
382
- "normalized": false,
383
- "special": true
384
- },
385
- "128046": {
386
- "content": "<|reserved_special_token_38|>",
387
- "single_word": false,
388
- "lstrip": false,
389
- "rstrip": false,
390
- "normalized": false,
391
- "special": true
392
- },
393
- "128047": {
394
- "content": "<|reserved_special_token_39|>",
395
- "single_word": false,
396
- "lstrip": false,
397
- "rstrip": false,
398
- "normalized": false,
399
- "special": true
400
- },
401
- "128048": {
402
- "content": "<|reserved_special_token_40|>",
403
- "single_word": false,
404
- "lstrip": false,
405
- "rstrip": false,
406
- "normalized": false,
407
- "special": true
408
- },
409
- "128049": {
410
- "content": "<|reserved_special_token_41|>",
411
- "single_word": false,
412
- "lstrip": false,
413
- "rstrip": false,
414
- "normalized": false,
415
- "special": true
416
- },
417
- "128050": {
418
- "content": "<|reserved_special_token_42|>",
419
- "single_word": false,
420
- "lstrip": false,
421
- "rstrip": false,
422
- "normalized": false,
423
- "special": true
424
- },
425
- "128051": {
426
- "content": "<|reserved_special_token_43|>",
427
- "single_word": false,
428
- "lstrip": false,
429
- "rstrip": false,
430
- "normalized": false,
431
- "special": true
432
- },
433
- "128052": {
434
- "content": "<|reserved_special_token_44|>",
435
- "single_word": false,
436
- "lstrip": false,
437
- "rstrip": false,
438
- "normalized": false,
439
- "special": true
440
- },
441
- "128053": {
442
- "content": "<|reserved_special_token_45|>",
443
- "single_word": false,
444
- "lstrip": false,
445
- "rstrip": false,
446
- "normalized": false,
447
- "special": true
448
- },
449
- "128054": {
450
- "content": "<|reserved_special_token_46|>",
451
- "single_word": false,
452
- "lstrip": false,
453
- "rstrip": false,
454
- "normalized": false,
455
- "special": true
456
- },
457
- "128055": {
458
- "content": "<|reserved_special_token_47|>",
459
- "single_word": false,
460
- "lstrip": false,
461
- "rstrip": false,
462
- "normalized": false,
463
- "special": true
464
- },
465
- "128056": {
466
- "content": "<|reserved_special_token_48|>",
467
- "single_word": false,
468
- "lstrip": false,
469
- "rstrip": false,
470
- "normalized": false,
471
- "special": true
472
- },
473
- "128057": {
474
- "content": "<|reserved_special_token_49|>",
475
- "single_word": false,
476
- "lstrip": false,
477
- "rstrip": false,
478
- "normalized": false,
479
- "special": true
480
- },
481
- "128058": {
482
- "content": "<|reserved_special_token_50|>",
483
- "single_word": false,
484
- "lstrip": false,
485
- "rstrip": false,
486
- "normalized": false,
487
- "special": true
488
- },
489
- "128059": {
490
- "content": "<|reserved_special_token_51|>",
491
- "single_word": false,
492
- "lstrip": false,
493
- "rstrip": false,
494
- "normalized": false,
495
- "special": true
496
- },
497
- "128060": {
498
- "content": "<|reserved_special_token_52|>",
499
- "single_word": false,
500
- "lstrip": false,
501
- "rstrip": false,
502
- "normalized": false,
503
- "special": true
504
- },
505
- "128061": {
506
- "content": "<|reserved_special_token_53|>",
507
- "single_word": false,
508
- "lstrip": false,
509
- "rstrip": false,
510
- "normalized": false,
511
- "special": true
512
- },
513
- "128062": {
514
- "content": "<|reserved_special_token_54|>",
515
- "single_word": false,
516
- "lstrip": false,
517
- "rstrip": false,
518
- "normalized": false,
519
- "special": true
520
- },
521
- "128063": {
522
- "content": "<|reserved_special_token_55|>",
523
- "single_word": false,
524
- "lstrip": false,
525
- "rstrip": false,
526
- "normalized": false,
527
- "special": true
528
- },
529
- "128064": {
530
- "content": "<|reserved_special_token_56|>",
531
- "single_word": false,
532
- "lstrip": false,
533
- "rstrip": false,
534
- "normalized": false,
535
- "special": true
536
- },
537
- "128065": {
538
- "content": "<|reserved_special_token_57|>",
539
- "single_word": false,
540
- "lstrip": false,
541
- "rstrip": false,
542
- "normalized": false,
543
- "special": true
544
- },
545
- "128066": {
546
- "content": "<|reserved_special_token_58|>",
547
- "single_word": false,
548
- "lstrip": false,
549
- "rstrip": false,
550
- "normalized": false,
551
- "special": true
552
- },
553
- "128067": {
554
- "content": "<|reserved_special_token_59|>",
555
- "single_word": false,
556
- "lstrip": false,
557
- "rstrip": false,
558
- "normalized": false,
559
- "special": true
560
- },
561
- "128068": {
562
- "content": "<|reserved_special_token_60|>",
563
- "single_word": false,
564
- "lstrip": false,
565
- "rstrip": false,
566
- "normalized": false,
567
- "special": true
568
- },
569
- "128069": {
570
- "content": "<|reserved_special_token_61|>",
571
- "single_word": false,
572
- "lstrip": false,
573
- "rstrip": false,
574
- "normalized": false,
575
- "special": true
576
- },
577
- "128070": {
578
- "content": "<|reserved_special_token_62|>",
579
- "single_word": false,
580
- "lstrip": false,
581
- "rstrip": false,
582
- "normalized": false,
583
- "special": true
584
- },
585
- "128071": {
586
- "content": "<|reserved_special_token_63|>",
587
- "single_word": false,
588
- "lstrip": false,
589
- "rstrip": false,
590
- "normalized": false,
591
- "special": true
592
- },
593
- "128072": {
594
- "content": "<|reserved_special_token_64|>",
595
- "single_word": false,
596
- "lstrip": false,
597
- "rstrip": false,
598
- "normalized": false,
599
- "special": true
600
- },
601
- "128073": {
602
- "content": "<|reserved_special_token_65|>",
603
- "single_word": false,
604
- "lstrip": false,
605
- "rstrip": false,
606
- "normalized": false,
607
- "special": true
608
- },
609
- "128074": {
610
- "content": "<|reserved_special_token_66|>",
611
- "single_word": false,
612
- "lstrip": false,
613
- "rstrip": false,
614
- "normalized": false,
615
- "special": true
616
- },
617
- "128075": {
618
- "content": "<|reserved_special_token_67|>",
619
- "single_word": false,
620
- "lstrip": false,
621
- "rstrip": false,
622
- "normalized": false,
623
- "special": true
624
- },
625
- "128076": {
626
- "content": "<|reserved_special_token_68|>",
627
- "single_word": false,
628
- "lstrip": false,
629
- "rstrip": false,
630
- "normalized": false,
631
- "special": true
632
- },
633
- "128077": {
634
- "content": "<|reserved_special_token_69|>",
635
- "single_word": false,
636
- "lstrip": false,
637
- "rstrip": false,
638
- "normalized": false,
639
- "special": true
640
- },
641
- "128078": {
642
- "content": "<|reserved_special_token_70|>",
643
- "single_word": false,
644
- "lstrip": false,
645
- "rstrip": false,
646
- "normalized": false,
647
- "special": true
648
- },
649
- "128079": {
650
- "content": "<|reserved_special_token_71|>",
651
- "single_word": false,
652
- "lstrip": false,
653
- "rstrip": false,
654
- "normalized": false,
655
- "special": true
656
- },
657
- "128080": {
658
- "content": "<|reserved_special_token_72|>",
659
- "single_word": false,
660
- "lstrip": false,
661
- "rstrip": false,
662
- "normalized": false,
663
- "special": true
664
- },
665
- "128081": {
666
- "content": "<|reserved_special_token_73|>",
667
- "single_word": false,
668
- "lstrip": false,
669
- "rstrip": false,
670
- "normalized": false,
671
- "special": true
672
- },
673
- "128082": {
674
- "content": "<|reserved_special_token_74|>",
675
- "single_word": false,
676
- "lstrip": false,
677
- "rstrip": false,
678
- "normalized": false,
679
- "special": true
680
- },
681
- "128083": {
682
- "content": "<|reserved_special_token_75|>",
683
- "single_word": false,
684
- "lstrip": false,
685
- "rstrip": false,
686
- "normalized": false,
687
- "special": true
688
- },
689
- "128084": {
690
- "content": "<|reserved_special_token_76|>",
691
- "single_word": false,
692
- "lstrip": false,
693
- "rstrip": false,
694
- "normalized": false,
695
- "special": true
696
- },
697
- "128085": {
698
- "content": "<|reserved_special_token_77|>",
699
- "single_word": false,
700
- "lstrip": false,
701
- "rstrip": false,
702
- "normalized": false,
703
- "special": true
704
- },
705
- "128086": {
706
- "content": "<|reserved_special_token_78|>",
707
- "single_word": false,
708
- "lstrip": false,
709
- "rstrip": false,
710
- "normalized": false,
711
- "special": true
712
- },
713
- "128087": {
714
- "content": "<|reserved_special_token_79|>",
715
- "single_word": false,
716
- "lstrip": false,
717
- "rstrip": false,
718
- "normalized": false,
719
- "special": true
720
- },
721
- "128088": {
722
- "content": "<|reserved_special_token_80|>",
723
- "single_word": false,
724
- "lstrip": false,
725
- "rstrip": false,
726
- "normalized": false,
727
- "special": true
728
- },
729
- "128089": {
730
- "content": "<|reserved_special_token_81|>",
731
- "single_word": false,
732
- "lstrip": false,
733
- "rstrip": false,
734
- "normalized": false,
735
- "special": true
736
- },
737
- "128090": {
738
- "content": "<|reserved_special_token_82|>",
739
- "single_word": false,
740
- "lstrip": false,
741
- "rstrip": false,
742
- "normalized": false,
743
- "special": true
744
- },
745
- "128091": {
746
- "content": "<|reserved_special_token_83|>",
747
- "single_word": false,
748
- "lstrip": false,
749
- "rstrip": false,
750
- "normalized": false,
751
- "special": true
752
- },
753
- "128092": {
754
- "content": "<|reserved_special_token_84|>",
755
- "single_word": false,
756
- "lstrip": false,
757
- "rstrip": false,
758
- "normalized": false,
759
- "special": true
760
- },
761
- "128093": {
762
- "content": "<|reserved_special_token_85|>",
763
- "single_word": false,
764
- "lstrip": false,
765
- "rstrip": false,
766
- "normalized": false,
767
- "special": true
768
- },
769
- "128094": {
770
- "content": "<|reserved_special_token_86|>",
771
- "single_word": false,
772
- "lstrip": false,
773
- "rstrip": false,
774
- "normalized": false,
775
- "special": true
776
- },
777
- "128095": {
778
- "content": "<|reserved_special_token_87|>",
779
- "single_word": false,
780
- "lstrip": false,
781
- "rstrip": false,
782
- "normalized": false,
783
- "special": true
784
- },
785
- "128096": {
786
- "content": "<|reserved_special_token_88|>",
787
- "single_word": false,
788
- "lstrip": false,
789
- "rstrip": false,
790
- "normalized": false,
791
- "special": true
792
- },
793
- "128097": {
794
- "content": "<|reserved_special_token_89|>",
795
- "single_word": false,
796
- "lstrip": false,
797
- "rstrip": false,
798
- "normalized": false,
799
- "special": true
800
- },
801
- "128098": {
802
- "content": "<|reserved_special_token_90|>",
803
- "single_word": false,
804
- "lstrip": false,
805
- "rstrip": false,
806
- "normalized": false,
807
- "special": true
808
- },
809
- "128099": {
810
- "content": "<|reserved_special_token_91|>",
811
- "single_word": false,
812
- "lstrip": false,
813
- "rstrip": false,
814
- "normalized": false,
815
- "special": true
816
- },
817
- "128100": {
818
- "content": "<|reserved_special_token_92|>",
819
- "single_word": false,
820
- "lstrip": false,
821
- "rstrip": false,
822
- "normalized": false,
823
- "special": true
824
- },
825
- "128101": {
826
- "content": "<|reserved_special_token_93|>",
827
- "single_word": false,
828
- "lstrip": false,
829
- "rstrip": false,
830
- "normalized": false,
831
- "special": true
832
- },
833
- "128102": {
834
- "content": "<|reserved_special_token_94|>",
835
- "single_word": false,
836
- "lstrip": false,
837
- "rstrip": false,
838
- "normalized": false,
839
- "special": true
840
- },
841
- "128103": {
842
- "content": "<|reserved_special_token_95|>",
843
- "single_word": false,
844
- "lstrip": false,
845
- "rstrip": false,
846
- "normalized": false,
847
- "special": true
848
- },
849
- "128104": {
850
- "content": "<|reserved_special_token_96|>",
851
- "single_word": false,
852
- "lstrip": false,
853
- "rstrip": false,
854
- "normalized": false,
855
- "special": true
856
- },
857
- "128105": {
858
- "content": "<|reserved_special_token_97|>",
859
- "single_word": false,
860
- "lstrip": false,
861
- "rstrip": false,
862
- "normalized": false,
863
- "special": true
864
- },
865
- "128106": {
866
- "content": "<|reserved_special_token_98|>",
867
- "single_word": false,
868
- "lstrip": false,
869
- "rstrip": false,
870
- "normalized": false,
871
- "special": true
872
- },
873
- "128107": {
874
- "content": "<|reserved_special_token_99|>",
875
- "single_word": false,
876
- "lstrip": false,
877
- "rstrip": false,
878
- "normalized": false,
879
- "special": true
880
- },
881
- "128108": {
882
- "content": "<|reserved_special_token_100|>",
883
- "single_word": false,
884
- "lstrip": false,
885
- "rstrip": false,
886
- "normalized": false,
887
- "special": true
888
- },
889
- "128109": {
890
- "content": "<|reserved_special_token_101|>",
891
- "single_word": false,
892
- "lstrip": false,
893
- "rstrip": false,
894
- "normalized": false,
895
- "special": true
896
- },
897
- "128110": {
898
- "content": "<|reserved_special_token_102|>",
899
- "single_word": false,
900
- "lstrip": false,
901
- "rstrip": false,
902
- "normalized": false,
903
- "special": true
904
- },
905
- "128111": {
906
- "content": "<|reserved_special_token_103|>",
907
- "single_word": false,
908
- "lstrip": false,
909
- "rstrip": false,
910
- "normalized": false,
911
- "special": true
912
- },
913
- "128112": {
914
- "content": "<|reserved_special_token_104|>",
915
- "single_word": false,
916
- "lstrip": false,
917
- "rstrip": false,
918
- "normalized": false,
919
- "special": true
920
- },
921
- "128113": {
922
- "content": "<|reserved_special_token_105|>",
923
- "single_word": false,
924
- "lstrip": false,
925
- "rstrip": false,
926
- "normalized": false,
927
- "special": true
928
- },
929
- "128114": {
930
- "content": "<|reserved_special_token_106|>",
931
- "single_word": false,
932
- "lstrip": false,
933
- "rstrip": false,
934
- "normalized": false,
935
- "special": true
936
- },
937
- "128115": {
938
- "content": "<|reserved_special_token_107|>",
939
- "single_word": false,
940
- "lstrip": false,
941
- "rstrip": false,
942
- "normalized": false,
943
- "special": true
944
- },
945
- "128116": {
946
- "content": "<|reserved_special_token_108|>",
947
- "single_word": false,
948
- "lstrip": false,
949
- "rstrip": false,
950
- "normalized": false,
951
- "special": true
952
- },
953
- "128117": {
954
- "content": "<|reserved_special_token_109|>",
955
- "single_word": false,
956
- "lstrip": false,
957
- "rstrip": false,
958
- "normalized": false,
959
- "special": true
960
- },
961
- "128118": {
962
- "content": "<|reserved_special_token_110|>",
963
- "single_word": false,
964
- "lstrip": false,
965
- "rstrip": false,
966
- "normalized": false,
967
- "special": true
968
- },
969
- "128119": {
970
- "content": "<|reserved_special_token_111|>",
971
- "single_word": false,
972
- "lstrip": false,
973
- "rstrip": false,
974
- "normalized": false,
975
- "special": true
976
- },
977
- "128120": {
978
- "content": "<|reserved_special_token_112|>",
979
- "single_word": false,
980
- "lstrip": false,
981
- "rstrip": false,
982
- "normalized": false,
983
- "special": true
984
- },
985
- "128121": {
986
- "content": "<|reserved_special_token_113|>",
987
- "single_word": false,
988
- "lstrip": false,
989
- "rstrip": false,
990
- "normalized": false,
991
- "special": true
992
- },
993
- "128122": {
994
- "content": "<|reserved_special_token_114|>",
995
- "single_word": false,
996
- "lstrip": false,
997
- "rstrip": false,
998
- "normalized": false,
999
- "special": true
1000
- },
1001
- "128123": {
1002
- "content": "<|reserved_special_token_115|>",
1003
- "single_word": false,
1004
- "lstrip": false,
1005
- "rstrip": false,
1006
- "normalized": false,
1007
- "special": true
1008
- },
1009
- "128124": {
1010
- "content": "<|reserved_special_token_116|>",
1011
- "single_word": false,
1012
- "lstrip": false,
1013
- "rstrip": false,
1014
- "normalized": false,
1015
- "special": true
1016
- },
1017
- "128125": {
1018
- "content": "<|reserved_special_token_117|>",
1019
- "single_word": false,
1020
- "lstrip": false,
1021
- "rstrip": false,
1022
- "normalized": false,
1023
- "special": true
1024
- },
1025
- "128126": {
1026
- "content": "<|reserved_special_token_118|>",
1027
- "single_word": false,
1028
- "lstrip": false,
1029
- "rstrip": false,
1030
- "normalized": false,
1031
- "special": true
1032
- },
1033
- "128127": {
1034
- "content": "<|reserved_special_token_119|>",
1035
- "single_word": false,
1036
- "lstrip": false,
1037
- "rstrip": false,
1038
- "normalized": false,
1039
- "special": true
1040
- },
1041
- "128128": {
1042
- "content": "<|reserved_special_token_120|>",
1043
- "single_word": false,
1044
- "lstrip": false,
1045
- "rstrip": false,
1046
- "normalized": false,
1047
- "special": true
1048
- },
1049
- "128129": {
1050
- "content": "<|reserved_special_token_121|>",
1051
- "single_word": false,
1052
- "lstrip": false,
1053
- "rstrip": false,
1054
- "normalized": false,
1055
- "special": true
1056
- },
1057
- "128130": {
1058
- "content": "<|reserved_special_token_122|>",
1059
- "single_word": false,
1060
- "lstrip": false,
1061
- "rstrip": false,
1062
- "normalized": false,
1063
- "special": true
1064
- },
1065
- "128131": {
1066
- "content": "<|reserved_special_token_123|>",
1067
- "single_word": false,
1068
- "lstrip": false,
1069
- "rstrip": false,
1070
- "normalized": false,
1071
- "special": true
1072
- },
1073
- "128132": {
1074
- "content": "<|reserved_special_token_124|>",
1075
- "single_word": false,
1076
- "lstrip": false,
1077
- "rstrip": false,
1078
- "normalized": false,
1079
- "special": true
1080
- },
1081
- "128133": {
1082
- "content": "<|reserved_special_token_125|>",
1083
- "single_word": false,
1084
- "lstrip": false,
1085
- "rstrip": false,
1086
- "normalized": false,
1087
- "special": true
1088
- },
1089
- "128134": {
1090
- "content": "<|reserved_special_token_126|>",
1091
- "single_word": false,
1092
- "lstrip": false,
1093
- "rstrip": false,
1094
- "normalized": false,
1095
- "special": true
1096
- },
1097
- "128135": {
1098
- "content": "<|reserved_special_token_127|>",
1099
- "single_word": false,
1100
- "lstrip": false,
1101
- "rstrip": false,
1102
- "normalized": false,
1103
- "special": true
1104
- },
1105
- "128136": {
1106
- "content": "<|reserved_special_token_128|>",
1107
- "single_word": false,
1108
- "lstrip": false,
1109
- "rstrip": false,
1110
- "normalized": false,
1111
- "special": true
1112
- },
1113
- "128137": {
1114
- "content": "<|reserved_special_token_129|>",
1115
- "single_word": false,
1116
- "lstrip": false,
1117
- "rstrip": false,
1118
- "normalized": false,
1119
- "special": true
1120
- },
1121
- "128138": {
1122
- "content": "<|reserved_special_token_130|>",
1123
- "single_word": false,
1124
- "lstrip": false,
1125
- "rstrip": false,
1126
- "normalized": false,
1127
- "special": true
1128
- },
1129
- "128139": {
1130
- "content": "<|reserved_special_token_131|>",
1131
- "single_word": false,
1132
- "lstrip": false,
1133
- "rstrip": false,
1134
- "normalized": false,
1135
- "special": true
1136
- },
1137
- "128140": {
1138
- "content": "<|reserved_special_token_132|>",
1139
- "single_word": false,
1140
- "lstrip": false,
1141
- "rstrip": false,
1142
- "normalized": false,
1143
- "special": true
1144
- },
1145
- "128141": {
1146
- "content": "<|reserved_special_token_133|>",
1147
- "single_word": false,
1148
- "lstrip": false,
1149
- "rstrip": false,
1150
- "normalized": false,
1151
- "special": true
1152
- },
1153
- "128142": {
1154
- "content": "<|reserved_special_token_134|>",
1155
- "single_word": false,
1156
- "lstrip": false,
1157
- "rstrip": false,
1158
- "normalized": false,
1159
- "special": true
1160
- },
1161
- "128143": {
1162
- "content": "<|reserved_special_token_135|>",
1163
- "single_word": false,
1164
- "lstrip": false,
1165
- "rstrip": false,
1166
- "normalized": false,
1167
- "special": true
1168
- },
1169
- "128144": {
1170
- "content": "<|reserved_special_token_136|>",
1171
- "single_word": false,
1172
- "lstrip": false,
1173
- "rstrip": false,
1174
- "normalized": false,
1175
- "special": true
1176
- },
1177
- "128145": {
1178
- "content": "<|reserved_special_token_137|>",
1179
- "single_word": false,
1180
- "lstrip": false,
1181
- "rstrip": false,
1182
- "normalized": false,
1183
- "special": true
1184
- },
1185
- "128146": {
1186
- "content": "<|reserved_special_token_138|>",
1187
- "single_word": false,
1188
- "lstrip": false,
1189
- "rstrip": false,
1190
- "normalized": false,
1191
- "special": true
1192
- },
1193
- "128147": {
1194
- "content": "<|reserved_special_token_139|>",
1195
- "single_word": false,
1196
- "lstrip": false,
1197
- "rstrip": false,
1198
- "normalized": false,
1199
- "special": true
1200
- },
1201
- "128148": {
1202
- "content": "<|reserved_special_token_140|>",
1203
- "single_word": false,
1204
- "lstrip": false,
1205
- "rstrip": false,
1206
- "normalized": false,
1207
- "special": true
1208
- },
1209
- "128149": {
1210
- "content": "<|reserved_special_token_141|>",
1211
- "single_word": false,
1212
- "lstrip": false,
1213
- "rstrip": false,
1214
- "normalized": false,
1215
- "special": true
1216
- },
1217
- "128150": {
1218
- "content": "<|reserved_special_token_142|>",
1219
- "single_word": false,
1220
- "lstrip": false,
1221
- "rstrip": false,
1222
- "normalized": false,
1223
- "special": true
1224
- },
1225
- "128151": {
1226
- "content": "<|reserved_special_token_143|>",
1227
- "single_word": false,
1228
- "lstrip": false,
1229
- "rstrip": false,
1230
- "normalized": false,
1231
- "special": true
1232
- },
1233
- "128152": {
1234
- "content": "<|reserved_special_token_144|>",
1235
- "single_word": false,
1236
- "lstrip": false,
1237
- "rstrip": false,
1238
- "normalized": false,
1239
- "special": true
1240
- },
1241
- "128153": {
1242
- "content": "<|reserved_special_token_145|>",
1243
- "single_word": false,
1244
- "lstrip": false,
1245
- "rstrip": false,
1246
- "normalized": false,
1247
- "special": true
1248
- },
1249
- "128154": {
1250
- "content": "<|reserved_special_token_146|>",
1251
- "single_word": false,
1252
- "lstrip": false,
1253
- "rstrip": false,
1254
- "normalized": false,
1255
- "special": true
1256
- },
1257
- "128155": {
1258
- "content": "<|reserved_special_token_147|>",
1259
- "single_word": false,
1260
- "lstrip": false,
1261
- "rstrip": false,
1262
- "normalized": false,
1263
- "special": true
1264
- },
1265
- "128156": {
1266
- "content": "<|reserved_special_token_148|>",
1267
- "single_word": false,
1268
- "lstrip": false,
1269
- "rstrip": false,
1270
- "normalized": false,
1271
- "special": true
1272
- },
1273
- "128157": {
1274
- "content": "<|reserved_special_token_149|>",
1275
- "single_word": false,
1276
- "lstrip": false,
1277
- "rstrip": false,
1278
- "normalized": false,
1279
- "special": true
1280
- },
1281
- "128158": {
1282
- "content": "<|reserved_special_token_150|>",
1283
- "single_word": false,
1284
- "lstrip": false,
1285
- "rstrip": false,
1286
- "normalized": false,
1287
- "special": true
1288
- },
1289
- "128159": {
1290
- "content": "<|reserved_special_token_151|>",
1291
- "single_word": false,
1292
- "lstrip": false,
1293
- "rstrip": false,
1294
- "normalized": false,
1295
- "special": true
1296
- },
1297
- "128160": {
1298
- "content": "<|reserved_special_token_152|>",
1299
- "single_word": false,
1300
- "lstrip": false,
1301
- "rstrip": false,
1302
- "normalized": false,
1303
- "special": true
1304
- },
1305
- "128161": {
1306
- "content": "<|reserved_special_token_153|>",
1307
- "single_word": false,
1308
- "lstrip": false,
1309
- "rstrip": false,
1310
- "normalized": false,
1311
- "special": true
1312
- },
1313
- "128162": {
1314
- "content": "<|reserved_special_token_154|>",
1315
- "single_word": false,
1316
- "lstrip": false,
1317
- "rstrip": false,
1318
- "normalized": false,
1319
- "special": true
1320
- },
1321
- "128163": {
1322
- "content": "<|reserved_special_token_155|>",
1323
- "single_word": false,
1324
- "lstrip": false,
1325
- "rstrip": false,
1326
- "normalized": false,
1327
- "special": true
1328
- },
1329
- "128164": {
1330
- "content": "<|reserved_special_token_156|>",
1331
- "single_word": false,
1332
- "lstrip": false,
1333
- "rstrip": false,
1334
- "normalized": false,
1335
- "special": true
1336
- },
1337
- "128165": {
1338
- "content": "<|reserved_special_token_157|>",
1339
- "single_word": false,
1340
- "lstrip": false,
1341
- "rstrip": false,
1342
- "normalized": false,
1343
- "special": true
1344
- },
1345
- "128166": {
1346
- "content": "<|reserved_special_token_158|>",
1347
- "single_word": false,
1348
- "lstrip": false,
1349
- "rstrip": false,
1350
- "normalized": false,
1351
- "special": true
1352
- },
1353
- "128167": {
1354
- "content": "<|reserved_special_token_159|>",
1355
- "single_word": false,
1356
- "lstrip": false,
1357
- "rstrip": false,
1358
- "normalized": false,
1359
- "special": true
1360
- },
1361
- "128168": {
1362
- "content": "<|reserved_special_token_160|>",
1363
- "single_word": false,
1364
- "lstrip": false,
1365
- "rstrip": false,
1366
- "normalized": false,
1367
- "special": true
1368
- },
1369
- "128169": {
1370
- "content": "<|reserved_special_token_161|>",
1371
- "single_word": false,
1372
- "lstrip": false,
1373
- "rstrip": false,
1374
- "normalized": false,
1375
- "special": true
1376
- },
1377
- "128170": {
1378
- "content": "<|reserved_special_token_162|>",
1379
- "single_word": false,
1380
- "lstrip": false,
1381
- "rstrip": false,
1382
- "normalized": false,
1383
- "special": true
1384
- },
1385
- "128171": {
1386
- "content": "<|reserved_special_token_163|>",
1387
- "single_word": false,
1388
- "lstrip": false,
1389
- "rstrip": false,
1390
- "normalized": false,
1391
- "special": true
1392
- },
1393
- "128172": {
1394
- "content": "<|reserved_special_token_164|>",
1395
- "single_word": false,
1396
- "lstrip": false,
1397
- "rstrip": false,
1398
- "normalized": false,
1399
- "special": true
1400
- },
1401
- "128173": {
1402
- "content": "<|reserved_special_token_165|>",
1403
- "single_word": false,
1404
- "lstrip": false,
1405
- "rstrip": false,
1406
- "normalized": false,
1407
- "special": true
1408
- },
1409
- "128174": {
1410
- "content": "<|reserved_special_token_166|>",
1411
- "single_word": false,
1412
- "lstrip": false,
1413
- "rstrip": false,
1414
- "normalized": false,
1415
- "special": true
1416
- },
1417
- "128175": {
1418
- "content": "<|reserved_special_token_167|>",
1419
- "single_word": false,
1420
- "lstrip": false,
1421
- "rstrip": false,
1422
- "normalized": false,
1423
- "special": true
1424
- },
1425
- "128176": {
1426
- "content": "<|reserved_special_token_168|>",
1427
- "single_word": false,
1428
- "lstrip": false,
1429
- "rstrip": false,
1430
- "normalized": false,
1431
- "special": true
1432
- },
1433
- "128177": {
1434
- "content": "<|reserved_special_token_169|>",
1435
- "single_word": false,
1436
- "lstrip": false,
1437
- "rstrip": false,
1438
- "normalized": false,
1439
- "special": true
1440
- },
1441
- "128178": {
1442
- "content": "<|reserved_special_token_170|>",
1443
- "single_word": false,
1444
- "lstrip": false,
1445
- "rstrip": false,
1446
- "normalized": false,
1447
- "special": true
1448
- },
1449
- "128179": {
1450
- "content": "<|reserved_special_token_171|>",
1451
- "single_word": false,
1452
- "lstrip": false,
1453
- "rstrip": false,
1454
- "normalized": false,
1455
- "special": true
1456
- },
1457
- "128180": {
1458
- "content": "<|reserved_special_token_172|>",
1459
- "single_word": false,
1460
- "lstrip": false,
1461
- "rstrip": false,
1462
- "normalized": false,
1463
- "special": true
1464
- },
1465
- "128181": {
1466
- "content": "<|reserved_special_token_173|>",
1467
- "single_word": false,
1468
- "lstrip": false,
1469
- "rstrip": false,
1470
- "normalized": false,
1471
- "special": true
1472
- },
1473
- "128182": {
1474
- "content": "<|reserved_special_token_174|>",
1475
- "single_word": false,
1476
- "lstrip": false,
1477
- "rstrip": false,
1478
- "normalized": false,
1479
- "special": true
1480
- },
1481
- "128183": {
1482
- "content": "<|reserved_special_token_175|>",
1483
- "single_word": false,
1484
- "lstrip": false,
1485
- "rstrip": false,
1486
- "normalized": false,
1487
- "special": true
1488
- },
1489
- "128184": {
1490
- "content": "<|reserved_special_token_176|>",
1491
- "single_word": false,
1492
- "lstrip": false,
1493
- "rstrip": false,
1494
- "normalized": false,
1495
- "special": true
1496
- },
1497
- "128185": {
1498
- "content": "<|reserved_special_token_177|>",
1499
- "single_word": false,
1500
- "lstrip": false,
1501
- "rstrip": false,
1502
- "normalized": false,
1503
- "special": true
1504
- },
1505
- "128186": {
1506
- "content": "<|reserved_special_token_178|>",
1507
- "single_word": false,
1508
- "lstrip": false,
1509
- "rstrip": false,
1510
- "normalized": false,
1511
- "special": true
1512
- },
1513
- "128187": {
1514
- "content": "<|reserved_special_token_179|>",
1515
- "single_word": false,
1516
- "lstrip": false,
1517
- "rstrip": false,
1518
- "normalized": false,
1519
- "special": true
1520
- },
1521
- "128188": {
1522
- "content": "<|reserved_special_token_180|>",
1523
- "single_word": false,
1524
- "lstrip": false,
1525
- "rstrip": false,
1526
- "normalized": false,
1527
- "special": true
1528
- },
1529
- "128189": {
1530
- "content": "<|reserved_special_token_181|>",
1531
- "single_word": false,
1532
- "lstrip": false,
1533
- "rstrip": false,
1534
- "normalized": false,
1535
- "special": true
1536
- },
1537
- "128190": {
1538
- "content": "<|reserved_special_token_182|>",
1539
- "single_word": false,
1540
- "lstrip": false,
1541
- "rstrip": false,
1542
- "normalized": false,
1543
- "special": true
1544
- },
1545
- "128191": {
1546
- "content": "<|reserved_special_token_183|>",
1547
- "single_word": false,
1548
- "lstrip": false,
1549
- "rstrip": false,
1550
- "normalized": false,
1551
- "special": true
1552
- },
1553
- "128192": {
1554
- "content": "<|reserved_special_token_184|>",
1555
- "single_word": false,
1556
- "lstrip": false,
1557
- "rstrip": false,
1558
- "normalized": false,
1559
- "special": true
1560
- },
1561
- "128193": {
1562
- "content": "<|reserved_special_token_185|>",
1563
- "single_word": false,
1564
- "lstrip": false,
1565
- "rstrip": false,
1566
- "normalized": false,
1567
- "special": true
1568
- },
1569
- "128194": {
1570
- "content": "<|reserved_special_token_186|>",
1571
- "single_word": false,
1572
- "lstrip": false,
1573
- "rstrip": false,
1574
- "normalized": false,
1575
- "special": true
1576
- },
1577
- "128195": {
1578
- "content": "<|reserved_special_token_187|>",
1579
- "single_word": false,
1580
- "lstrip": false,
1581
- "rstrip": false,
1582
- "normalized": false,
1583
- "special": true
1584
- },
1585
- "128196": {
1586
- "content": "<|reserved_special_token_188|>",
1587
- "single_word": false,
1588
- "lstrip": false,
1589
- "rstrip": false,
1590
- "normalized": false,
1591
- "special": true
1592
- },
1593
- "128197": {
1594
- "content": "<|reserved_special_token_189|>",
1595
- "single_word": false,
1596
- "lstrip": false,
1597
- "rstrip": false,
1598
- "normalized": false,
1599
- "special": true
1600
- },
1601
- "128198": {
1602
- "content": "<|reserved_special_token_190|>",
1603
- "single_word": false,
1604
- "lstrip": false,
1605
- "rstrip": false,
1606
- "normalized": false,
1607
- "special": true
1608
- },
1609
- "128199": {
1610
- "content": "<|reserved_special_token_191|>",
1611
- "single_word": false,
1612
- "lstrip": false,
1613
- "rstrip": false,
1614
- "normalized": false,
1615
- "special": true
1616
- },
1617
- "128200": {
1618
- "content": "<|reserved_special_token_192|>",
1619
- "single_word": false,
1620
- "lstrip": false,
1621
- "rstrip": false,
1622
- "normalized": false,
1623
- "special": true
1624
- },
1625
- "128201": {
1626
- "content": "<|reserved_special_token_193|>",
1627
- "single_word": false,
1628
- "lstrip": false,
1629
- "rstrip": false,
1630
- "normalized": false,
1631
- "special": true
1632
- },
1633
- "128202": {
1634
- "content": "<|reserved_special_token_194|>",
1635
- "single_word": false,
1636
- "lstrip": false,
1637
- "rstrip": false,
1638
- "normalized": false,
1639
- "special": true
1640
- },
1641
- "128203": {
1642
- "content": "<|reserved_special_token_195|>",
1643
- "single_word": false,
1644
- "lstrip": false,
1645
- "rstrip": false,
1646
- "normalized": false,
1647
- "special": true
1648
- },
1649
- "128204": {
1650
- "content": "<|reserved_special_token_196|>",
1651
- "single_word": false,
1652
- "lstrip": false,
1653
- "rstrip": false,
1654
- "normalized": false,
1655
- "special": true
1656
- },
1657
- "128205": {
1658
- "content": "<|reserved_special_token_197|>",
1659
- "single_word": false,
1660
- "lstrip": false,
1661
- "rstrip": false,
1662
- "normalized": false,
1663
- "special": true
1664
- },
1665
- "128206": {
1666
- "content": "<|reserved_special_token_198|>",
1667
- "single_word": false,
1668
- "lstrip": false,
1669
- "rstrip": false,
1670
- "normalized": false,
1671
- "special": true
1672
- },
1673
- "128207": {
1674
- "content": "<|reserved_special_token_199|>",
1675
- "single_word": false,
1676
- "lstrip": false,
1677
- "rstrip": false,
1678
- "normalized": false,
1679
- "special": true
1680
- },
1681
- "128208": {
1682
- "content": "<|reserved_special_token_200|>",
1683
- "single_word": false,
1684
- "lstrip": false,
1685
- "rstrip": false,
1686
- "normalized": false,
1687
- "special": true
1688
- },
1689
- "128209": {
1690
- "content": "<|reserved_special_token_201|>",
1691
- "single_word": false,
1692
- "lstrip": false,
1693
- "rstrip": false,
1694
- "normalized": false,
1695
- "special": true
1696
- },
1697
- "128210": {
1698
- "content": "<|reserved_special_token_202|>",
1699
- "single_word": false,
1700
- "lstrip": false,
1701
- "rstrip": false,
1702
- "normalized": false,
1703
- "special": true
1704
- },
1705
- "128211": {
1706
- "content": "<|reserved_special_token_203|>",
1707
- "single_word": false,
1708
- "lstrip": false,
1709
- "rstrip": false,
1710
- "normalized": false,
1711
- "special": true
1712
- },
1713
- "128212": {
1714
- "content": "<|reserved_special_token_204|>",
1715
- "single_word": false,
1716
- "lstrip": false,
1717
- "rstrip": false,
1718
- "normalized": false,
1719
- "special": true
1720
- },
1721
- "128213": {
1722
- "content": "<|reserved_special_token_205|>",
1723
- "single_word": false,
1724
- "lstrip": false,
1725
- "rstrip": false,
1726
- "normalized": false,
1727
- "special": true
1728
- },
1729
- "128214": {
1730
- "content": "<|reserved_special_token_206|>",
1731
- "single_word": false,
1732
- "lstrip": false,
1733
- "rstrip": false,
1734
- "normalized": false,
1735
- "special": true
1736
- },
1737
- "128215": {
1738
- "content": "<|reserved_special_token_207|>",
1739
- "single_word": false,
1740
- "lstrip": false,
1741
- "rstrip": false,
1742
- "normalized": false,
1743
- "special": true
1744
- },
1745
- "128216": {
1746
- "content": "<|reserved_special_token_208|>",
1747
- "single_word": false,
1748
- "lstrip": false,
1749
- "rstrip": false,
1750
- "normalized": false,
1751
- "special": true
1752
- },
1753
- "128217": {
1754
- "content": "<|reserved_special_token_209|>",
1755
- "single_word": false,
1756
- "lstrip": false,
1757
- "rstrip": false,
1758
- "normalized": false,
1759
- "special": true
1760
- },
1761
- "128218": {
1762
- "content": "<|reserved_special_token_210|>",
1763
- "single_word": false,
1764
- "lstrip": false,
1765
- "rstrip": false,
1766
- "normalized": false,
1767
- "special": true
1768
- },
1769
- "128219": {
1770
- "content": "<|reserved_special_token_211|>",
1771
- "single_word": false,
1772
- "lstrip": false,
1773
- "rstrip": false,
1774
- "normalized": false,
1775
- "special": true
1776
- },
1777
- "128220": {
1778
- "content": "<|reserved_special_token_212|>",
1779
- "single_word": false,
1780
- "lstrip": false,
1781
- "rstrip": false,
1782
- "normalized": false,
1783
- "special": true
1784
- },
1785
- "128221": {
1786
- "content": "<|reserved_special_token_213|>",
1787
- "single_word": false,
1788
- "lstrip": false,
1789
- "rstrip": false,
1790
- "normalized": false,
1791
- "special": true
1792
- },
1793
- "128222": {
1794
- "content": "<|reserved_special_token_214|>",
1795
- "single_word": false,
1796
- "lstrip": false,
1797
- "rstrip": false,
1798
- "normalized": false,
1799
- "special": true
1800
- },
1801
- "128223": {
1802
- "content": "<|reserved_special_token_215|>",
1803
- "single_word": false,
1804
- "lstrip": false,
1805
- "rstrip": false,
1806
- "normalized": false,
1807
- "special": true
1808
- },
1809
- "128224": {
1810
- "content": "<|reserved_special_token_216|>",
1811
- "single_word": false,
1812
- "lstrip": false,
1813
- "rstrip": false,
1814
- "normalized": false,
1815
- "special": true
1816
- },
1817
- "128225": {
1818
- "content": "<|reserved_special_token_217|>",
1819
- "single_word": false,
1820
- "lstrip": false,
1821
- "rstrip": false,
1822
- "normalized": false,
1823
- "special": true
1824
- },
1825
- "128226": {
1826
- "content": "<|reserved_special_token_218|>",
1827
- "single_word": false,
1828
- "lstrip": false,
1829
- "rstrip": false,
1830
- "normalized": false,
1831
- "special": true
1832
- },
1833
- "128227": {
1834
- "content": "<|reserved_special_token_219|>",
1835
- "single_word": false,
1836
- "lstrip": false,
1837
- "rstrip": false,
1838
- "normalized": false,
1839
- "special": true
1840
- },
1841
- "128228": {
1842
- "content": "<|reserved_special_token_220|>",
1843
- "single_word": false,
1844
- "lstrip": false,
1845
- "rstrip": false,
1846
- "normalized": false,
1847
- "special": true
1848
- },
1849
- "128229": {
1850
- "content": "<|reserved_special_token_221|>",
1851
- "single_word": false,
1852
- "lstrip": false,
1853
- "rstrip": false,
1854
- "normalized": false,
1855
- "special": true
1856
- },
1857
- "128230": {
1858
- "content": "<|reserved_special_token_222|>",
1859
- "single_word": false,
1860
- "lstrip": false,
1861
- "rstrip": false,
1862
- "normalized": false,
1863
- "special": true
1864
- },
1865
- "128231": {
1866
- "content": "<|reserved_special_token_223|>",
1867
- "single_word": false,
1868
- "lstrip": false,
1869
- "rstrip": false,
1870
- "normalized": false,
1871
- "special": true
1872
- },
1873
- "128232": {
1874
- "content": "<|reserved_special_token_224|>",
1875
  "single_word": false,
1876
  "lstrip": false,
1877
  "rstrip": false,
1878
  "normalized": false,
1879
  "special": true
1880
  },
1881
- "128233": {
1882
- "content": "<|reserved_special_token_225|>",
1883
  "single_word": false,
1884
  "lstrip": false,
1885
  "rstrip": false,
1886
  "normalized": false,
1887
  "special": true
1888
  },
1889
- "128234": {
1890
- "content": "<|reserved_special_token_226|>",
1891
  "single_word": false,
1892
  "lstrip": false,
1893
  "rstrip": false,
1894
  "normalized": false,
1895
  "special": true
1896
  },
1897
- "128235": {
1898
- "content": "<|reserved_special_token_227|>",
1899
  "single_word": false,
1900
  "lstrip": false,
1901
  "rstrip": false,
1902
  "normalized": false,
1903
  "special": true
1904
  },
1905
- "128236": {
1906
- "content": "<|reserved_special_token_228|>",
1907
  "single_word": false,
1908
  "lstrip": false,
1909
  "rstrip": false,
1910
  "normalized": false,
1911
  "special": true
1912
  },
1913
- "128237": {
1914
- "content": "<|reserved_special_token_229|>",
1915
  "single_word": false,
1916
  "lstrip": false,
1917
  "rstrip": false,
1918
  "normalized": false,
1919
  "special": true
1920
  },
1921
- "128238": {
1922
- "content": "<|reserved_special_token_230|>",
1923
  "single_word": false,
1924
  "lstrip": false,
1925
  "rstrip": false,
1926
  "normalized": false,
1927
  "special": true
1928
  },
1929
- "128239": {
1930
- "content": "<|reserved_special_token_231|>",
1931
  "single_word": false,
1932
  "lstrip": false,
1933
  "rstrip": false,
1934
  "normalized": false,
1935
  "special": true
1936
  },
1937
- "128240": {
1938
- "content": "<|reserved_special_token_232|>",
1939
  "single_word": false,
1940
  "lstrip": false,
1941
  "rstrip": false,
1942
  "normalized": false,
1943
  "special": true
1944
  },
1945
- "128241": {
1946
- "content": "<|reserved_special_token_233|>",
1947
  "single_word": false,
1948
  "lstrip": false,
1949
  "rstrip": false,
1950
  "normalized": false,
1951
  "special": true
1952
  },
1953
- "128242": {
1954
- "content": "<|reserved_special_token_234|>",
1955
  "single_word": false,
1956
  "lstrip": false,
1957
  "rstrip": false,
1958
  "normalized": false,
1959
  "special": true
1960
  },
1961
- "128243": {
1962
- "content": "<|reserved_special_token_235|>",
1963
  "single_word": false,
1964
  "lstrip": false,
1965
  "rstrip": false,
1966
  "normalized": false,
1967
  "special": true
1968
  },
1969
- "128244": {
1970
- "content": "<|reserved_special_token_236|>",
1971
  "single_word": false,
1972
  "lstrip": false,
1973
  "rstrip": false,
1974
  "normalized": false,
1975
  "special": true
1976
  },
1977
- "128245": {
1978
- "content": "<|reserved_special_token_237|>",
1979
  "single_word": false,
1980
  "lstrip": false,
1981
  "rstrip": false,
1982
  "normalized": false,
1983
  "special": true
1984
  },
1985
- "128246": {
1986
- "content": "<|reserved_special_token_238|>",
1987
  "single_word": false,
1988
  "lstrip": false,
1989
  "rstrip": false,
1990
  "normalized": false,
1991
  "special": true
1992
  },
1993
- "128247": {
1994
- "content": "<|reserved_special_token_239|>",
1995
  "single_word": false,
1996
  "lstrip": false,
1997
  "rstrip": false,
1998
  "normalized": false,
1999
  "special": true
2000
  },
2001
- "128248": {
2002
- "content": "<|reserved_special_token_240|>",
2003
  "single_word": false,
2004
  "lstrip": false,
2005
  "rstrip": false,
2006
  "normalized": false,
2007
  "special": true
2008
  },
2009
- "128249": {
2010
- "content": "<|reserved_special_token_241|>",
2011
  "single_word": false,
2012
  "lstrip": false,
2013
  "rstrip": false,
2014
  "normalized": false,
2015
  "special": true
2016
  },
2017
- "128250": {
2018
- "content": "<|reserved_special_token_242|>",
2019
  "single_word": false,
2020
  "lstrip": false,
2021
  "rstrip": false,
2022
  "normalized": false,
2023
  "special": true
2024
  },
2025
- "128251": {
2026
- "content": "<|reserved_special_token_243|>",
2027
  "single_word": false,
2028
  "lstrip": false,
2029
  "rstrip": false,
2030
  "normalized": false,
2031
  "special": true
2032
  },
2033
- "128252": {
2034
- "content": "<|reserved_special_token_244|>",
2035
  "single_word": false,
2036
  "lstrip": false,
2037
  "rstrip": false,
2038
  "normalized": false,
2039
  "special": true
2040
  },
2041
- "128253": {
2042
- "content": "<|reserved_special_token_245|>",
2043
  "single_word": false,
2044
  "lstrip": false,
2045
  "rstrip": false,
2046
  "normalized": false,
2047
  "special": true
2048
  },
2049
- "128254": {
2050
- "content": "<|reserved_special_token_246|>",
2051
  "single_word": false,
2052
  "lstrip": false,
2053
  "rstrip": false,
2054
  "normalized": false,
2055
  "special": true
2056
  },
2057
- "128255": {
2058
- "content": "<|reserved_special_token_247|>",
2059
  "single_word": false,
2060
  "lstrip": false,
2061
  "rstrip": false,
@@ -2063,5 +286,5 @@
2063
  "special": true
2064
  }
2065
  },
2066
- "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n {%- if strftime_now is defined %}\n {%- set date_string = strftime_now(\"%d %b %Y\") %}\n {%- else %}\n {%- set date_string = \"26 Jul 2024\" %}\n {%- endif %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if tools is not none %}\n {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n {#- Extract the first user message so we can plug it in here #}\n {%- if messages | length != 0 %}\n {%- set first_user_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n {%- else %}\n {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n {%- elif 'tool_calls' in message %}\n {%- if not message.tool_calls|length == 1 %}\n {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n {%- endif %}\n {%- set tool_call = message.tool_calls[0].function %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n {{- '\"parameters\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- \"}\" }}\n {{- \"<|eot_id|>\" }}\n {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n {%- if message.content is mapping or message.content is iterable %}\n {{- message.content | tojson }}\n {%- else %}\n {{- message.content }}\n {%- endif %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n"
2067
  }
 
1
  {
2
+ "audio_token": "<|audio|>",
3
  "backend": "tokenizers",
4
+ "boa_token": "<|audio>",
5
+ "boi_token": "<|image>",
6
+ "bos_token": "<bos>",
7
+ "eoa_token": "<audio|>",
8
+ "eoc_token": "<channel|>",
9
+ "eoi_token": "<image|>",
10
+ "eos_token": "<eos>",
11
+ "eot_token": "<turn|>",
12
+ "escape_token": "<|\"|>",
13
+ "etc_token": "<tool_call|>",
14
+ "etd_token": "<tool|>",
15
+ "etr_token": "<tool_response|>",
16
+ "extra_special_tokens": [
17
+ "<|video|>"
18
  ],
19
+ "image_token": "<|image|>",
20
+ "is_local": false,
21
+ "mask_token": "<mask>",
22
  "model_max_length": 131072,
23
+ "model_specific_special_tokens": {
24
+ "audio_token": "<|audio|>",
25
+ "boa_token": "<|audio>",
26
+ "boi_token": "<|image>",
27
+ "eoa_token": "<audio|>",
28
+ "eoc_token": "<channel|>",
29
+ "eoi_token": "<image|>",
30
+ "eot_token": "<turn|>",
31
+ "escape_token": "<|\"|>",
32
+ "etc_token": "<tool_call|>",
33
+ "etd_token": "<tool|>",
34
+ "etr_token": "<tool_response|>",
35
+ "image_token": "<|image|>",
36
+ "soc_token": "<|channel>",
37
+ "sot_token": "<|turn>",
38
+ "stc_token": "<|tool_call>",
39
+ "std_token": "<|tool>",
40
+ "str_token": "<|tool_response>",
41
+ "think_token": "<|think|>"
42
+ },
43
+ "pad_token": "<pad>",
44
+ "padding_side": "right",
45
+ "processor_class": "Gemma4Processor",
46
+ "response_schema": {
47
+ "properties": {
48
+ "content": {
49
+ "type": "string"
50
+ },
51
+ "role": {
52
+ "const": "assistant"
53
+ },
54
+ "thinking": {
55
+ "type": "string"
56
+ },
57
+ "tool_calls": {
58
+ "items": {
59
+ "properties": {
60
+ "function": {
61
+ "properties": {
62
+ "arguments": {
63
+ "additionalProperties": {},
64
+ "type": "object",
65
+ "x-parser": "gemma4-tool-call"
66
+ },
67
+ "name": {
68
+ "type": "string"
69
+ }
70
+ },
71
+ "type": "object",
72
+ "x-regex": "call\\:(?P<name>\\w+)(?P<arguments>\\{.*\\})"
73
+ },
74
+ "type": {
75
+ "const": "function"
76
+ }
77
+ },
78
+ "type": "object"
79
+ },
80
+ "type": "array",
81
+ "x-regex-iterator": "<\\|tool_call>(.*?)<tool_call\\|>"
82
+ }
83
+ },
84
+ "type": "object",
85
+ "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?P<content>(?:(?!\\<turn\\|\\>)(?!\\<\\|tool_response\\>).)+)?(?:\\<turn\\|\\>|\\<\\|tool_response\\>)?"
86
+ },
87
+ "soc_token": "<|channel>",
88
+ "sot_token": "<|turn>",
89
+ "stc_token": "<|tool_call>",
90
+ "std_token": "<|tool>",
91
+ "str_token": "<|tool_response>",
92
+ "think_token": "<|think|>",
93
+ "tokenizer_class": "GemmaTokenizer",
94
+ "unk_token": "<unk>",
95
  "added_tokens_decoder": {
96
+ "0": {
97
+ "content": "<pad>",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
98
  "single_word": false,
99
  "lstrip": false,
100
  "rstrip": false,
101
  "normalized": false,
102
  "special": true
103
  },
104
+ "1": {
105
+ "content": "<eos>",
106
  "single_word": false,
107
  "lstrip": false,
108
  "rstrip": false,
109
  "normalized": false,
110
  "special": true
111
  },
112
+ "2": {
113
+ "content": "<bos>",
114
  "single_word": false,
115
  "lstrip": false,
116
  "rstrip": false,
117
  "normalized": false,
118
  "special": true
119
  },
120
+ "3": {
121
+ "content": "<unk>",
122
  "single_word": false,
123
  "lstrip": false,
124
  "rstrip": false,
125
  "normalized": false,
126
  "special": true
127
  },
128
+ "4": {
129
+ "content": "<mask>",
130
  "single_word": false,
131
  "lstrip": false,
132
  "rstrip": false,
133
  "normalized": false,
134
  "special": true
135
  },
136
+ "46": {
137
+ "content": "<|tool>",
138
  "single_word": false,
139
  "lstrip": false,
140
  "rstrip": false,
141
  "normalized": false,
142
  "special": true
143
  },
144
+ "47": {
145
+ "content": "<tool|>",
146
  "single_word": false,
147
  "lstrip": false,
148
  "rstrip": false,
149
  "normalized": false,
150
  "special": true
151
  },
152
+ "48": {
153
+ "content": "<|tool_call>",
154
  "single_word": false,
155
  "lstrip": false,
156
  "rstrip": false,
157
  "normalized": false,
158
  "special": true
159
  },
160
+ "49": {
161
+ "content": "<tool_call|>",
162
  "single_word": false,
163
  "lstrip": false,
164
  "rstrip": false,
165
  "normalized": false,
166
  "special": true
167
  },
168
+ "50": {
169
+ "content": "<|tool_response>",
170
  "single_word": false,
171
  "lstrip": false,
172
  "rstrip": false,
173
  "normalized": false,
174
  "special": true
175
  },
176
+ "51": {
177
+ "content": "<tool_response|>",
178
  "single_word": false,
179
  "lstrip": false,
180
  "rstrip": false,
181
  "normalized": false,
182
  "special": true
183
  },
184
+ "52": {
185
+ "content": "<|\"|>",
186
  "single_word": false,
187
  "lstrip": false,
188
  "rstrip": false,
189
  "normalized": false,
190
  "special": true
191
  },
192
+ "98": {
193
+ "content": "<|think|>",
194
  "single_word": false,
195
  "lstrip": false,
196
  "rstrip": false,
197
  "normalized": false,
198
  "special": true
199
  },
200
+ "100": {
201
+ "content": "<|channel>",
202
  "single_word": false,
203
  "lstrip": false,
204
  "rstrip": false,
205
  "normalized": false,
206
  "special": true
207
  },
208
+ "101": {
209
+ "content": "<channel|>",
210
  "single_word": false,
211
  "lstrip": false,
212
  "rstrip": false,
213
  "normalized": false,
214
  "special": true
215
  },
216
+ "105": {
217
+ "content": "<|turn>",
218
  "single_word": false,
219
  "lstrip": false,
220
  "rstrip": false,
221
  "normalized": false,
222
  "special": true
223
  },
224
+ "106": {
225
+ "content": "<turn|>",
226
  "single_word": false,
227
  "lstrip": false,
228
  "rstrip": false,
229
  "normalized": false,
230
  "special": true
231
  },
232
+ "255999": {
233
+ "content": "<|image>",
234
  "single_word": false,
235
  "lstrip": false,
236
  "rstrip": false,
237
  "normalized": false,
238
  "special": true
239
  },
240
+ "256000": {
241
+ "content": "<|audio>",
242
  "single_word": false,
243
  "lstrip": false,
244
  "rstrip": false,
245
  "normalized": false,
246
  "special": true
247
  },
248
+ "258880": {
249
+ "content": "<|image|>",
250
  "single_word": false,
251
  "lstrip": false,
252
  "rstrip": false,
253
  "normalized": false,
254
  "special": true
255
  },
256
+ "258881": {
257
+ "content": "<|audio|>",
258
  "single_word": false,
259
  "lstrip": false,
260
  "rstrip": false,
261
  "normalized": false,
262
  "special": true
263
  },
264
+ "258882": {
265
+ "content": "<image|>",
266
  "single_word": false,
267
  "lstrip": false,
268
  "rstrip": false,
269
  "normalized": false,
270
  "special": true
271
  },
272
+ "258883": {
273
+ "content": "<audio|>",
274
  "single_word": false,
275
  "lstrip": false,
276
  "rstrip": false,
277
  "normalized": false,
278
  "special": true
279
  },
280
+ "258884": {
281
+ "content": "<|video|>",
282
  "single_word": false,
283
  "lstrip": false,
284
  "rstrip": false,
 
286
  "special": true
287
  }
288
  },
289
+ "chat_template": "{{ bos_token }}{%- macro strip_thinking(text) -%}\n {%- set ns = namespace(result='') -%}\n {%- for part in text.split('<channel|>') -%}\n {%- if '<|channel>' in part -%}\n {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}\n {%- else -%}\n {%- set ns.result = ns.result + part -%}\n {%- endif -%}\n {%- endfor -%}\n {{- ns.result | trim -}}\n{%- endmacro -%}\n{%- set thinking = enable_thinking is defined and enable_thinking -%}\n{%- set loop_messages = messages -%}\n{%- if messages[0]['role'] in ['system', 'developer'] or thinking -%}\n {{ '<|turn>system\n' }}\n {%- if thinking -%}\n {{ '<|think|>\n' }}\n {%- endif -%}\n {%- if messages[0]['role'] in ['system', 'developer'] -%}\n {{ messages[0]['content'] | trim }}\n {%- set loop_messages = messages[1:] -%}\n {%- endif -%}\n {{ '<turn|>\n' }}\n{%- endif -%}\n{%- for message in loop_messages -%}\n {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}\n {{ raise_exception(\"Conversation roles must alternate user/assistant/user/assistant/...\") }}\n {%- endif -%}\n {%- if (message['role'] == 'assistant') -%}\n {%- set role = \"model\" -%}\n {%- else -%}\n {%- set role = message['role'] -%}\n {%- endif -%}\n {{ '<|turn>' + role + '\n' }}\n {%- if message['content'] is string -%}\n {%- if role == \"model\" -%}\n {{ strip_thinking(message['content']) }}\n {%- else -%}\n {{ message['content'] | trim }}\n {%- endif -%}\n {%- elif message['content'] is iterable -%}\n {%- for item in message['content'] -%}\n {%- if item['type'] == 'audio' -%}\n {{ '<|audio|>' }}\n {%- elif item['type'] == 'image' -%}\n {{ '<|image|>' }}\n {%- elif item['type'] == 'video' -%}\n {{ '<|video|>' }}\n {%- elif item['type'] == 'text' -%}\n {%- if role == \"model\" -%}\n {{ strip_thinking(item['text']) }}\n {%- else -%}\n {{ item['text'] | trim }}\n {%- endif -%}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {{ raise_exception(\"Invalid content type\") }}\n {%- endif -%}\n {{ '<turn|>\n' }}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{'<|turn>model\n'}}\n{%- endif -%}\n"
290
  }