Hugging Face's logo

hf-internal-testing
/
tiny-processor-glm4v

glm4v
Model card Files
xet
 Community
hf-transformers-bot commited on
Commit
14b902e
·
verified ·
1 Parent(s): 4d306e5

Add tiny processor fixture for testing

Browse files
Files changed (9) hide show
  1. .gitattributes +1 -0
  2. chat_template.jinja +33 -0
  3. config.json +66 -0
  4. generation_config.json +15 -0
  5. notebook.ipynb +0 -0
  6. preprocessor_config.json +11 -0
  7. tokenizer.json +1876 -0
  8. tokenizer_config.json +218 -0
  9. video_preprocessor_config.json +11 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
chat_template.jinja ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [gMASK]<sop>
2
+ {%- for msg in messages %}
3
+ {%- if msg.role == 'system' %}
4
+ <|system|>
5
+ {{ msg.content }}
6
+ {%- elif msg.role == 'user' %}
7
+ <|user|>{{ '\n' }}
8
+
9
+ {%- if msg.content is string %}
10
+ {{ msg.content }}
11
+ {%- else %}
12
+ {%- for item in msg.content %}
13
+ {%- if item.type == 'video' or 'video' in item %}
14
+ <|begin_of_video|><|video|><|end_of_video|>
15
+ {%- elif item.type == 'image' or 'image' in item %}
16
+ <|begin_of_image|><|image|><|end_of_image|>
17
+ {%- elif item.type == 'text' %}
18
+ {{ item.text }}
19
+ {%- endif %}
20
+ {%- endfor %}
21
+ {%- endif %}
22
+ {%- elif msg.role == 'assistant' %}
23
+ {%- if msg.metadata %}
24
+ <|assistant|>{{ msg.metadata }}
25
+ {{ msg.content }}
26
+ {%- else %}
27
+ <|assistant|>
28
+ {{ msg.content }}
29
+ {%- endif %}
30
+ {%- endif %}
31
+ {%- endfor %}
32
+ {% if add_generation_prompt %}<|assistant|>
33
+ {% endif %}
config.json ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Glm4vForConditionalGeneration"
4
+ ],
5
+ "model_type": "glm4v",
6
+ "image_start_token_id": 151339,
7
+ "image_end_token_id": 151340,
8
+ "video_start_token_id": 151341,
9
+ "video_end_token_id": 151342,
10
+ "image_token_id": 151343,
11
+ "video_token_id": 151344,
12
+ "tie_word_embeddings": false,
13
+ "transformers_version": "4.57.1",
14
+ "text_config": {
15
+ "model_type": "glm4v_text",
16
+ "attention_bias": true,
17
+ "attention_dropout": 0.0,
18
+ "pad_token_id": 151329,
19
+ "eos_token_id": [
20
+ 151329,
21
+ 151336,
22
+ 151338,
23
+ 151348
24
+ ],
25
+ "hidden_act": "silu",
26
+ "hidden_size": 4096,
27
+ "initializer_range": 0.02,
28
+ "intermediate_size": 13696,
29
+ "max_position_embeddings": 65536,
30
+ "num_attention_heads": 32,
31
+ "num_hidden_layers": 40,
32
+ "num_key_value_heads": 2,
33
+ "rms_norm_eps": 1e-05,
34
+ "dtype": "bfloat16",
35
+ "use_cache": true,
36
+ "vocab_size": 151552,
37
+ "partial_rotary_factor": 0.5,
38
+ "rope_theta": 10000,
39
+ "rope_scaling": {
40
+ "rope_type": "default",
41
+ "mrope_section": [
42
+ 8,
43
+ 12,
44
+ 12
45
+ ]
46
+ }
47
+ },
48
+ "vision_config": {
49
+ "model_type": "glm4v",
50
+ "hidden_size": 1536,
51
+ "depth": 24,
52
+ "num_heads": 12,
53
+ "attention_bias": false,
54
+ "intermediate_size": 13696,
55
+ "hidden_act": "silu",
56
+ "hidden_dropout_prob": 0.0,
57
+ "initializer_range": 0.02,
58
+ "image_size": 336,
59
+ "patch_size": 14,
60
+ "out_hidden_size": 4096,
61
+ "rms_norm_eps": 1e-05,
62
+ "spatial_merge_size": 2,
63
+ "temporal_patch_size": 2
64
+ },
65
+ "vocab_size": 500
66
+ }
generation_config.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "do_sample": true,
4
+ "eos_token_id": [
5
+ 151329,
6
+ 151336,
7
+ 151338,
8
+ 151348
9
+ ],
10
+ "pad_token_id": 151329,
11
+ "top_p": 0.6,
12
+ "temperature": 0.8,
13
+ "top_k": 2,
14
+ "transformers_version": "4.57.1"
15
+ }
notebook.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
preprocessor_config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "size": {"shortest_edge": 12544, "longest_edge": 9633792},
3
+ "do_rescale": true,
4
+ "patch_size": 14,
5
+ "temporal_patch_size": 2,
6
+ "merge_size": 2,
7
+ "image_mean": [0.48145466, 0.4578275, 0.40821073],
8
+ "image_std": [0.26862954, 0.26130258, 0.27577711],
9
+ "image_processor_type": "Glm4vImageProcessor",
10
+ "processor_class": "Glm4vProcessor"
11
+ }
tokenizer.json ADDED
@@ -0,0 +1,1876 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
5
+ "added_tokens": [
6
+ {
7
+ "id": 500,
8
+ "content": "<|endoftext|>",
9
+ "single_word": false,
10
+ "lstrip": false,
11
+ "rstrip": false,
12
+ "normalized": false,
13
+ "special": true
14
+ },
15
+ {
16
+ "id": 501,
17
+ "content": "[MASK]",
18
+ "single_word": false,
19
+ "lstrip": false,
20
+ "rstrip": false,
21
+ "normalized": false,
22
+ "special": true
23
+ },
24
+ {
25
+ "id": 502,
26
+ "content": "[gMASK]",
27
+ "single_word": false,
28
+ "lstrip": false,
29
+ "rstrip": false,
30
+ "normalized": false,
31
+ "special": true
32
+ },
33
+ {
34
+ "id": 503,
35
+ "content": "[sMASK]",
36
+ "single_word": false,
37
+ "lstrip": false,
38
+ "rstrip": false,
39
+ "normalized": false,
40
+ "special": true
41
+ },
42
+ {
43
+ "id": 504,
44
+ "content": "<sop>",
45
+ "single_word": false,
46
+ "lstrip": false,
47
+ "rstrip": false,
48
+ "normalized": false,
49
+ "special": true
50
+ },
51
+ {
52
+ "id": 505,
53
+ "content": "<eop>",
54
+ "single_word": false,
55
+ "lstrip": false,
56
+ "rstrip": false,
57
+ "normalized": false,
58
+ "special": true
59
+ },
60
+ {
61
+ "id": 506,
62
+ "content": "<|system|>",
63
+ "single_word": false,
64
+ "lstrip": false,
65
+ "rstrip": false,
66
+ "normalized": false,
67
+ "special": true
68
+ },
69
+ {
70
+ "id": 507,
71
+ "content": "<|user|>",
72
+ "single_word": false,
73
+ "lstrip": false,
74
+ "rstrip": false,
75
+ "normalized": false,
76
+ "special": true
77
+ },
78
+ {
79
+ "id": 508,
80
+ "content": "<|assistant|>",
81
+ "single_word": false,
82
+ "lstrip": false,
83
+ "rstrip": false,
84
+ "normalized": false,
85
+ "special": true
86
+ },
87
+ {
88
+ "id": 509,
89
+ "content": "<|observation|>",
90
+ "single_word": false,
91
+ "lstrip": false,
92
+ "rstrip": false,
93
+ "normalized": false,
94
+ "special": true
95
+ },
96
+ {
97
+ "id": 510,
98
+ "content": "<|begin_of_image|>",
99
+ "single_word": false,
100
+ "lstrip": false,
101
+ "rstrip": false,
102
+ "normalized": false,
103
+ "special": true
104
+ },
105
+ {
106
+ "id": 511,
107
+ "content": "<|end_of_image|>",
108
+ "single_word": false,
109
+ "lstrip": false,
110
+ "rstrip": false,
111
+ "normalized": false,
112
+ "special": true
113
+ },
114
+ {
115
+ "id": 512,
116
+ "content": "<|begin_of_video|>",
117
+ "single_word": false,
118
+ "lstrip": false,
119
+ "rstrip": false,
120
+ "normalized": false,
121
+ "special": true
122
+ },
123
+ {
124
+ "id": 513,
125
+ "content": "<|end_of_video|>",
126
+ "single_word": false,
127
+ "lstrip": false,
128
+ "rstrip": false,
129
+ "normalized": false,
130
+ "special": true
131
+ }
132
+ ],
133
+ "normalizer": null,
134
+ "pre_tokenizer": {
135
+ "type": "Sequence",
136
+ "pretokenizers": [
137
+ {
138
+ "type": "Split",
139
+ "pattern": {
140
+ "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
141
+ },
142
+ "behavior": "Isolated",
143
+ "invert": false
144
+ },
145
+ {
146
+ "type": "ByteLevel",
147
+ "add_prefix_space": false,
148
+ "trim_offsets": true,
149
+ "use_regex": false
150
+ }
151
+ ]
152
+ },
153
+ "post_processor": {
154
+ "type": "Sequence",
155
+ "processors": [
156
+ {
157
+ "type": "ByteLevel",
158
+ "add_prefix_space": true,
159
+ "trim_offsets": false,
160
+ "use_regex": true
161
+ }
162
+ ]
163
+ },
164
+ "decoder": {
165
+ "type": "ByteLevel",
166
+ "add_prefix_space": true,
167
+ "trim_offsets": true,
168
+ "use_regex": true
169
+ },
170
+ "model": {
171
+ "type": "BPE",
172
+ "dropout": null,
173
+ "unk_token": null,
174
+ "continuing_subword_prefix": null,
175
+ "end_of_word_suffix": null,
176
+ "fuse_unk": false,
177
+ "byte_fallback": false,
178
+ "ignore_merges": true,
179
+ "vocab": {
180
+ "!": 0,
181
+ "\"": 1,
182
+ "#": 2,
183
+ "$": 3,
184
+ "%": 4,
185
+ "&": 5,
186
+ "'": 6,
187
+ "(": 7,
188
+ ")": 8,
189
+ "*": 9,
190
+ "+": 10,
191
+ ",": 11,
192
+ "-": 12,
193
+ ".": 13,
194
+ "/": 14,
195
+ "0": 15,
196
+ "1": 16,
197
+ "2": 17,
198
+ "3": 18,
199
+ "4": 19,
200
+ "5": 20,
201
+ "6": 21,
202
+ "7": 22,
203
+ "8": 23,
204
+ "9": 24,
205
+ ":": 25,
206
+ ";": 26,
207
+ "<": 27,
208
+ "=": 28,
209
+ ">": 29,
210
+ "?": 30,
211
+ "@": 31,
212
+ "A": 32,
213
+ "B": 33,
214
+ "C": 34,
215
+ "D": 35,
216
+ "E": 36,
217
+ "F": 37,
218
+ "G": 38,
219
+ "H": 39,
220
+ "I": 40,
221
+ "J": 41,
222
+ "K": 42,
223
+ "L": 43,
224
+ "M": 44,
225
+ "N": 45,
226
+ "O": 46,
227
+ "P": 47,
228
+ "Q": 48,
229
+ "R": 49,
230
+ "S": 50,
231
+ "T": 51,
232
+ "U": 52,
233
+ "V": 53,
234
+ "W": 54,
235
+ "X": 55,
236
+ "Y": 56,
237
+ "Z": 57,
238
+ "[": 58,
239
+ "\\": 59,
240
+ "]": 60,
241
+ "^": 61,
242
+ "_": 62,
243
+ "`": 63,
244
+ "a": 64,
245
+ "b": 65,
246
+ "c": 66,
247
+ "d": 67,
248
+ "e": 68,
249
+ "f": 69,
250
+ "g": 70,
251
+ "h": 71,
252
+ "i": 72,
253
+ "j": 73,
254
+ "k": 74,
255
+ "l": 75,
256
+ "m": 76,
257
+ "n": 77,
258
+ "o": 78,
259
+ "p": 79,
260
+ "q": 80,
261
+ "r": 81,
262
+ "s": 82,
263
+ "t": 83,
264
+ "u": 84,
265
+ "v": 85,
266
+ "w": 86,
267
+ "x": 87,
268
+ "y": 88,
269
+ "z": 89,
270
+ "{": 90,
271
+ "|": 91,
272
+ "}": 92,
273
+ "~": 93,
274
+ "¡": 94,
275
+ "¢": 95,
276
+ "£": 96,
277
+ "¤": 97,
278
+ "¥": 98,
279
+ "¦": 99,
280
+ "§": 100,
281
+ "¨": 101,
282
+ "©": 102,
283
+ "ª": 103,
284
+ "«": 104,
285
+ "¬": 105,
286
+ "®": 106,
287
+ "¯": 107,
288
+ "°": 108,
289
+ "±": 109,
290
+ "²": 110,
291
+ "³": 111,
292
+ "´": 112,
293
+ "µ": 113,
294
+ "¶": 114,
295
+ "·": 115,
296
+ "¸": 116,
297
+ "¹": 117,
298
+ "º": 118,
299
+ "»": 119,
300
+ "¼": 120,
301
+ "½": 121,
302
+ "¾": 122,
303
+ "¿": 123,
304
+ "À": 124,
305
+ "Á": 125,
306
+ "Â": 126,
307
+ "Ã": 127,
308
+ "Ä": 128,
309
+ "Å": 129,
310
+ "Æ": 130,
311
+ "Ç": 131,
312
+ "È": 132,
313
+ "É": 133,
314
+ "Ê": 134,
315
+ "Ë": 135,
316
+ "Ì": 136,
317
+ "Í": 137,
318
+ "Î": 138,
319
+ "Ï": 139,
320
+ "Ð": 140,
321
+ "Ñ": 141,
322
+ "Ò": 142,
323
+ "Ó": 143,
324
+ "Ô": 144,
325
+ "Õ": 145,
326
+ "Ö": 146,
327
+ "×": 147,
328
+ "Ø": 148,
329
+ "Ù": 149,
330
+ "Ú": 150,
331
+ "Û": 151,
332
+ "Ü": 152,
333
+ "Ý": 153,
334
+ "Þ": 154,
335
+ "ß": 155,
336
+ "à": 156,
337
+ "á": 157,
338
+ "â": 158,
339
+ "ã": 159,
340
+ "ä": 160,
341
+ "å": 161,
342
+ "æ": 162,
343
+ "ç": 163,
344
+ "è": 164,
345
+ "é": 165,
346
+ "ê": 166,
347
+ "ë": 167,
348
+ "ì": 168,
349
+ "í": 169,
350
+ "î": 170,
351
+ "ï": 171,
352
+ "ð": 172,
353
+ "ñ": 173,
354
+ "ò": 174,
355
+ "ó": 175,
356
+ "ô": 176,
357
+ "õ": 177,
358
+ "ö": 178,
359
+ "÷": 179,
360
+ "ø": 180,
361
+ "ù": 181,
362
+ "ú": 182,
363
+ "û": 183,
364
+ "ü": 184,
365
+ "ý": 185,
366
+ "þ": 186,
367
+ "ÿ": 187,
368
+ "Ā": 188,
369
+ "ā": 189,
370
+ "Ă": 190,
371
+ "ă": 191,
372
+ "Ą": 192,
373
+ "ą": 193,
374
+ "Ć": 194,
375
+ "ć": 195,
376
+ "Ĉ": 196,
377
+ "ĉ": 197,
378
+ "Ċ": 198,
379
+ "ċ": 199,
380
+ "Č": 200,
381
+ "č": 201,
382
+ "Ď": 202,
383
+ "ď": 203,
384
+ "Đ": 204,
385
+ "đ": 205,
386
+ "Ē": 206,
387
+ "ē": 207,
388
+ "Ĕ": 208,
389
+ "ĕ": 209,
390
+ "Ė": 210,
391
+ "ė": 211,
392
+ "Ę": 212,
393
+ "ę": 213,
394
+ "Ě": 214,
395
+ "ě": 215,
396
+ "Ĝ": 216,
397
+ "ĝ": 217,
398
+ "Ğ": 218,
399
+ "ğ": 219,
400
+ "Ġ": 220,
401
+ "ġ": 221,
402
+ "Ģ": 222,
403
+ "ģ": 223,
404
+ "Ĥ": 224,
405
+ "ĥ": 225,
406
+ "Ħ": 226,
407
+ "ħ": 227,
408
+ "Ĩ": 228,
409
+ "ĩ": 229,
410
+ "Ī": 230,
411
+ "ī": 231,
412
+ "Ĭ": 232,
413
+ "ĭ": 233,
414
+ "Į": 234,
415
+ "į": 235,
416
+ "İ": 236,
417
+ "ı": 237,
418
+ "IJ": 238,
419
+ "ij": 239,
420
+ "Ĵ": 240,
421
+ "ĵ": 241,
422
+ "Ķ": 242,
423
+ "ķ": 243,
424
+ "ĸ": 244,
425
+ "Ĺ": 245,
426
+ "ĺ": 246,
427
+ "Ļ": 247,
428
+ "ļ": 248,
429
+ "Ľ": 249,
430
+ "ľ": 250,
431
+ "Ŀ": 251,
432
+ "ŀ": 252,
433
+ "Ł": 253,
434
+ "ł": 254,
435
+ "Ń": 255,
436
+ "ĠĠ": 256,
437
+ "ĠĠĠĠ": 257,
438
+ "in": 258,
439
+ "Ġt": 259,
440
+ "ĠĠĠĠĠĠĠĠ": 260,
441
+ "er": 261,
442
+ "ĠĠĠ": 262,
443
+ "on": 263,
444
+ "Ġa": 264,
445
+ "re": 265,
446
+ "at": 266,
447
+ "st": 267,
448
+ "en": 268,
449
+ "or": 269,
450
+ "Ġth": 270,
451
+ "ĊĊ": 271,
452
+ "Ġc": 272,
453
+ "le": 273,
454
+ "Ġs": 274,
455
+ "it": 275,
456
+ "an": 276,
457
+ "ar": 277,
458
+ "al": 278,
459
+ "Ġthe": 279,
460
+ ";Ċ": 280,
461
+ "Ġp": 281,
462
+ "Ġf": 282,
463
+ "ou": 283,
464
+ "Ġ=": 284,
465
+ "is": 285,
466
+ "ĠĠĠĠĠĠĠ": 286,
467
+ "ing": 287,
468
+ "es": 288,
469
+ "Ġw": 289,
470
+ "ion": 290,
471
+ "ed": 291,
472
+ "ic": 292,
473
+ "Ġb": 293,
474
+ "Ġd": 294,
475
+ "et": 295,
476
+ "Ġm": 296,
477
+ "Ġo": 297,
478
+ "ĉĉ": 298,
479
+ "ro": 299,
480
+ "as": 300,
481
+ "el": 301,
482
+ "ct": 302,
483
+ "nd": 303,
484
+ "Ġin": 304,
485
+ "Ġh": 305,
486
+ "ent": 306,
487
+ "id": 307,
488
+ "Ġn": 308,
489
+ "am": 309,
490
+ "ĠĠĠĠĠĠĠĠĠĠĠ": 310,
491
+ "Ġto": 311,
492
+ "Ġre": 312,
493
+ "--": 313,
494
+ "Ġ{": 314,
495
+ "Ġof": 315,
496
+ "om": 316,
497
+ ");Ċ": 317,
498
+ "im": 318,
499
+ "čĊ": 319,
500
+ "Ġ(": 320,
501
+ "il": 321,
502
+ "//": 322,
503
+ "Ġand": 323,
504
+ "ur": 324,
505
+ "se": 325,
506
+ "Ġl": 326,
507
+ "ex": 327,
508
+ "ĠS": 328,
509
+ "ad": 329,
510
+ "Ġ\"": 330,
511
+ "ch": 331,
512
+ "ut": 332,
513
+ "if": 333,
514
+ "**": 334,
515
+ "Ġ}": 335,
516
+ "em": 336,
517
+ "ol": 337,
518
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 338,
519
+ "th": 339,
520
+ ")Ċ": 340,
521
+ "Ġ{Ċ": 341,
522
+ "Ġg": 342,
523
+ "ig": 343,
524
+ "iv": 344,
525
+ ",Ċ": 345,
526
+ "ce": 346,
527
+ "od": 347,
528
+ "Ġv": 348,
529
+ "ate": 349,
530
+ "ĠT": 350,
531
+ "ag": 351,
532
+ "ay": 352,
533
+ "Ġ*": 353,
534
+ "ot": 354,
535
+ "us": 355,
536
+ "ĠC": 356,
537
+ "Ġst": 357,
538
+ "ĠI": 358,
539
+ "un": 359,
540
+ "ul": 360,
541
+ "ue": 361,
542
+ "ĠA": 362,
543
+ "ow": 363,
544
+ "Ġ'": 364,
545
+ "ew": 365,
546
+ "Ġ<": 366,
547
+ "ation": 367,
548
+ "()": 368,
549
+ "Ġfor": 369,
550
+ "ab": 370,
551
+ "ort": 371,
552
+ "um": 372,
553
+ "ame": 373,
554
+ "Ġis": 374,
555
+ "pe": 375,
556
+ "tr": 376,
557
+ "ck": 377,
558
+ "âĢ": 378,
559
+ "Ġy": 379,
560
+ "ist": 380,
561
+ "----": 381,
562
+ ".ĊĊ": 382,
563
+ "he": 383,
564
+ "Ġe": 384,
565
+ "lo": 385,
566
+ "ĠM": 386,
567
+ "Ġbe": 387,
568
+ "ers": 388,
569
+ "Ġon": 389,
570
+ "Ġcon": 390,
571
+ "ap": 391,
572
+ "ub": 392,
573
+ "ĠP": 393,
574
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 394,
575
+ "ass": 395,
576
+ "int": 396,
577
+ ">Ċ": 397,
578
+ "ly": 398,
579
+ "urn": 399,
580
+ "Ġ$": 400,
581
+ ";ĊĊ": 401,
582
+ "av": 402,
583
+ "port": 403,
584
+ "ir": 404,
585
+ "->": 405,
586
+ "nt": 406,
587
+ "ction": 407,
588
+ "end": 408,
589
+ "Ġde": 409,
590
+ "ith": 410,
591
+ "out": 411,
592
+ "turn": 412,
593
+ "our": 413,
594
+ "ĠĠĠĠĠ": 414,
595
+ "lic": 415,
596
+ "res": 416,
597
+ "pt": 417,
598
+ "==": 418,
599
+ "Ġthis": 419,
600
+ "Ġwh": 420,
601
+ "Ġif": 421,
602
+ "ĠD": 422,
603
+ "ver": 423,
604
+ "age": 424,
605
+ "ĠB": 425,
606
+ "ht": 426,
607
+ "ext": 427,
608
+ "=\"": 428,
609
+ "Ġthat": 429,
610
+ "****": 430,
611
+ "ĠR": 431,
612
+ "Ġit": 432,
613
+ "ess": 433,
614
+ "ĠF": 434,
615
+ "Ġr": 435,
616
+ "os": 436,
617
+ "and": 437,
618
+ "Ġas": 438,
619
+ "ect": 439,
620
+ "ke": 440,
621
+ "rom": 441,
622
+ "Ġ//": 442,
623
+ "con": 443,
624
+ "ĠL": 444,
625
+ "(\"": 445,
626
+ "qu": 446,
627
+ "lass": 447,
628
+ "Ġwith": 448,
629
+ "iz": 449,
630
+ "de": 450,
631
+ "ĠN": 451,
632
+ "Ġal": 452,
633
+ "op": 453,
634
+ "up": 454,
635
+ "get": 455,
636
+ "Ġ}Ċ": 456,
637
+ "ile": 457,
638
+ "Ġan": 458,
639
+ "ata": 459,
640
+ "ore": 460,
641
+ "ri": 461,
642
+ "Ġpro": 462,
643
+ ";čĊ": 463,
644
+ "ĉĉĉĉ": 464,
645
+ "ter": 465,
646
+ "ain": 466,
647
+ "ĠW": 467,
648
+ "ĠE": 468,
649
+ "Ġcom": 469,
650
+ "Ġreturn": 470,
651
+ "art": 471,
652
+ "ĠH": 472,
653
+ "ack": 473,
654
+ "import": 474,
655
+ "ublic": 475,
656
+ "Ġor": 476,
657
+ "est": 477,
658
+ "ment": 478,
659
+ "ĠG": 479,
660
+ "able": 480,
661
+ "Ġ-": 481,
662
+ "ine": 482,
663
+ "ill": 483,
664
+ "ind": 484,
665
+ "ere": 485,
666
+ "::": 486,
667
+ "ity": 487,
668
+ "Ġ+": 488,
669
+ "Ġtr": 489,
670
+ "elf": 490,
671
+ "ight": 491,
672
+ "('": 492,
673
+ "orm": 493,
674
+ "ult": 494,
675
+ "str": 495,
676
+ "..": 496,
677
+ "\",": 497,
678
+ "Ġyou": 498,
679
+ "ype": 499
680
+ },
681
+ "merges": [
682
+ [
683
+ "Ġ",
684
+ "Ġ"
685
+ ],
686
+ [
687
+ "Ġ",
688
+ "ĠĠĠ"
689
+ ],
690
+ [
691
+ "ĠĠ",
692
+ "ĠĠ"
693
+ ],
694
+ [
695
+ "ĠĠĠ",
696
+ "Ġ"
697
+ ],
698
+ [
699
+ "i",
700
+ "n"
701
+ ],
702
+ [
703
+ "Ġ",
704
+ "t"
705
+ ],
706
+ [
707
+ "Ġ",
708
+ "ĠĠĠĠĠĠĠ"
709
+ ],
710
+ [
711
+ "ĠĠĠĠ",
712
+ "ĠĠĠĠ"
713
+ ],
714
+ [
715
+ "ĠĠĠ",
716
+ "ĠĠĠĠĠ"
717
+ ],
718
+ [
719
+ "ĠĠĠĠĠĠĠ",
720
+ "Ġ"
721
+ ],
722
+ [
723
+ "ĠĠĠĠĠ",
724
+ "ĠĠĠ"
725
+ ],
726
+ [
727
+ "e",
728
+ "r"
729
+ ],
730
+ [
731
+ "Ġ",
732
+ "ĠĠ"
733
+ ],
734
+ [
735
+ "ĠĠ",
736
+ "Ġ"
737
+ ],
738
+ [
739
+ "o",
740
+ "n"
741
+ ],
742
+ [
743
+ "Ġ",
744
+ "a"
745
+ ],
746
+ [
747
+ "r",
748
+ "e"
749
+ ],
750
+ [
751
+ "a",
752
+ "t"
753
+ ],
754
+ [
755
+ "s",
756
+ "t"
757
+ ],
758
+ [
759
+ "e",
760
+ "n"
761
+ ],
762
+ [
763
+ "o",
764
+ "r"
765
+ ],
766
+ [
767
+ "Ġ",
768
+ "th"
769
+ ],
770
+ [
771
+ "Ġt",
772
+ "h"
773
+ ],
774
+ [
775
+ "Ċ",
776
+ "Ċ"
777
+ ],
778
+ [
779
+ "Ġ",
780
+ "c"
781
+ ],
782
+ [
783
+ "l",
784
+ "e"
785
+ ],
786
+ [
787
+ "Ġ",
788
+ "s"
789
+ ],
790
+ [
791
+ "i",
792
+ "t"
793
+ ],
794
+ [
795
+ "a",
796
+ "n"
797
+ ],
798
+ [
799
+ "a",
800
+ "r"
801
+ ],
802
+ [
803
+ "a",
804
+ "l"
805
+ ],
806
+ [
807
+ "Ġt",
808
+ "he"
809
+ ],
810
+ [
811
+ "Ġth",
812
+ "e"
813
+ ],
814
+ [
815
+ ";",
816
+ "Ċ"
817
+ ],
818
+ [
819
+ "Ġ",
820
+ "p"
821
+ ],
822
+ [
823
+ "Ġ",
824
+ "f"
825
+ ],
826
+ [
827
+ "o",
828
+ "u"
829
+ ],
830
+ [
831
+ "Ġ",
832
+ "="
833
+ ],
834
+ [
835
+ "i",
836
+ "s"
837
+ ],
838
+ [
839
+ "ĠĠ",
840
+ "ĠĠĠĠĠ"
841
+ ],
842
+ [
843
+ "ĠĠĠĠ",
844
+ "ĠĠĠ"
845
+ ],
846
+ [
847
+ "ĠĠĠ",
848
+ "ĠĠĠĠ"
849
+ ],
850
+ [
851
+ "ĠĠĠĠĠ",
852
+ "ĠĠ"
853
+ ],
854
+ [
855
+ "in",
856
+ "g"
857
+ ],
858
+ [
859
+ "e",
860
+ "s"
861
+ ],
862
+ [
863
+ "Ġ",
864
+ "w"
865
+ ],
866
+ [
867
+ "i",
868
+ "on"
869
+ ],
870
+ [
871
+ "e",
872
+ "d"
873
+ ],
874
+ [
875
+ "i",
876
+ "c"
877
+ ],
878
+ [
879
+ "Ġ",
880
+ "b"
881
+ ],
882
+ [
883
+ "Ġ",
884
+ "d"
885
+ ],
886
+ [
887
+ "e",
888
+ "t"
889
+ ],
890
+ [
891
+ "Ġ",
892
+ "m"
893
+ ],
894
+ [
895
+ "Ġ",
896
+ "o"
897
+ ],
898
+ [
899
+ "ĉ",
900
+ "ĉ"
901
+ ],
902
+ [
903
+ "r",
904
+ "o"
905
+ ],
906
+ [
907
+ "a",
908
+ "s"
909
+ ],
910
+ [
911
+ "e",
912
+ "l"
913
+ ],
914
+ [
915
+ "c",
916
+ "t"
917
+ ],
918
+ [
919
+ "n",
920
+ "d"
921
+ ],
922
+ [
923
+ "Ġ",
924
+ "in"
925
+ ],
926
+ [
927
+ "Ġ",
928
+ "h"
929
+ ],
930
+ [
931
+ "e",
932
+ "nt"
933
+ ],
934
+ [
935
+ "en",
936
+ "t"
937
+ ],
938
+ [
939
+ "i",
940
+ "d"
941
+ ],
942
+ [
943
+ "Ġ",
944
+ "n"
945
+ ],
946
+ [
947
+ "a",
948
+ "m"
949
+ ],
950
+ [
951
+ "ĠĠĠĠ",
952
+ "ĠĠĠĠĠĠĠ"
953
+ ],
954
+ [
955
+ "ĠĠĠĠĠĠĠĠ",
956
+ "ĠĠĠ"
957
+ ],
958
+ [
959
+ "ĠĠĠ",
960
+ "ĠĠĠĠĠĠĠĠ"
961
+ ],
962
+ [
963
+ "ĠĠĠĠĠĠĠ",
964
+ "ĠĠĠĠ"
965
+ ],
966
+ [
967
+ "Ġt",
968
+ "o"
969
+ ],
970
+ [
971
+ "Ġ",
972
+ "re"
973
+ ],
974
+ [
975
+ "Ġr",
976
+ "e"
977
+ ],
978
+ [
979
+ "-",
980
+ "-"
981
+ ],
982
+ [
983
+ "Ġ",
984
+ "{"
985
+ ],
986
+ [
987
+ "Ġo",
988
+ "f"
989
+ ],
990
+ [
991
+ "o",
992
+ "m"
993
+ ],
994
+ [
995
+ ")",
996
+ ";Ċ"
997
+ ],
998
+ [
999
+ "i",
1000
+ "m"
1001
+ ],
1002
+ [
1003
+ "č",
1004
+ "Ċ"
1005
+ ],
1006
+ [
1007
+ "Ġ",
1008
+ "("
1009
+ ],
1010
+ [
1011
+ "i",
1012
+ "l"
1013
+ ],
1014
+ [
1015
+ "/",
1016
+ "/"
1017
+ ],
1018
+ [
1019
+ "Ġ",
1020
+ "and"
1021
+ ],
1022
+ [
1023
+ "Ġa",
1024
+ "nd"
1025
+ ],
1026
+ [
1027
+ "Ġan",
1028
+ "d"
1029
+ ],
1030
+ [
1031
+ "u",
1032
+ "r"
1033
+ ],
1034
+ [
1035
+ "s",
1036
+ "e"
1037
+ ],
1038
+ [
1039
+ "Ġ",
1040
+ "l"
1041
+ ],
1042
+ [
1043
+ "e",
1044
+ "x"
1045
+ ],
1046
+ [
1047
+ "Ġ",
1048
+ "S"
1049
+ ],
1050
+ [
1051
+ "a",
1052
+ "d"
1053
+ ],
1054
+ [
1055
+ "Ġ",
1056
+ "\""
1057
+ ],
1058
+ [
1059
+ "c",
1060
+ "h"
1061
+ ],
1062
+ [
1063
+ "u",
1064
+ "t"
1065
+ ],
1066
+ [
1067
+ "i",
1068
+ "f"
1069
+ ],
1070
+ [
1071
+ "*",
1072
+ "*"
1073
+ ],
1074
+ [
1075
+ "Ġ",
1076
+ "}"
1077
+ ],
1078
+ [
1079
+ "e",
1080
+ "m"
1081
+ ],
1082
+ [
1083
+ "o",
1084
+ "l"
1085
+ ],
1086
+ [
1087
+ "Ġ",
1088
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ"
1089
+ ],
1090
+ [
1091
+ "ĠĠĠĠĠĠĠĠ",
1092
+ "ĠĠĠĠĠĠĠĠ"
1093
+ ],
1094
+ [
1095
+ "ĠĠĠĠĠĠĠĠĠĠĠ",
1096
+ "ĠĠĠĠĠ"
1097
+ ],
1098
+ [
1099
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ",
1100
+ "Ġ"
1101
+ ],
1102
+ [
1103
+ "ĠĠĠĠĠ",
1104
+ "ĠĠĠĠĠĠĠĠĠĠĠ"
1105
+ ],
1106
+ [
1107
+ "t",
1108
+ "h"
1109
+ ],
1110
+ [
1111
+ ")",
1112
+ "Ċ"
1113
+ ],
1114
+ [
1115
+ "Ġ{",
1116
+ "Ċ"
1117
+ ],
1118
+ [
1119
+ "Ġ",
1120
+ "g"
1121
+ ],
1122
+ [
1123
+ "i",
1124
+ "g"
1125
+ ],
1126
+ [
1127
+ "i",
1128
+ "v"
1129
+ ],
1130
+ [
1131
+ ",",
1132
+ "Ċ"
1133
+ ],
1134
+ [
1135
+ "c",
1136
+ "e"
1137
+ ],
1138
+ [
1139
+ "o",
1140
+ "d"
1141
+ ],
1142
+ [
1143
+ "Ġ",
1144
+ "v"
1145
+ ],
1146
+ [
1147
+ "at",
1148
+ "e"
1149
+ ],
1150
+ [
1151
+ "Ġ",
1152
+ "T"
1153
+ ],
1154
+ [
1155
+ "a",
1156
+ "g"
1157
+ ],
1158
+ [
1159
+ "a",
1160
+ "y"
1161
+ ],
1162
+ [
1163
+ "Ġ",
1164
+ "*"
1165
+ ],
1166
+ [
1167
+ "o",
1168
+ "t"
1169
+ ],
1170
+ [
1171
+ "u",
1172
+ "s"
1173
+ ],
1174
+ [
1175
+ "Ġ",
1176
+ "C"
1177
+ ],
1178
+ [
1179
+ "Ġ",
1180
+ "st"
1181
+ ],
1182
+ [
1183
+ "Ġs",
1184
+ "t"
1185
+ ],
1186
+ [
1187
+ "Ġ",
1188
+ "I"
1189
+ ],
1190
+ [
1191
+ "u",
1192
+ "n"
1193
+ ],
1194
+ [
1195
+ "u",
1196
+ "l"
1197
+ ],
1198
+ [
1199
+ "u",
1200
+ "e"
1201
+ ],
1202
+ [
1203
+ "Ġ",
1204
+ "A"
1205
+ ],
1206
+ [
1207
+ "o",
1208
+ "w"
1209
+ ],
1210
+ [
1211
+ "Ġ",
1212
+ "'"
1213
+ ],
1214
+ [
1215
+ "e",
1216
+ "w"
1217
+ ],
1218
+ [
1219
+ "Ġ",
1220
+ "<"
1221
+ ],
1222
+ [
1223
+ "at",
1224
+ "ion"
1225
+ ],
1226
+ [
1227
+ "(",
1228
+ ")"
1229
+ ],
1230
+ [
1231
+ "Ġf",
1232
+ "or"
1233
+ ],
1234
+ [
1235
+ "a",
1236
+ "b"
1237
+ ],
1238
+ [
1239
+ "or",
1240
+ "t"
1241
+ ],
1242
+ [
1243
+ "u",
1244
+ "m"
1245
+ ],
1246
+ [
1247
+ "am",
1248
+ "e"
1249
+ ],
1250
+ [
1251
+ "Ġ",
1252
+ "is"
1253
+ ],
1254
+ [
1255
+ "p",
1256
+ "e"
1257
+ ],
1258
+ [
1259
+ "t",
1260
+ "r"
1261
+ ],
1262
+ [
1263
+ "c",
1264
+ "k"
1265
+ ],
1266
+ [
1267
+ "â",
1268
+ "Ģ"
1269
+ ],
1270
+ [
1271
+ "Ġ",
1272
+ "y"
1273
+ ],
1274
+ [
1275
+ "i",
1276
+ "st"
1277
+ ],
1278
+ [
1279
+ "is",
1280
+ "t"
1281
+ ],
1282
+ [
1283
+ "--",
1284
+ "--"
1285
+ ],
1286
+ [
1287
+ ".",
1288
+ "ĊĊ"
1289
+ ],
1290
+ [
1291
+ "h",
1292
+ "e"
1293
+ ],
1294
+ [
1295
+ "Ġ",
1296
+ "e"
1297
+ ],
1298
+ [
1299
+ "l",
1300
+ "o"
1301
+ ],
1302
+ [
1303
+ "Ġ",
1304
+ "M"
1305
+ ],
1306
+ [
1307
+ "Ġb",
1308
+ "e"
1309
+ ],
1310
+ [
1311
+ "er",
1312
+ "s"
1313
+ ],
1314
+ [
1315
+ "Ġ",
1316
+ "on"
1317
+ ],
1318
+ [
1319
+ "Ġo",
1320
+ "n"
1321
+ ],
1322
+ [
1323
+ "Ġ",
1324
+ "con"
1325
+ ],
1326
+ [
1327
+ "Ġc",
1328
+ "on"
1329
+ ],
1330
+ [
1331
+ "a",
1332
+ "p"
1333
+ ],
1334
+ [
1335
+ "u",
1336
+ "b"
1337
+ ],
1338
+ [
1339
+ "Ġ",
1340
+ "P"
1341
+ ],
1342
+ [
1343
+ "ĠĠĠĠ",
1344
+ "ĠĠĠĠĠĠĠĠĠĠĠ"
1345
+ ],
1346
+ [
1347
+ "ĠĠĠĠĠĠĠĠ",
1348
+ "ĠĠĠĠĠĠĠ"
1349
+ ],
1350
+ [
1351
+ "ĠĠĠĠĠĠĠ",
1352
+ "ĠĠĠĠĠĠĠĠ"
1353
+ ],
1354
+ [
1355
+ "ĠĠĠĠĠĠĠĠĠĠĠ",
1356
+ "ĠĠĠĠ"
1357
+ ],
1358
+ [
1359
+ "as",
1360
+ "s"
1361
+ ],
1362
+ [
1363
+ "i",
1364
+ "nt"
1365
+ ],
1366
+ [
1367
+ "in",
1368
+ "t"
1369
+ ],
1370
+ [
1371
+ ">",
1372
+ "Ċ"
1373
+ ],
1374
+ [
1375
+ "l",
1376
+ "y"
1377
+ ],
1378
+ [
1379
+ "ur",
1380
+ "n"
1381
+ ],
1382
+ [
1383
+ "Ġ",
1384
+ "$"
1385
+ ],
1386
+ [
1387
+ ";",
1388
+ "ĊĊ"
1389
+ ],
1390
+ [
1391
+ ";Ċ",
1392
+ "Ċ"
1393
+ ],
1394
+ [
1395
+ "a",
1396
+ "v"
1397
+ ],
1398
+ [
1399
+ "p",
1400
+ "ort"
1401
+ ],
1402
+ [
1403
+ "i",
1404
+ "r"
1405
+ ],
1406
+ [
1407
+ "-",
1408
+ ">"
1409
+ ],
1410
+ [
1411
+ "n",
1412
+ "t"
1413
+ ],
1414
+ [
1415
+ "ct",
1416
+ "ion"
1417
+ ],
1418
+ [
1419
+ "e",
1420
+ "nd"
1421
+ ],
1422
+ [
1423
+ "en",
1424
+ "d"
1425
+ ],
1426
+ [
1427
+ "Ġ",
1428
+ "de"
1429
+ ],
1430
+ [
1431
+ "Ġd",
1432
+ "e"
1433
+ ],
1434
+ [
1435
+ "i",
1436
+ "th"
1437
+ ],
1438
+ [
1439
+ "it",
1440
+ "h"
1441
+ ],
1442
+ [
1443
+ "o",
1444
+ "ut"
1445
+ ],
1446
+ [
1447
+ "ou",
1448
+ "t"
1449
+ ],
1450
+ [
1451
+ "t",
1452
+ "urn"
1453
+ ],
1454
+ [
1455
+ "o",
1456
+ "ur"
1457
+ ],
1458
+ [
1459
+ "ou",
1460
+ "r"
1461
+ ],
1462
+ [
1463
+ "Ġ",
1464
+ "ĠĠĠĠ"
1465
+ ],
1466
+ [
1467
+ "ĠĠ",
1468
+ "ĠĠĠ"
1469
+ ],
1470
+ [
1471
+ "ĠĠĠĠ",
1472
+ "Ġ"
1473
+ ],
1474
+ [
1475
+ "ĠĠĠ",
1476
+ "ĠĠ"
1477
+ ],
1478
+ [
1479
+ "l",
1480
+ "ic"
1481
+ ],
1482
+ [
1483
+ "r",
1484
+ "es"
1485
+ ],
1486
+ [
1487
+ "re",
1488
+ "s"
1489
+ ],
1490
+ [
1491
+ "p",
1492
+ "t"
1493
+ ],
1494
+ [
1495
+ "=",
1496
+ "="
1497
+ ],
1498
+ [
1499
+ "Ġth",
1500
+ "is"
1501
+ ],
1502
+ [
1503
+ "Ġw",
1504
+ "h"
1505
+ ],
1506
+ [
1507
+ "Ġ",
1508
+ "if"
1509
+ ],
1510
+ [
1511
+ "Ġ",
1512
+ "D"
1513
+ ],
1514
+ [
1515
+ "v",
1516
+ "er"
1517
+ ],
1518
+ [
1519
+ "ag",
1520
+ "e"
1521
+ ],
1522
+ [
1523
+ "Ġ",
1524
+ "B"
1525
+ ],
1526
+ [
1527
+ "h",
1528
+ "t"
1529
+ ],
1530
+ [
1531
+ "ex",
1532
+ "t"
1533
+ ],
1534
+ [
1535
+ "=",
1536
+ "\""
1537
+ ],
1538
+ [
1539
+ "Ġth",
1540
+ "at"
1541
+ ],
1542
+ [
1543
+ "**",
1544
+ "**"
1545
+ ],
1546
+ [
1547
+ "Ġ",
1548
+ "R"
1549
+ ],
1550
+ [
1551
+ "Ġ",
1552
+ "it"
1553
+ ],
1554
+ [
1555
+ "es",
1556
+ "s"
1557
+ ],
1558
+ [
1559
+ "Ġ",
1560
+ "F"
1561
+ ],
1562
+ [
1563
+ "Ġ",
1564
+ "r"
1565
+ ],
1566
+ [
1567
+ "o",
1568
+ "s"
1569
+ ],
1570
+ [
1571
+ "a",
1572
+ "nd"
1573
+ ],
1574
+ [
1575
+ "an",
1576
+ "d"
1577
+ ],
1578
+ [
1579
+ "Ġ",
1580
+ "as"
1581
+ ],
1582
+ [
1583
+ "Ġa",
1584
+ "s"
1585
+ ],
1586
+ [
1587
+ "e",
1588
+ "ct"
1589
+ ],
1590
+ [
1591
+ "k",
1592
+ "e"
1593
+ ],
1594
+ [
1595
+ "r",
1596
+ "om"
1597
+ ],
1598
+ [
1599
+ "ro",
1600
+ "m"
1601
+ ],
1602
+ [
1603
+ "Ġ",
1604
+ "//"
1605
+ ],
1606
+ [
1607
+ "c",
1608
+ "on"
1609
+ ],
1610
+ [
1611
+ "Ġ",
1612
+ "L"
1613
+ ],
1614
+ [
1615
+ "(",
1616
+ "\""
1617
+ ],
1618
+ [
1619
+ "q",
1620
+ "u"
1621
+ ],
1622
+ [
1623
+ "l",
1624
+ "ass"
1625
+ ],
1626
+ [
1627
+ "Ġw",
1628
+ "ith"
1629
+ ],
1630
+ [
1631
+ "i",
1632
+ "z"
1633
+ ],
1634
+ [
1635
+ "d",
1636
+ "e"
1637
+ ],
1638
+ [
1639
+ "Ġ",
1640
+ "N"
1641
+ ],
1642
+ [
1643
+ "Ġ",
1644
+ "al"
1645
+ ],
1646
+ [
1647
+ "Ġa",
1648
+ "l"
1649
+ ],
1650
+ [
1651
+ "o",
1652
+ "p"
1653
+ ],
1654
+ [
1655
+ "u",
1656
+ "p"
1657
+ ],
1658
+ [
1659
+ "g",
1660
+ "et"
1661
+ ],
1662
+ [
1663
+ "Ġ}",
1664
+ "Ċ"
1665
+ ],
1666
+ [
1667
+ "i",
1668
+ "le"
1669
+ ],
1670
+ [
1671
+ "il",
1672
+ "e"
1673
+ ],
1674
+ [
1675
+ "Ġ",
1676
+ "an"
1677
+ ],
1678
+ [
1679
+ "Ġa",
1680
+ "n"
1681
+ ],
1682
+ [
1683
+ "at",
1684
+ "a"
1685
+ ],
1686
+ [
1687
+ "o",
1688
+ "re"
1689
+ ],
1690
+ [
1691
+ "or",
1692
+ "e"
1693
+ ],
1694
+ [
1695
+ "r",
1696
+ "i"
1697
+ ],
1698
+ [
1699
+ "Ġp",
1700
+ "ro"
1701
+ ],
1702
+ [
1703
+ ";",
1704
+ "čĊ"
1705
+ ],
1706
+ [
1707
+ "ĉĉ",
1708
+ "ĉĉ"
1709
+ ],
1710
+ [
1711
+ "t",
1712
+ "er"
1713
+ ],
1714
+ [
1715
+ "a",
1716
+ "in"
1717
+ ],
1718
+ [
1719
+ "Ġ",
1720
+ "W"
1721
+ ],
1722
+ [
1723
+ "Ġ",
1724
+ "E"
1725
+ ],
1726
+ [
1727
+ "Ġc",
1728
+ "om"
1729
+ ],
1730
+ [
1731
+ "Ġre",
1732
+ "turn"
1733
+ ],
1734
+ [
1735
+ "ar",
1736
+ "t"
1737
+ ],
1738
+ [
1739
+ "Ġ",
1740
+ "H"
1741
+ ],
1742
+ [
1743
+ "a",
1744
+ "ck"
1745
+ ],
1746
+ [
1747
+ "im",
1748
+ "port"
1749
+ ],
1750
+ [
1751
+ "ub",
1752
+ "lic"
1753
+ ],
1754
+ [
1755
+ "Ġ",
1756
+ "or"
1757
+ ],
1758
+ [
1759
+ "Ġo",
1760
+ "r"
1761
+ ],
1762
+ [
1763
+ "e",
1764
+ "st"
1765
+ ],
1766
+ [
1767
+ "es",
1768
+ "t"
1769
+ ],
1770
+ [
1771
+ "m",
1772
+ "ent"
1773
+ ],
1774
+ [
1775
+ "Ġ",
1776
+ "G"
1777
+ ],
1778
+ [
1779
+ "ab",
1780
+ "le"
1781
+ ],
1782
+ [
1783
+ "Ġ",
1784
+ "-"
1785
+ ],
1786
+ [
1787
+ "in",
1788
+ "e"
1789
+ ],
1790
+ [
1791
+ "il",
1792
+ "l"
1793
+ ],
1794
+ [
1795
+ "i",
1796
+ "nd"
1797
+ ],
1798
+ [
1799
+ "in",
1800
+ "d"
1801
+ ],
1802
+ [
1803
+ "e",
1804
+ "re"
1805
+ ],
1806
+ [
1807
+ "er",
1808
+ "e"
1809
+ ],
1810
+ [
1811
+ ":",
1812
+ ":"
1813
+ ],
1814
+ [
1815
+ "it",
1816
+ "y"
1817
+ ],
1818
+ [
1819
+ "Ġ",
1820
+ "+"
1821
+ ],
1822
+ [
1823
+ "Ġ",
1824
+ "tr"
1825
+ ],
1826
+ [
1827
+ "Ġt",
1828
+ "r"
1829
+ ],
1830
+ [
1831
+ "el",
1832
+ "f"
1833
+ ],
1834
+ [
1835
+ "ig",
1836
+ "ht"
1837
+ ],
1838
+ [
1839
+ "(",
1840
+ "'"
1841
+ ],
1842
+ [
1843
+ "or",
1844
+ "m"
1845
+ ],
1846
+ [
1847
+ "ul",
1848
+ "t"
1849
+ ],
1850
+ [
1851
+ "s",
1852
+ "tr"
1853
+ ],
1854
+ [
1855
+ "st",
1856
+ "r"
1857
+ ],
1858
+ [
1859
+ ".",
1860
+ "."
1861
+ ],
1862
+ [
1863
+ "\"",
1864
+ ","
1865
+ ],
1866
+ [
1867
+ "Ġy",
1868
+ "ou"
1869
+ ],
1870
+ [
1871
+ "y",
1872
+ "pe"
1873
+ ]
1874
+ ]
1875
+ }
1876
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,218 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "151329": {
4
+ "content": "<|endoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "151330": {
12
+ "content": "[MASK]",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "151331": {
20
+ "content": "[gMASK]",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "151332": {
28
+ "content": "[sMASK]",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "151333": {
36
+ "content": "<sop>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "151334": {
44
+ "content": "<eop>",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ },
51
+ "151335": {
52
+ "content": "<|system|>",
53
+ "lstrip": false,
54
+ "normalized": false,
55
+ "rstrip": false,
56
+ "single_word": false,
57
+ "special": true
58
+ },
59
+ "151336": {
60
+ "content": "<|user|>",
61
+ "lstrip": false,
62
+ "normalized": false,
63
+ "rstrip": false,
64
+ "single_word": false,
65
+ "special": true
66
+ },
67
+ "151337": {
68
+ "content": "<|assistant|>",
69
+ "lstrip": false,
70
+ "normalized": false,
71
+ "rstrip": false,
72
+ "single_word": false,
73
+ "special": true
74
+ },
75
+ "151338": {
76
+ "content": "<|observation|>",
77
+ "lstrip": false,
78
+ "normalized": false,
79
+ "rstrip": false,
80
+ "single_word": false,
81
+ "special": true
82
+ },
83
+ "151339": {
84
+ "content": "<|begin_of_image|>",
85
+ "lstrip": false,
86
+ "normalized": false,
87
+ "rstrip": false,
88
+ "single_word": false,
89
+ "special": true
90
+ },
91
+ "151340": {
92
+ "content": "<|end_of_image|>",
93
+ "lstrip": false,
94
+ "normalized": false,
95
+ "rstrip": false,
96
+ "single_word": false,
97
+ "special": true
98
+ },
99
+ "151341": {
100
+ "content": "<|begin_of_video|>",
101
+ "lstrip": false,
102
+ "normalized": false,
103
+ "rstrip": false,
104
+ "single_word": false,
105
+ "special": true
106
+ },
107
+ "151342": {
108
+ "content": "<|end_of_video|>",
109
+ "lstrip": false,
110
+ "normalized": false,
111
+ "rstrip": false,
112
+ "single_word": false,
113
+ "special": true
114
+ },
115
+ "151343": {
116
+ "content": "<|image|>",
117
+ "lstrip": false,
118
+ "normalized": false,
119
+ "rstrip": false,
120
+ "single_word": false,
121
+ "special": true
122
+ },
123
+ "151344": {
124
+ "content": "<|video|>",
125
+ "lstrip": false,
126
+ "normalized": false,
127
+ "rstrip": false,
128
+ "single_word": false,
129
+ "special": true
130
+ },
131
+ "151345": {
132
+ "content": "<think>",
133
+ "lstrip": false,
134
+ "normalized": false,
135
+ "rstrip": false,
136
+ "single_word": false,
137
+ "special": false
138
+ },
139
+ "151346": {
140
+ "content": "</think>",
141
+ "lstrip": false,
142
+ "normalized": false,
143
+ "rstrip": false,
144
+ "single_word": false,
145
+ "special": false
146
+ },
147
+ "151347": {
148
+ "content": "<answer>",
149
+ "lstrip": false,
150
+ "normalized": false,
151
+ "rstrip": false,
152
+ "single_word": false,
153
+ "special": false
154
+ },
155
+ "151348": {
156
+ "content": "</answer>",
157
+ "lstrip": false,
158
+ "normalized": false,
159
+ "rstrip": false,
160
+ "single_word": false,
161
+ "special": false
162
+ },
163
+ "151349": {
164
+ "content": "<|begin_of_box|>",
165
+ "lstrip": false,
166
+ "normalized": false,
167
+ "rstrip": false,
168
+ "single_word": false,
169
+ "special": false
170
+ },
171
+ "151350": {
172
+ "content": "<|end_of_box|>",
173
+ "lstrip": false,
174
+ "normalized": false,
175
+ "rstrip": false,
176
+ "single_word": false,
177
+ "special": false
178
+ },
179
+ "151351": {
180
+ "content": "<|sep|>",
181
+ "lstrip": false,
182
+ "normalized": false,
183
+ "rstrip": false,
184
+ "single_word": false,
185
+ "special": false
186
+ }
187
+ },
188
+ "additional_special_tokens": [
189
+ "<|endoftext|>",
190
+ "[MASK]",
191
+ "[gMASK]",
192
+ "[sMASK]",
193
+ "<sop>",
194
+ "<eop>",
195
+ "<|system|>",
196
+ "<|user|>",
197
+ "<|assistant|>",
198
+ "<|observation|>",
199
+ "<|begin_of_image|>",
200
+ "<|end_of_image|>",
201
+ "<|begin_of_video|>",
202
+ "<|end_of_video|>",
203
+ "<|image|>",
204
+ "<|video|>"
205
+ ],
206
+ "clean_up_tokenization_spaces": false,
207
+ "do_lower_case": false,
208
+ "eos_token": "<|endoftext|>",
209
+ "pad_token": "<|endoftext|>",
210
+ "model_input_names": [
211
+ "input_ids",
212
+ "attention_mask"
213
+ ],
214
+ "model_max_length": 65536,
215
+ "padding_side": "left",
216
+ "remove_space": false,
217
+ "tokenizer_class": "PreTrainedTokenizer"
218
+ }
video_preprocessor_config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "size": {"shortest_edge": 12544, "longest_edge": 47040000},
3
+ "do_rescale": true,
4
+ "patch_size": 14,
5
+ "temporal_patch_size": 2,
6
+ "merge_size": 2,
7
+ "image_mean": [0.48145466, 0.4578275, 0.40821073],
8
+ "image_std": [0.26862954, 0.26130258, 0.27577711],
9
+ "video_processor_type": "Glm4vVideoProcessor",
10
+ "processor_class": "Glm4vProcessor"
11
+ }