Download tokenizer.json from shreyansh-programmer/Intelligence_OS_1p5B: direct link, hf CLI and curl.
- Browser
- Download file 27.1 kB
-
https://huggingface.co/shreyansh-programmer/Intelligence_OS_1p5B/resolve/main/tokenizer.json
- Command line
-
hf download hf://shreyansh-programmer/Intelligence_OS_1p5B/tokenizer.json
-
curl -L -o tokenizer.json https://huggingface.co/shreyansh-programmer/Intelligence_OS_1p5B/resolve/main/tokenizer.json
27.1 kB
{
"target_vocab_size": 512,
"special_tokens": [
"<pad>",
"<s>",
"</s>",
"<unk>"
],
"merges": [
{
"p0": 105,
"p1": 118,
"id": 260
},
{
"p0": 119,
"p1": 36,
"id": 261
},
{
"p0": 105,
"p1": 36,
"id": 262
},
{
"p0": 109,
"p1": 114,
"id": 263
},
{
"p0": 101,
"p1": 120,
"id": 264
},
{
"p0": 105,
"p1": 114,
"id": 265
},
{
"p0": 101,
"p1": 114,
"id": 266
},
{
"p0": 115,
"p1": 114,
"id": 267
},
{
"p0": 101,
"p1": 112,
"id": 268
},
{
"p0": 62,
"p1": 36,
"id": 269
},
{
"p0": 125,
"p1": 36,
"id": 270
},
{
"p0": 104,
"p1": 36,
"id": 271
},
{
"p0": 101,
"p1": 118,
"id": 272
},
{
"p0": 115,
"p1": 118,
"id": 273
},
{
"p0": 109,
"p1": 267,
"id": 274
},
{
"p0": 118,
"p1": 115,
"id": 275
},
{
"p0": 105,
"p1": 112,
"id": 276
},
{
"p0": 263,
"p1": 107,
"id": 277
},
{
"p0": 50,
"p1": 14,
"id": 278
},
{
"p0": 109,
"p1": 103,
"id": 279
},
{
"p0": 120,
"p1": 108,
"id": 280
},
{
"p0": 278,
"p1": 14,
"id": 281
},
{
"p0": 265,
"p1": 103,
"id": 282
},
{
"p0": 69,
"p1": 77,
"id": 283
},
{
"p0": 36,
"p1": 119,
"id": 284
},
{
"p0": 266,
"p1": 271,
"id": 285
},
{
"p0": 48,
"p1": 36,
"id": 286
},
{
"p0": 264,
"p1": 274,
"id": 287
},
{
"p0": 121,
"p1": 120,
"id": 288
},
{
"p0": 103,
"p1": 115,
"id": 289
},
{
"p0": 101,
"p1": 103,
"id": 290
},
{
"p0": 89,
"p1": 119,
"id": 291
},
{
"p0": 109,
"p1": 261,
"id": 292
},
{
"p0": 109,
"p1": 107,
"id": 293
},
{
"p0": 113,
"p1": 116,
"id": 294
},
{
"p0": 291,
"p1": 260,
"id": 295
},
{
"p0": 295,
"p1": 269,
"id": 296
},
{
"p0": 283,
"p1": 269,
"id": 297
},
{
"p0": 119,
"p1": 120,
"id": 298
},
{
"p0": 281,
"p1": 296,
"id": 299
},
{
"p0": 36,
"p1": 116,
"id": 300
},
{
"p0": 112,
"p1": 105,
"id": 301
},
{
"p0": 265,
"p1": 120,
"id": 302
},
{
"p0": 120,
"p1": 109,
"id": 303
},
{
"p0": 106,
"p1": 273,
"id": 304
},
{
"p0": 67,
"p1": 14,
"id": 305
},
{
"p0": 305,
"p1": 297,
"id": 306
},
{
"p0": 276,
"p1": 112,
"id": 307
},
{
"p0": 264,
"p1": 36,
"id": 308
},
{
"p0": 121,
"p1": 118,
"id": 309
},
{
"p0": 277,
"p1": 36,
"id": 310
},
{
"p0": 289,
"p1": 294,
"id": 311
},
{
"p0": 105,
"p1": 104,
"id": 312
},
{
"p0": 105,
"p1": 261,
"id": 313
},
{
"p0": 117,
"p1": 121,
"id": 314
},
{
"p0": 279,
"p1": 109,
"id": 315
},
{
"p0": 120,
"p1": 115,
"id": 316
},
{
"p0": 268,
"p1": 36,
"id": 317
},
{
"p0": 105,
"p1": 113,
"id": 318
},
{
"p0": 91,
"p1": 108,
"id": 319
},
{
"p0": 106,
"p1": 315,
"id": 320
},
{
"p0": 116,
"p1": 260,
"id": 321
},
{
"p0": 115,
"p1": 106,
"id": 322
},
{
"p0": 311,
"p1": 288,
"id": 323
},
{
"p0": 304,
"p1": 113,
"id": 324
},
{
"p0": 36,
"p1": 113,
"id": 325
},
{
"p0": 121,
"p1": 112,
"id": 326
},
{
"p0": 299,
"p1": 319,
"id": 327
},
{
"p0": 327,
"p1": 308,
"id": 328
},
{
"p0": 328,
"p1": 292,
"id": 329
},
{
"p0": 120,
"p1": 307,
"id": 330
},
{
"p0": 330,
"p1": 293,
"id": 331
},
{
"p0": 331,
"p1": 282,
"id": 332
},
{
"p0": 36,
"p1": 104,
"id": 333
},
{
"p0": 109,
"p1": 280,
"id": 334
},
{
"p0": 264,
"p1": 120,
"id": 335
},
{
"p0": 50,
"p1": 36,
"id": 336
},
{
"p0": 119,
"p1": 262,
"id": 337
},
{
"p0": 118,
"p1": 105,
"id": 338
},
{
"p0": 113,
"p1": 109,
"id": 339
},
{
"p0": 122,
"p1": 262,
"id": 340
},
{
"p0": 113,
"p1": 36,
"id": 341
},
{
"p0": 120,
"p1": 118,
"id": 342
},
{
"p0": 262,
"p1": 322,
"id": 343
},
{
"p0": 105,
"p1": 119,
"id": 344
},
{
"p0": 301,
"p1": 272,
"id": 345
},
{
"p0": 345,
"p1": 114,
"id": 346
},
{
"p0": 105,
"p1": 103,
"id": 347
},
{
"p0": 275,
"p1": 121,
"id": 348
},
{
"p0": 101,
"p1": 107,
"id": 349
},
{
"p0": 312,
"p1": 49,
"id": 350
},
{
"p0": 105,
"p1": 124,
"id": 351
},
{
"p0": 260,
"p1": 107,
"id": 352
},
{
"p0": 112,
"p1": 115,
"id": 353
},
{
"p0": 317,
"p1": 263,
"id": 354
},
{
"p0": 101,
"p1": 119,
"id": 355
},
{
"p0": 332,
"p1": 105,
"id": 356
},
{
"p0": 266,
"p1": 36,
"id": 357
},
{
"p0": 112,
"p1": 266,
"id": 358
},
{
"p0": 358,
"p1": 107,
"id": 359
},
{
"p0": 359,
"p1": 121,
"id": 360
},
{
"p0": 360,
"p1": 349,
"id": 361
},
{
"p0": 361,
"p1": 262,
"id": 362
},
{
"p0": 119,
"p1": 105,
"id": 363
},
{
"p0": 264,
"p1": 262,
"id": 364
},
{
"p0": 105,
"p1": 106,
"id": 365
},
{
"p0": 365,
"p1": 320,
"id": 366
},
{
"p0": 286,
"p1": 285,
"id": 367
},
{
"p0": 260,
"p1": 261,
"id": 368
},
{
"p0": 335,
"p1": 302,
"id": 369
},
{
"p0": 369,
"p1": 274,
"id": 370
},
{
"p0": 314,
"p1": 260,
"id": 371
},
{
"p0": 371,
"p1": 270,
"id": 372
},
{
"p0": 270,
"p1": 285,
"id": 373
},
{
"p0": 120,
"p1": 309,
"id": 374
},
{
"p0": 343,
"p1": 36,
"id": 375
},
{
"p0": 351,
"p1": 321,
"id": 376
},
{
"p0": 105,
"p1": 271,
"id": 377
},
{
"p0": 103,
"p1": 108,
"id": 378
},
{
"p0": 116,
"p1": 115,
"id": 379
},
{
"p0": 323,
"p1": 287,
"id": 380
},
{
"p0": 103,
"p1": 268,
"id": 381
},
{
"p0": 381,
"p1": 103,
"id": 382
},
{
"p0": 382,
"p1": 326,
"id": 383
},
{
"p0": 101,
"p1": 104,
"id": 384
},
{
"p0": 265,
"p1": 352,
"id": 385
},
{
"p0": 115,
"p1": 104,
"id": 386
},
{
"p0": 121,
"p1": 114,
"id": 387
},
{
"p0": 103,
"p1": 109,
"id": 388
},
{
"p0": 122,
"p1": 260,
"id": 389
},
{
"p0": 339,
"p1": 126,
"id": 390
},
{
"p0": 36,
"p1": 272,
"id": 391
},
{
"p0": 306,
"p1": 81,
"id": 392
},
{
"p0": 303,
"p1": 320,
"id": 393
},
{
"p0": 393,
"p1": 354,
"id": 394
},
{
"p0": 115,
"p1": 288,
"id": 395
},
{
"p0": 395,
"p1": 36,
"id": 396
},
{
"p0": 261,
"p1": 280,
"id": 397
},
{
"p0": 300,
"p1": 260,
"id": 398
},
{
"p0": 300,
"p1": 275,
"id": 399
}
],
"vocab": {
"0": [
60,
112,
97,
100,
62
],
"1": [
60,
115,
62
],
"2": [
60,
47,
115,
62
],
"3": [
60,
117,
110,
107,
62
],
"4": [
0
],
"5": [
1
],
"6": [
2
],
"7": [
3
],
"8": [
4
],
"9": [
5
],
"10": [
6
],
"11": [
7
],
"12": [
8
],
"13": [
9
],
"14": [
10
],
"15": [
11
],
"16": [
12
],
"17": [
13
],
"18": [
14
],
"19": [
15
],
"20": [
16
],
"21": [
17
],
"22": [
18
],
"23": [
19
],
"24": [
20
],
"25": [
21
],
"26": [
22
],
"27": [
23
],
"28": [
24
],
"29": [
25
],
"30": [
26
],
"31": [
27
],
"32": [
28
],
"33": [
29
],
"34": [
30
],
"35": [
31
],
"36": [
32
],
"37": [
33
],
"38": [
34
],
"39": [
35
],
"40": [
36
],
"41": [
37
],
"42": [
38
],
"43": [
39
],
"44": [
40
],
"45": [
41
],
"46": [
42
],
"47": [
43
],
"48": [
44
],
"49": [
45
],
"50": [
46
],
"51": [
47
],
"52": [
48
],
"53": [
49
],
"54": [
50
],
"55": [
51
],
"56": [
52
],
"57": [
53
],
"58": [
54
],
"59": [
55
],
"60": [
56
],
"61": [
57
],
"62": [
58
],
"63": [
59
],
"64": [
60
],
"65": [
61
],
"66": [
62
],
"67": [
63
],
"68": [
64
],
"69": [
65
],
"70": [
66
],
"71": [
67
],
"72": [
68
],
"73": [
69
],
"74": [
70
],
"75": [
71
],
"76": [
72
],
"77": [
73
],
"78": [
74
],
"79": [
75
],
"80": [
76
],
"81": [
77
],
"82": [
78
],
"83": [
79
],
"84": [
80
],
"85": [
81
],
"86": [
82
],
"87": [
83
],
"88": [
84
],
"89": [
85
],
"90": [
86
],
"91": [
87
],
"92": [
88
],
"93": [
89
],
"94": [
90
],
"95": [
91
],
"96": [
92
],
"97": [
93
],
"98": [
94
],
"99": [
95
],
"100": [
96
],
"101": [
97
],
"102": [
98
],
"103": [
99
],
"104": [
100
],
"105": [
101
],
"106": [
102
],
"107": [
103
],
"108": [
104
],
"109": [
105
],
"110": [
106
],
"111": [
107
],
"112": [
108
],
"113": [
109
],
"114": [
110
],
"115": [
111
],
"116": [
112
],
"117": [
113
],
"118": [
114
],
"119": [
115
],
"120": [
116
],
"121": [
117
],
"122": [
118
],
"123": [
119
],
"124": [
120
],
"125": [
121
],
"126": [
122
],
"127": [
123
],
"128": [
124
],
"129": [
125
],
"130": [
126
],
"131": [
127
],
"132": [
128
],
"133": [
129
],
"134": [
130
],
"135": [
131
],
"136": [
132
],
"137": [
133
],
"138": [
134
],
"139": [
135
],
"140": [
136
],
"141": [
137
],
"142": [
138
],
"143": [
139
],
"144": [
140
],
"145": [
141
],
"146": [
142
],
"147": [
143
],
"148": [
144
],
"149": [
145
],
"150": [
146
],
"151": [
147
],
"152": [
148
],
"153": [
149
],
"154": [
150
],
"155": [
151
],
"156": [
152
],
"157": [
153
],
"158": [
154
],
"159": [
155
],
"160": [
156
],
"161": [
157
],
"162": [
158
],
"163": [
159
],
"164": [
160
],
"165": [
161
],
"166": [
162
],
"167": [
163
],
"168": [
164
],
"169": [
165
],
"170": [
166
],
"171": [
167
],
"172": [
168
],
"173": [
169
],
"174": [
170
],
"175": [
171
],
"176": [
172
],
"177": [
173
],
"178": [
174
],
"179": [
175
],
"180": [
176
],
"181": [
177
],
"182": [
178
],
"183": [
179
],
"184": [
180
],
"185": [
181
],
"186": [
182
],
"187": [
183
],
"188": [
184
],
"189": [
185
],
"190": [
186
],
"191": [
187
],
"192": [
188
],
"193": [
189
],
"194": [
190
],
"195": [
191
],
"196": [
192
],
"197": [
193
],
"198": [
194
],
"199": [
195
],
"200": [
196
],
"201": [
197
],
"202": [
198
],
"203": [
199
],
"204": [
200
],
"205": [
201
],
"206": [
202
],
"207": [
203
],
"208": [
204
],
"209": [
205
],
"210": [
206
],
"211": [
207
],
"212": [
208
],
"213": [
209
],
"214": [
210
],
"215": [
211
],
"216": [
212
],
"217": [
213
],
"218": [
214
],
"219": [
215
],
"220": [
216
],
"221": [
217
],
"222": [
218
],
"223": [
219
],
"224": [
220
],
"225": [
221
],
"226": [
222
],
"227": [
223
],
"228": [
224
],
"229": [
225
],
"230": [
226
],
"231": [
227
],
"232": [
228
],
"233": [
229
],
"234": [
230
],
"235": [
231
],
"236": [
232
],
"237": [
233
],
"238": [
234
],
"239": [
235
],
"240": [
236
],
"241": [
237
],
"242": [
238
],
"243": [
239
],
"244": [
240
],
"245": [
241
],
"246": [
242
],
"247": [
243
],
"248": [
244
],
"249": [
245
],
"250": [
246
],
"251": [
247
],
"252": [
248
],
"253": [
249
],
"254": [
250
],
"255": [
251
],
"256": [
252
],
"257": [
253
],
"258": [
254
],
"259": [
255
],
"260": [
101,
114
],
"261": [
115,
32
],
"262": [
101,
32
],
"263": [
105,
110
],
"264": [
97,
116
],
"265": [
101,
110
],
"266": [
97,
110
],
"267": [
111,
110
],
"268": [
97,
108
],
"269": [
58,
32
],
"270": [
121,
32
],
"271": [
100,
32
],
"272": [
97,
114
],
"273": [
111,
114
],
"274": [
105,
111,
110
],
"275": [
114,
111
],
"276": [
101,
108
],
"277": [
105,
110,
103
],
"278": [
46,
10
],
"279": [
105,
99
],
"280": [
116,
104
],
"281": [
46,
10,
10
],
"282": [
101,
110,
99
],
"283": [
65,
73
],
"284": [
32,
115
],
"285": [
97,
110,
100,
32
],
"286": [
44,
32
],
"287": [
97,
116,
105,
111,
110
],
"288": [
117,
116
],
"289": [
99,
111
],
"290": [
97,
99
],
"291": [
85,
115
],
"292": [
105,
115,
32
],
"293": [
105,
103
],
"294": [
109,
112
],
"295": [
85,
115,
101,
114
],
"296": [
85,
115,
101,
114,
58,
32
],
"297": [
65,
73,
58,
32
],
"298": [
115,
116
],
"299": [
46,
10,
10,
85,
115,
101,
114,
58,
32
],
"300": [
32,
112
],
"301": [
108,
101
],
"302": [
101,
110,
116
],
"303": [
116,
105
],
"304": [
102,
111,
114
],
"305": [
63,
10
],
"306": [
63,
10,
65,
73,
58,
32
],
"307": [
101,
108,
108
],
"308": [
97,
116,
32
],
"309": [
117,
114
],
"310": [
105,
110,
103,
32
],
"311": [
99,
111,
109,
112
],
"312": [
101,
100
],
"313": [
101,
115,
32
],
"314": [
113,
117
],
"315": [
105,
99,
105
],
"316": [
116,
111
],
"317": [
97,
108,
32
],
"318": [
101,
109
],
"319": [
87,
104
],
"320": [
102,
105,
99,
105
],
"321": [
112,
101,
114
],
"322": [
111,
102
],
"323": [
99,
111,
109,
112,
117,
116
],
"324": [
102,
111,
114,
109
],
"325": [
32,
109
],
"326": [
117,
108
],
"327": [
46,
10,
10,
85,
115,
101,
114,
58,
32,
87,
104
],
"328": [
46,
10,
10,
85,
115,
101,
114,
58,
32,
87,
104,
97,
116,
32
],
"329": [
46,
10,
10,
85,
115,
101,
114,
58,
32,
87,
104,
97,
116,
32,
105,
115,
32
],
"330": [
116,
101,
108,
108
],
"331": [
116,
101,
108,
108,
105,
103
],
"332": [
116,
101,
108,
108,
105,
103,
101,
110,
99
],
"333": [
32,
100
],
"334": [
105,
116,
104
],
"335": [
97,
116,
116
],
"336": [
46,
32
],
"337": [
115,
101,
32
],
"338": [
114,
101
],
"339": [
109,
105
],
"340": [
118,
101,
32
],
"341": [
109,
32
],
"342": [
116,
114
],
"343": [
101,
32,
111,
102
],
"344": [
101,
115
],
"345": [
108,
101,
97,
114
],
"346": [
108,
101,
97,
114,
110
],
"347": [
101,
99
],
"348": [
114,
111,
117
],
"349": [
97,
103
],
"350": [
101,
100,
45
],
"351": [
101,
120
],
"352": [
101,
114,
103
],
"353": [
108,
111
],
"354": [
97,
108,
32,
105,
110
],
"355": [
97,
115
],
"356": [
116,
101,
108,
108,
105,
103,
101,
110,
99,
101
],
"357": [
97,
110,
32
],
"358": [
108,
97,
110
],
"359": [
108,
97,
110,
103
],
"360": [
108,
97,
110,
103,
117
],
"361": [
108,
97,
110,
103,
117,
97,
103
],
"362": [
108,
97,
110,
103,
117,
97,
103,
101,
32
],
"363": [
115,
101
],
"364": [
97,
116,
101,
32
],
"365": [
101,
102
],
"366": [
101,
102,
102,
105,
99,
105
],
"367": [
44,
32,
97,
110,
100,
32
],
"368": [
101,
114,
115,
32
],
"369": [
97,
116,
116,
101,
110,
116
],
"370": [
97,
116,
116,
101,
110,
116,
105,
111,
110
],
"371": [
113,
117,
101,
114
],
"372": [
113,
117,
101,
114,
121,
32
],
"373": [
121,
32,
97,
110,
100,
32
],
"374": [
116,
117,
114
],
"375": [
101,
32,
111,
102,
32
],
"376": [
101,
120,
112,
101,
114
],
"377": [
101,
100,
32
],
"378": [
99,
104
],
"379": [
112,
111
],
"380": [
99,
111,
109,
112,
117,
116,
97,
116,
105,
111,
110
],
"381": [
99,
97,
108
],
"382": [
99,
97,
108,
99
],
"383": [
99,
97,
108,
99,
117,
108
],
"384": [
97,
100
],
"385": [
101,
110,
101,
114,
103
],
"386": [
111,
100
],
"387": [
117,
110
],
"388": [
99,
105
],
"389": [
118,
101,
114
],
"390": [
109,
105,
122
],
"391": [
32,
97,
114
],
"392": [
63,
10,
65,
73,
58,
32,
77
],
"393": [
116,
105,
102,
105,
99,
105
],
"394": [
116,
105,
102,
105,
99,
105,
97,
108,
32,
105,
110
],
"395": [
111,
117,
116
],
"396": [
111,
117,
116,
32
],
"397": [
115,
32,
116,
104
],
"398": [
32,
112,
101,
114
],
"399": [
32,
112,
114,
111
]
}
}
{
"target_vocab_size": 512,
"special_tokens": [
"<pad>",
"<s>",
"</s>",
"<unk>"
],
"merges": [
{
"p0": 105,
"p1": 118,
"id": 260
},
{
"p0": 119,
"p1": 36,
"id": 261
},
{
"p0": 105,
"p1": 36,
"id": 262
},
{
"p0": 109,
"p1": 114,
"id": 263
},
{
"p0": 101,
"p1": 120,
"id": 264
},
{
"p0": 105,
"p1": 114,
"id": 265
},
{
"p0": 101,
"p1": 114,
"id": 266
},
{
"p0": 115,
"p1": 114,
"id": 267
},
{
"p0": 101,
"p1": 112,
"id": 268
},
{
"p0": 62,
"p1": 36,
"id": 269
},
{
"p0": 125,
"p1": 36,
"id": 270
},
{
"p0": 104,
"p1": 36,
"id": 271
},
{
"p0": 101,
"p1": 118,
"id": 272
},
{
"p0": 115,
"p1": 118,
"id": 273
},
{
"p0": 109,
"p1": 267,
"id": 274
},
{
"p0": 118,
"p1": 115,
"id": 275
},
{
"p0": 105,
"p1": 112,
"id": 276
},
{
"p0": 263,
"p1": 107,
"id": 277
},
{