RuVI-AI
/

ArushCodes commited on
Commit
4f30f38
·
verified ·
1 Parent(s): 1c7baa8

Delete tokenizer.vocab

Browse files
Files changed (1) hide show
  1. tokenizer.vocab +0 -512
tokenizer.vocab DELETED
@@ -1,512 +0,0 @@
1
- <unk> 0
2
- <pad> 0
3
- <s> 0
4
- </s> 0
5
- <0x00> 0
6
- <0x01> 0
7
- <0x02> 0
8
- <0x03> 0
9
- <0x04> 0
10
- <0x05> 0
11
- <0x06> 0
12
- <0x07> 0
13
- <0x08> 0
14
- <0x09> 0
15
- <0x0A> 0
16
- <0x0B> 0
17
- <0x0C> 0
18
- <0x0D> 0
19
- <0x0E> 0
20
- <0x0F> 0
21
- <0x10> 0
22
- <0x11> 0
23
- <0x12> 0
24
- <0x13> 0
25
- <0x14> 0
26
- <0x15> 0
27
- <0x16> 0
28
- <0x17> 0
29
- <0x18> 0
30
- <0x19> 0
31
- <0x1A> 0
32
- <0x1B> 0
33
- <0x1C> 0
34
- <0x1D> 0
35
- <0x1E> 0
36
- <0x1F> 0
37
- <0x20> 0
38
- <0x21> 0
39
- <0x22> 0
40
- <0x23> 0
41
- <0x24> 0
42
- <0x25> 0
43
- <0x26> 0
44
- <0x27> 0
45
- <0x28> 0
46
- <0x29> 0
47
- <0x2A> 0
48
- <0x2B> 0
49
- <0x2C> 0
50
- <0x2D> 0
51
- <0x2E> 0
52
- <0x2F> 0
53
- <0x30> 0
54
- <0x31> 0
55
- <0x32> 0
56
- <0x33> 0
57
- <0x34> 0
58
- <0x35> 0
59
- <0x36> 0
60
- <0x37> 0
61
- <0x38> 0
62
- <0x39> 0
63
- <0x3A> 0
64
- <0x3B> 0
65
- <0x3C> 0
66
- <0x3D> 0
67
- <0x3E> 0
68
- <0x3F> 0
69
- <0x40> 0
70
- <0x41> 0
71
- <0x42> 0
72
- <0x43> 0
73
- <0x44> 0
74
- <0x45> 0
75
- <0x46> 0
76
- <0x47> 0
77
- <0x48> 0
78
- <0x49> 0
79
- <0x4A> 0
80
- <0x4B> 0
81
- <0x4C> 0
82
- <0x4D> 0
83
- <0x4E> 0
84
- <0x4F> 0
85
- <0x50> 0
86
- <0x51> 0
87
- <0x52> 0
88
- <0x53> 0
89
- <0x54> 0
90
- <0x55> 0
91
- <0x56> 0
92
- <0x57> 0
93
- <0x58> 0
94
- <0x59> 0
95
- <0x5A> 0
96
- <0x5B> 0
97
- <0x5C> 0
98
- <0x5D> 0
99
- <0x5E> 0
100
- <0x5F> 0
101
- <0x60> 0
102
- <0x61> 0
103
- <0x62> 0
104
- <0x63> 0
105
- <0x64> 0
106
- <0x65> 0
107
- <0x66> 0
108
- <0x67> 0
109
- <0x68> 0
110
- <0x69> 0
111
- <0x6A> 0
112
- <0x6B> 0
113
- <0x6C> 0
114
- <0x6D> 0
115
- <0x6E> 0
116
- <0x6F> 0
117
- <0x70> 0
118
- <0x71> 0
119
- <0x72> 0
120
- <0x73> 0
121
- <0x74> 0
122
- <0x75> 0
123
- <0x76> 0
124
- <0x77> 0
125
- <0x78> 0
126
- <0x79> 0
127
- <0x7A> 0
128
- <0x7B> 0
129
- <0x7C> 0
130
- <0x7D> 0
131
- <0x7E> 0
132
- <0x7F> 0
133
- <0x80> 0
134
- <0x81> 0
135
- <0x82> 0
136
- <0x83> 0
137
- <0x84> 0
138
- <0x85> 0
139
- <0x86> 0
140
- <0x87> 0
141
- <0x88> 0
142
- <0x89> 0
143
- <0x8A> 0
144
- <0x8B> 0
145
- <0x8C> 0
146
- <0x8D> 0
147
- <0x8E> 0
148
- <0x8F> 0
149
- <0x90> 0
150
- <0x91> 0
151
- <0x92> 0
152
- <0x93> 0
153
- <0x94> 0
154
- <0x95> 0
155
- <0x96> 0
156
- <0x97> 0
157
- <0x98> 0
158
- <0x99> 0
159
- <0x9A> 0
160
- <0x9B> 0
161
- <0x9C> 0
162
- <0x9D> 0
163
- <0x9E> 0
164
- <0x9F> 0
165
- <0xA0> 0
166
- <0xA1> 0
167
- <0xA2> 0
168
- <0xA3> 0
169
- <0xA4> 0
170
- <0xA5> 0
171
- <0xA6> 0
172
- <0xA7> 0
173
- <0xA8> 0
174
- <0xA9> 0
175
- <0xAA> 0
176
- <0xAB> 0
177
- <0xAC> 0
178
- <0xAD> 0
179
- <0xAE> 0
180
- <0xAF> 0
181
- <0xB0> 0
182
- <0xB1> 0
183
- <0xB2> 0
184
- <0xB3> 0
185
- <0xB4> 0
186
- <0xB5> 0
187
- <0xB6> 0
188
- <0xB7> 0
189
- <0xB8> 0
190
- <0xB9> 0
191
- <0xBA> 0
192
- <0xBB> 0
193
- <0xBC> 0
194
- <0xBD> 0
195
- <0xBE> 0
196
- <0xBF> 0
197
- <0xC0> 0
198
- <0xC1> 0
199
- <0xC2> 0
200
- <0xC3> 0
201
- <0xC4> 0
202
- <0xC5> 0
203
- <0xC6> 0
204
- <0xC7> 0
205
- <0xC8> 0
206
- <0xC9> 0
207
- <0xCA> 0
208
- <0xCB> 0
209
- <0xCC> 0
210
- <0xCD> 0
211
- <0xCE> 0
212
- <0xCF> 0
213
- <0xD0> 0
214
- <0xD1> 0
215
- <0xD2> 0
216
- <0xD3> 0
217
- <0xD4> 0
218
- <0xD5> 0
219
- <0xD6> 0
220
- <0xD7> 0
221
- <0xD8> 0
222
- <0xD9> 0
223
- <0xDA> 0
224
- <0xDB> 0
225
- <0xDC> 0
226
- <0xDD> 0
227
- <0xDE> 0
228
- <0xDF> 0
229
- <0xE0> 0
230
- <0xE1> 0
231
- <0xE2> 0
232
- <0xE3> 0
233
- <0xE4> 0
234
- <0xE5> 0
235
- <0xE6> 0
236
- <0xE7> 0
237
- <0xE8> 0
238
- <0xE9> 0
239
- <0xEA> 0
240
- <0xEB> 0
241
- <0xEC> 0
242
- <0xED> 0
243
- <0xEE> 0
244
- <0xEF> 0
245
- <0xF0> 0
246
- <0xF1> 0
247
- <0xF2> 0
248
- <0xF3> 0
249
- <0xF4> 0
250
- <0xF5> 0
251
- <0xF6> 0
252
- <0xF7> 0
253
- <0xF8> 0
254
- <0xF9> 0
255
- <0xFA> 0
256
- <0xFB> 0
257
- <0xFC> 0
258
- <0xFD> 0
259
- <0xFE> 0
260
- <0xFF> 0
261
- he -0
262
- ▁t -1
263
- an -2
264
- ▁s -3
265
- in -4
266
- ▁w -5
267
- ed -6
268
- ▁the -7
269
- al -8
270
- on -9
271
- ▁c -10
272
- er -11
273
- ▁f -12
274
- and -13
275
- at -14
276
- ▁b -15
277
- is -16
278
- ▁a -17
279
- re -18
280
- it -19
281
- ▁and -20
282
- ar -21
283
- ▁o -22
284
- ▁h -23
285
- es -24
286
- en -25
287
- ▁l -26
288
- nd -27
289
- ▁r -28
290
- ▁m -29
291
- ▁T -30
292
- ▁p -31
293
- ▁of -32
294
- ▁is -33
295
- ing -34
296
- ▁in -35
297
- ▁I -36
298
- as -37
299
- ou -38
300
- or -39
301
- ▁g -40
302
- am -41
303
- ▁The -42
304
- ic -43
305
- ▁d -44
306
- el -45
307
- her -46
308
- et -47
309
- ▁Ind -48
310
- ▁n -49
311
- ▁Indi -50
312
- ig -51
313
- ol -52
314
- il -53
315
- ot -54
316
- ▁to -55
317
- all -56
318
- ith -57
319
- ver -58
320
- ▁India -59
321
- ay -60
322
- ow -61
323
- ac -62
324
- om -63
325
- ▁st -64
326
- ▁with -65
327
- ht -66
328
- ▁A -67
329
- ▁cl -68
330
- ▁on -69
331
- ad -70
332
- ul -71
333
- ▁M -72
334
- ▁R -73
335
- ang -74
336
- ion -75
337
- ▁S -76
338
- est -77
339
- ight -78
340
- ft -79
341
- ain -80
342
- ▁br -81
343
- ▁th -82
344
- ▁his -83
345
- ly -84
346
- ▁G -85
347
- ent -86
348
- ▁fl -87
349
- ▁fr -88
350
- ▁gre -89
351
- ut -90
352
- we -91
353
- ▁C -92
354
- ▁D -93
355
- ▁e -94
356
- res -95
357
- ation -96
358
- im -97
359
- le -98
360
- un -99
361
- ▁sp -100
362
- weet -101
363
- ▁wat -102
364
- ▁sweet -103
365
- av -104
366
- ir -105
367
- pp -106
368
- ▁B -107
369
- ▁H -108
370
- ful -109
371
- old -110
372
- ong -111
373
- ▁Ch -112
374
- ▁by -113
375
- ▁co -114
376
- ational -115
377
- ab -116
378
- ag -117
379
- ak -118
380
- ur -119
381
- arm -120
382
- ill -121
383
- ite -122
384
- its -123
385
- one -124
386
- ous -125
387
- ▁an -126
388
- ▁tr -127
389
- ▁wh -128
390
- alled -129
391
- ▁green -130
392
- ▁called -131
393
- ▁national -132
394
- ch -133
395
- iv -134
396
- lu -135
397
- ▁N -136
398
- ▁k -137
399
- ers -138
400
- ick -139
401
- rou -140
402
- ▁for -141
403
- ▁tre -142
404
- ▁was -143
405
- ▁warm -144
406
- ok -145
407
- th -146
408
- to -147
409
- us -148
410
- ▁W -149
411
- ▁y -150
412
- ell -151
413
- hed -152
414
- ter -153
415
- ▁gr -154
416
- ▁ri -155
417
- ▁sh -156
418
- ▁sm -157
419
- ▁so -158
420
- pped -159
421
- ▁fam -160
422
- ▁from -161
423
- ▁bright -162
424
- ds -163
425
- ld -164
426
- tu -165
427
- um -166
428
- ace -167
429
- ade -168
430
- lue -169
431
- oon -170
432
- ore -171
433
- und -172
434
- ▁al -173
435
- ▁be -174
436
- ▁pe -175
437
- ▁ro -176
438
- ands -177
439
- ated -178
440
- very -179
441
- ▁its -180
442
- ▁pot -181
443
- ▁whe -182
444
- other -183
445
- ▁blue -184
446
- ▁water -185
447
- ▁famous -186
448
- ew -187
449
- op -188
450
- os -189
451
- ov -190
452
- ▁P -191
453
- ▁v -192
454
- ack -193
455
- anc -194
456
- any -195
457
- ▁ -196
458
- e -197
459
- a -198
460
- t -199
461
- n -200
462
- i -201
463
- o -202
464
- r -203
465
- s -204
466
- h -205
467
- l -206
468
- d -207
469
- c -208
470
- g -209
471
- f -210
472
- u -211
473
- w -212
474
- m -213
475
- y -214
476
- p -215
477
- b -216
478
- . -217
479
- , -218
480
- k -219
481
- v -220
482
- T -221
483
- I -222
484
- A -223
485
- M -224
486
- R -225
487
- S -226
488
- G -227
489
- C -228
490
- D -229
491
- j -230
492
- B -231
493
- H -232
494
- N -233
495
- W -234
496
- P -235
497
- E -236
498
- K -237
499
- ! -238
500
- J -239
501
- O -240
502
- 2 -241
503
- : -242
504
- F -243
505
- L -244
506
- q -245
507
- x -246
508
- z -247
509
- 5 -248
510
- 8 -249
511
- ' -250
512
- 3 -251