Tensor-2-0.6m / tokenizer.json
VGribAI's picture
Upload 6 files
4ebd4c9 verified
Raw
History Blame Contribute Delete
24.5 kB
{
"version": "1.0",
"truncation": {
"direction": "Right",
"max_length": 128,
"strategy": "LongestFirst",
"stride": 0
},
"padding": {
"strategy": "BatchLongest",
"direction": "Right",
"pad_to_multiple_of": null,
"pad_id": 1,
"pad_type_id": 0,
"pad_token": "<|pad|>"
},
"added_tokens": [
{
"id": 1,
"content": "<|pad|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<|unk|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": true
},
"post_processor": null,
"decoder": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": "<|unk|>",
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"": 0,
"<|pad|>": 1,
"<|unk|>": 2,
"!": 3,
"\"": 4,
"#": 5,
"$": 6,
"%": 7,
"&": 8,
"'": 9,
"(": 10,
")": 11,
"*": 12,
"+": 13,
",": 14,
"-": 15,
".": 16,
"/": 17,
"0": 18,
"1": 19,
"2": 20,
"3": 21,
"4": 22,
"5": 23,
"6": 24,
"7": 25,
"8": 26,
"9": 27,
":": 28,
";": 29,
"<": 30,
"=": 31,
">": 32,
"?": 33,
"@": 34,
"A": 35,
"B": 36,
"C": 37,
"D": 38,
"E": 39,
"F": 40,
"G": 41,
"H": 42,
"I": 43,
"J": 44,
"K": 45,
"L": 46,
"M": 47,
"N": 48,
"O": 49,
"P": 50,
"Q": 51,
"R": 52,
"S": 53,
"T": 54,
"U": 55,
"V": 56,
"W": 57,
"X": 58,
"Y": 59,
"Z": 60,
"[": 61,
"\\": 62,
"]": 63,
"^": 64,
"_": 65,
"`": 66,
"a": 67,
"b": 68,
"c": 69,
"d": 70,
"e": 71,
"f": 72,
"g": 73,
"h": 74,
"i": 75,
"j": 76,
"k": 77,
"l": 78,
"m": 79,
"n": 80,
"o": 81,
"p": 82,
"q": 83,
"r": 84,
"s": 85,
"t": 86,
"u": 87,
"v": 88,
"w": 89,
"x": 90,
"y": 91,
"z": 92,
"{": 93,
"|": 94,
"}": 95,
"~": 96,
"¡": 97,
"¢": 98,
"£": 99,
"¤": 100,
"¥": 101,
"¦": 102,
"§": 103,
"¨": 104,
"©": 105,
"ª": 106,
"«": 107,
"¬": 108,
"®": 109,
"¯": 110,
"°": 111,
"±": 112,
"²": 113,
"³": 114,
"´": 115,
"µ": 116,
"¶": 117,
"·": 118,
"¸": 119,
"¹": 120,
"º": 121,
"»": 122,
"¼": 123,
"½": 124,
"¾": 125,
"¿": 126,
"Â": 127,
"Ã": 128,
"Ä": 129,
"Å": 130,
"È": 131,
"É": 132,
"Ê": 133,
"Ë": 134,
"Ì": 135,
"Í": 136,
"Î": 137,
"Ï": 138,
"Ð": 139,
"Ñ": 140,
"Ò": 141,
"Ó": 142,
"Ô": 143,
"Õ": 144,
"Ö": 145,
"Ø": 146,
"Ù": 147,
"Û": 148,
"Ü": 149,
"à": 150,
"á": 151,
"â": 152,
"ã": 153,
"å": 154,
"æ": 155,
"ç": 156,
"ï": 157,
"ð": 158,
"Ā": 159,
"ā": 160,
"Ă": 161,
"ă": 162,
"Ą": 163,
"ą": 164,
"Ć": 165,
"ć": 166,
"Ĉ": 167,
"ĉ": 168,
"ċ": 169,
"Č": 170,
"Ď": 171,
"ď": 172,
"Đ": 173,
"đ": 174,
"Ē": 175,
"ē": 176,
"Ĕ": 177,
"ĕ": 178,
"Ė": 179,
"ė": 180,
"Ę": 181,
"ę": 182,
"Ě": 183,
"ě": 184,
"Ĝ": 185,
"ĝ": 186,
"ğ": 187,
"Ġ": 188,
"Ģ": 189,
"ģ": 190,
"Ĥ": 191,
"ĥ": 192,
"Ħ": 193,
"ħ": 194,
"Ĩ": 195,
"ĩ": 196,
"Ī": 197,
"ī": 198,
"Ĭ": 199,
"ĭ": 200,
"Į": 201,
"į": 202,
"İ": 203,
"ı": 204,
"IJ": 205,
"ij": 206,
"Ĵ": 207,
"ĵ": 208,
"Ķ": 209,
"ķ": 210,
"ĸ": 211,
"Ĺ": 212,
"ĺ": 213,
"Ļ": 214,
"ļ": 215,
"Ľ": 216,
"ľ": 217,
"Ŀ": 218,
"ŀ": 219,
"Ł": 220,
"ł": 221,
"Ń": 222,
"ĠÐ": 223,
"о": 224,
"а": 225,
"е": 226,
"ÑĤ": 227,
"и": 228,
"Ñģ": 229,
"оÐ": 230,
"ÑĢ": 231,
"н": 232,
"аÐ": 233,
"Ñĥ": 234,
"еÐ": 235,
"Ñı": 236,
"ÑĮ": 237,
"Ñĭ": 238,
"л": 239,
"к": 240,
"Ġп": 241,
"Ġн": 242,
"Ġв": 243,
"ĠÑģ": 244,
"Ñĩ": 245,
"в": 246,
"д": 247,
"ÑģÑĤ": 248,
"м": 249,
"Ġи": 250,
"Ñħ": 251,
"ен": 252,
"ов": 253,
"Ġо": 254,
"ĠÑĤ": 255,
"Ġк": 256,
"Ġд": 257,
"п": 258,
"еÑĤ": 259,
"ÑĪ": 260,
"Ġм": 261,
"Ñİ": 262,
"ÑĤо": 263,
"ол": 264,
"еÑĢ": 265,
"Ġб": 266,
"ал": 267,
"б": 268,
"но": 269,
"аÑĤ": 270,
"иÑĤ": 271,
"ак": 272,
"од": 273,
"ĠÑĢ": 274,
"ĠпÑĢ": 275,
"ан": 276,
"з": 277,
"ел": 278,
"ÑĨ": 279,
"ом": 280,
"ог": 281,
"оÑĢ": 282,
"Ġз": 283,
"..": 284,
"ĠÑĥ": 285,
"й": 286,
"нÑĭ": 287,
"Ġна": 288,
"Ġне": 289,
"г": 290,
"ĠÑĩ": 291,
"ĠÑ": 292,
"ем": 293,
"ой": 294,
"оÑĤ": 295,
"ли": 296,
"он": 297,
"ам": 298,
"ав": 299,
"аÑģ": 300,
"Ġг": 301,
"ж": 302,
"ки": 303,
"оÑģ": 304,
"Ġа": 305,
"аÑĢ": 306,
"Ñī": 307,
"ед": 308,
"аз": 309,
"ени": 310,
"ÑģÑı": 311,
"ни": 312,
"!!": 313,
"ĠÑį": 314,
"на": 315,
"Ġл": 316,
"Ġе": 317,
"об": 318,
"аÑĤÑĮ": 319,
"еÑģ": 320,
"ож": 321,
"ого": 322,
"ÑĨи": 323,
"ей": 324,
"ка": 325,
"ÑģÑĤв": 326,
"ðŁ": 327,
"им": 328,
"ин": 329,
"Ġж": 330,
"ад": 331,
"ив": 332,
"ил": 333,
"оÑģÑĤ": 334,
"ĠÑħ": 335,
"олÑĮ": 336,
"Ñĥд": 337,
"ĠоÑĤ": 338,
"ок": 339,
"ĠвÑģ": 340,
"ĠбÑĭ": 341,
"Ñij": 342,
"лÑı": 343,
"ег": 344,
"ÑĥÑĤ": 345,
"ĠÑĤак": 346,
"ĠвÑĭ": 347,
"ек": 348,
"ĠÑĩÑĤо": 349,
"ÑĦ": 350,
"оÑĩ": 351,
"ĠÑģÑĤ": 352,
"ÑĢÑĥ": 353,
"иÑĤÑĮ": 354,
"Ġоб": 355,
"еб": 356,
"ани": 357,
"ÑıÑĤ": 358,
"да": 359,
"аб": 360,
"ÑĢа": 361,
"ÑģÑĤÑĮ": 362,
"Ġкак": 363,
"иÑģ": 364,
"ид": 365,
"Ġпо": 366,
"Ġ1": 367,
"из": 368,
"иÑĩ": 369,
"ев": 370,
"оз": 371,
"иÑħ": 372,
"ко": 373,
"алÑĮ": 374,
"ез": 375,
"пÑĢ": 376,
"елÑĮ": 377,
"Ġза": 378,
"ÑİÑĤ": 379,
"âĢ": 380,
"оп": 381,
"...": 382,
"Ġ2": 383,
"аж": 384,
"Ġ(": 385,
"ĠÑįÑĤо": 386,
"ик": 387,
"нÑĭÑħ": 388,
"ениÑı": 389,
"ĠÑı": 390,
"енÑĮ": 391,
"аеÑĤ": 392,
"Ñģк": 393,
"ÑĤоÑĢ": 394,
"Ġ,": 395,
"не": 396,
"ĠÑģв": 397,
"ай": 398,
"ÑĥÑĩ": 399,
"нÑĭе": 400,
"аÑı": 401,
"иÑĢ": 402,
"ĠÑĦ": 403,
"ĠÑįÑĤ": 404,
"ного": 405,
"ной": 406,
"Ġиз": 407,
"ĠÑĢаз": 408,
"нÑĭй": 409,
"ÑģÑĤи": 410,
"кÑĥ": 411,
"еÑĩ": 412,
"ĠĠ": 413,
"ĠвÑģе": 414,
"али": 415,
"ие": 416,
"Ġпод": 417,
"ла": 418,
"ап": 419,
"Ñĥж": 420,
"00": 421,
"Ñģи": 422,
"ло": 423,
"енÑĤ": 424,
"Ġпол": 425,
"ÑĥÑİ": 426,
"олÑĮко": 427,
"его": 428,
"аÑĨи": 429,
"!!!": 430,
"еÑĪ": 431,
"on": 432,
"ами": 433,
"Ñīе": 434,
"Ñį": 435,
"ение": 436,
"ĠпеÑĢ": 437,
"ĠкÑĢ": 438,
"ĠлÑİ": 439,
"Ñĭе": 440,
"ĠÑģл": 441,
"Ñĭй": 442,
"ĠÑĤо": 443,
"еÑģÑĤ": 444,
"âĢĵ": 445,
"ti": 446,
"наÑı": 447,
"Ñĭв": 448,
"ÑĪе": 449,
"еп": 450,
"ĠпÑĢо": 451,
"ĠбÑĥд": 452,
"Ðŀ": 453,
"Ġ-": 454,
"ÑĥÑĢ": 455,
"ми": 456,
"Ġин": 457,
"ĠпÑĢи": 458,
"an": 459,
"аг": 460,
"er": 461,
"Ñģп": 462,
"Ñĥм": 463,
"in": 464,
"Ġмож": 465,
"аÑİÑĤ": 466,
"ðŁĺ": 467,
"иÑı": 468,
"или": 469,
"ÑĢав": 470,
"ное": 471,
"иÑĩеÑģ": 472,
"ĠС": 473,
"еж": 474,
"ÑĢан": 475,
"ÑĤе": 476,
"ĠдлÑı": 477,
"ĠÑģк": 478,
"ĠÑĪ": 479,
"ое": 480,
"ÑĭÑħ": 481,
"иÑĤе": 482,
"ðŁij": 483,
"Ñĥп": 484,
"ĠÑģп": 485,
"аÑİ": 486,
"иб": 487,
"al": 488,
"ĠÑģо": 489,
"омÑĥ": 490,
"01": 491,
"же": 492,
"en": 493,
"аÑħ": 494,
"ĠÑĤÑĭ": 495,
"Ġнад": 496,
"Ġ.": 497,
"Ñħод": 498,
"лÑİ": 499
},
"merges": [
[
"Ġ",
"Ð"
],
[
"Ð",
"¾"
],
[
"Ð",
"°"
],
[
"Ð",
"µ"
],
[
"Ñ",
"Ĥ"
],
[
"Ð",
"¸"
],
[
"Ñ",
"ģ"
],
[
"о",
"Ð"
],
[
"Ñ",
"Ģ"
],
[
"Ð",
"½"
],
[
"а",
"Ð"
],
[
"Ñ",
"ĥ"
],
[
"е",
"Ð"
],
[
"Ñ",
"ı"
],
[
"Ñ",
"Į"
],
[
"Ñ",
"ĭ"
],
[
"Ð",
"»"
],
[
"Ð",
"º"
],
[
"ĠÐ",
"¿"
],
[
"ĠÐ",
"½"
],
[
"ĠÐ",
"²"
],
[
"Ġ",
"Ñģ"
],
[
"Ñ",
"ĩ"
],
[
"Ð",
"²"
],
[
"Ð",
"´"
],
[
"Ñģ",
"ÑĤ"
],
[
"Ð",
"¼"
],
[
"ĠÐ",
"¸"
],
[
"Ñ",
"ħ"
],
[
"е",
"н"
],
[
"оÐ",
"²"
],
[
"ĠÐ",
"¾"
],
[
"Ġ",
"ÑĤ"
],
[
"ĠÐ",
"º"
],
[
"ĠÐ",
"´"
],
[
"Ð",
"¿"
],
[
"е",
"ÑĤ"
],
[
"Ñ",
"Ī"
],
[
"ĠÐ",
"¼"
],
[
"Ñ",
"İ"
],
[
"ÑĤ",
"о"
],
[
"оÐ",
"»"
],
[
"е",
"ÑĢ"
],
[
"ĠÐ",
"±"
],
[
"аÐ",
"»"
],
[
"Ð",
"±"
],
[
"н",
"о"
],
[
"а",
"ÑĤ"
],
[
"и",
"ÑĤ"
],
[
"аÐ",
"º"
],
[
"оÐ",
"´"
],
[
"Ġ",
"ÑĢ"
],
[
"Ġп",
"ÑĢ"
],
[
"а",
"н"
],
[
"Ð",
"·"
],
[
"еÐ",
"»"
],
[
"Ñ",
"Ĩ"
],
[
"оÐ",
"¼"
],
[
"оÐ",
"³"
],
[
"о",
"ÑĢ"
],
[
"ĠÐ",
"·"
],
[
".",
"."
],
[
"Ġ",
"Ñĥ"
],
[
"Ð",
"¹"
],
[
"н",
"Ñĭ"
],
[
"Ġн",
"а"
],
[
"Ġн",
"е"
],
[
"Ð",
"³"
],
[
"Ġ",
"Ñĩ"
],
[
"Ġ",
"Ñ"
],
[
"еÐ",
"¼"
],
[
"оÐ",
"¹"
],
[
"о",
"ÑĤ"
],
[
"л",
"и"
],
[
"оÐ",
"½"
],
[
"аÐ",
"¼"
],
[
"аÐ",
"²"
],
[
"а",
"Ñģ"
],
[
"ĠÐ",
"³"
],
[
"Ð",
"¶"
],
[
"к",
"и"
],
[
"о",
"Ñģ"
],
[
"ĠÐ",
"°"
],
[
"а",
"ÑĢ"
],
[
"Ñ",
"ī"
],
[
"еÐ",
"´"
],
[
"аÐ",
"·"
],
[
"ен",
"и"
],
[
"Ñģ",
"Ñı"
],
[
"н",
"и"
],
[
"!",
"!"
],
[
"ĠÑ",
"į"
],
[
"н",
"а"
],
[
"ĠÐ",
"»"
],
[
"ĠÐ",
"µ"
],
[
"оÐ",
"±"
],
[
"аÑĤ",
"ÑĮ"
],
[
"е",
"Ñģ"
],
[
"оÐ",
"¶"
],
[
"ог",
"о"
],
[
"ÑĨ",
"и"
],
[
"еÐ",
"¹"
],
[
"к",
"а"
],
[
"ÑģÑĤ",
"в"
],
[
"ð",
"Ł"
],
[
"и",
"м"
],
[
"и",
"н"
],
[
"ĠÐ",
"¶"
],
[
"аÐ",
"´"
],
[
"и",
"в"
],
[
"и",
"л"
],
[
"о",
"ÑģÑĤ"
],
[
"Ġ",
"Ñħ"
],
[
"ол",
"ÑĮ"
],
[
"Ñĥ",
"д"
],
[
"Ġо",
"ÑĤ"
],
[
"оÐ",
"º"
],
[
"Ġв",
"Ñģ"
],
[
"Ġб",
"Ñĭ"
],
[
"Ñ",
"ij"
],
[
"л",
"Ñı"
],
[
"еÐ",
"³"
],
[
"Ñĥ",
"ÑĤ"
],
[
"ĠÑĤ",
"ак"
],
[
"Ġв",
"Ñĭ"
],
[
"еÐ",
"º"
],
[
"ĠÑĩ",
"ÑĤо"
],
[
"Ñ",
"Ħ"
],
[
"о",
"Ñĩ"
],
[
"ĠÑģ",
"ÑĤ"
],
[
"ÑĢ",
"Ñĥ"
],
[
"иÑĤ",
"ÑĮ"
],
[
"Ġо",
"б"
],
[
"еÐ",
"±"
],
[
"ан",
"и"
],
[
"Ñı",
"ÑĤ"
],
[
"д",
"а"
],
[
"аÐ",
"±"
],
[
"ÑĢ",
"а"
],
[
"ÑģÑĤ",
"ÑĮ"
],
[
"Ġк",
"ак"
],
[
"и",
"Ñģ"
],
[
"и",
"д"
],
[
"Ġп",
"о"
],
[
"Ġ",
"1"
],
[
"и",
"з"
],
[
"и",
"Ñĩ"
],
[
"еÐ",
"²"
],
[
"оÐ",
"·"
],
[
"и",
"Ñħ"
],
[
"к",
"о"
],
[
"ал",
"ÑĮ"
],
[
"еÐ",
"·"
],
[
"п",
"ÑĢ"
],
[
"ел",
"ÑĮ"
],
[
"Ġз",
"а"
],
[
"Ñİ",
"ÑĤ"
],
[
"â",
"Ģ"
],
[
"оÐ",
"¿"
],
[
"..",
"."
],
[
"Ġ",
"2"
],
[
"аÐ",
"¶"
],
[
"Ġ",
"("
],
[
"ĠÑį",
"ÑĤо"
],
[
"и",
"к"
],
[
"нÑĭ",
"Ñħ"
],
[
"ени",
"Ñı"
],
[
"Ġ",
"Ñı"
],
[
"ен",
"ÑĮ"
],
[
"а",
"еÑĤ"
],
[
"Ñģ",
"к"
],
[
"ÑĤо",
"ÑĢ"
],
[
"Ġ",
","
],
[
"н",
"е"
],
[
"ĠÑģ",
"в"
],
[
"аÐ",
"¹"
],
[
"Ñĥ",
"Ñĩ"
],
[
"нÑĭ",
"е"
],
[
"а",
"Ñı"
],
[
"и",
"ÑĢ"
],
[
"ĠÑ",
"Ħ"
],
[
"ĠÑį",
"ÑĤ"
],
[
"н",
"ого"
],
[
"н",
"ой"
],
[
"Ġи",
"з"
],
[
"ĠÑĢ",
"аз"
],
[
"нÑĭ",
"й"
],
[
"ÑģÑĤ",
"и"
],
[
"к",
"Ñĥ"
],
[
"е",
"Ñĩ"
],
[
"Ġ",
"Ġ"
],
[
"ĠвÑģ",
"е"
],
[
"ал",
"и"
],
[
"и",
"е"
],
[
"Ġп",
"од"
],
[
"л",
"а"
],
[
"аÐ",
"¿"
],
[
"Ñĥ",
"ж"
],
[
"0",
"0"
],
[
"Ñģ",
"и"
],
[
"л",
"о"
],
[
"ен",
"ÑĤ"
],
[
"Ġп",
"ол"
],
[
"Ñĥ",
"Ñİ"
],
[
"олÑĮ",
"ко"
],
[
"ег",
"о"
],
[
"а",
"ÑĨи"
],
[
"!!",
"!"
],
[
"е",
"ÑĪ"
],
[
"o",
"n"
],
[
"ам",
"и"
],
[
"Ñī",
"е"
],
[
"Ñ",
"į"
],
[
"ени",
"е"
],
[
"Ġп",
"еÑĢ"
],
[
"Ġк",
"ÑĢ"
],
[
"Ġл",
"Ñİ"
],
[
"Ñĭ",
"е"
],
[
"ĠÑģ",
"л"
],
[
"Ñĭ",
"й"
],
[
"ĠÑĤ",
"о"
],
[
"е",
"ÑģÑĤ"
],
[
"âĢ",
"ĵ"
],
[
"t",
"i"
],
[
"на",
"Ñı"
],
[
"Ñĭ",
"в"
],
[
"ÑĪ",
"е"
],
[
"еÐ",
"¿"
],
[
"ĠпÑĢ",
"о"
],
[
"Ġб",
"Ñĥд"
],
[
"Ð",
"ŀ"
],
[
"Ġ",
"-"
],
[
"Ñĥ",
"ÑĢ"
],
[
"м",
"и"
],
[
"Ġи",
"н"
],
[
"ĠпÑĢ",
"и"
],
[
"a",
"n"
],
[
"аÐ",
"³"
],
[
"e",
"r"
],
[
"Ñģ",
"п"
],
[
"Ñĥ",
"м"
],
[
"i",
"n"
],
[
"Ġм",
"ож"
],
[
"а",
"ÑİÑĤ"
],
[
"ðŁ",
"ĺ"
],
[
"и",
"Ñı"
],
[
"и",
"ли"
],
[
"ÑĢ",
"ав"
],
[
"но",
"е"
],
[
"иÑĩ",
"еÑģ"
],
[
"ĠÐ",
"¡"
],
[
"еÐ",
"¶"
],
[
"ÑĢ",
"ан"
],
[
"ÑĤ",
"е"
],
[
"Ġд",
"лÑı"
],
[
"ĠÑģ",
"к"
],
[
"Ġ",
"ÑĪ"
],
[
"о",
"е"
],
[
"Ñĭ",
"Ñħ"
],
[
"иÑĤ",
"е"
],
[
"ðŁ",
"ij"
],
[
"Ñĥ",
"п"
],
[
"ĠÑģ",
"п"
],
[
"а",
"Ñİ"
],
[
"и",
"б"
],
[
"a",
"l"
],
[
"ĠÑģ",
"о"
],
[
"ом",
"Ñĥ"
],
[
"0",
"1"
],
[
"ж",
"е"
],
[
"e",
"n"
],
[
"а",
"Ñħ"
],
[
"ĠÑĤ",
"Ñĭ"
],
[
"Ġн",
"ад"
],
[
"Ġ",
"."
],
[
"Ñħ",
"од"
],
[
"л",
"Ñİ"
]
]
}
}