tiny-clip-text-2-mic / tokenizer.json
tarekziade's picture
tarekziade HF Staff
Publish MIC usage, slim fixture and reviewed Transformers expectations
c9fce36 verified
Raw History Blame Contribute Delete
31.9 kB
{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "<|startoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": true }, { "id": 1, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": true } ], "normalizer": { "type": "Sequence", "normalizers": [ { "type": "NFC" }, { "type": "Replace", "pattern": { "Regex": "\\s+" }, "content": " " }, { "type": "Lowercase" } ] }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "'s|'t|'re|'ve|'m|'ll|'d|[\\p{L}]+|[\\p{N}]|[^\\s\\p{L}\\p{N}]+" }, "behavior": "Removed", "invert": true }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true } ] }, "post_processor": { "type": "RobertaProcessing", "sep": [ "<|endoftext|>", 1 ], "cls": [ "<|startoftext|>", 0 ], "trim_offsets": false, "add_prefix_space": false }, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": "<|endoftext|>", "continuing_subword_prefix": "", "end_of_word_suffix": "</w>", "fuse_unk": false, "byte_fallback": false, "vocab": { "<|startoftext|>": 0, "<|endoftext|>": 1, "!": 2, "\"": 3, "#": 4, "$": 5, "%": 6, "&": 7, "'": 8, "(": 9, ")": 10, "*": 11, "+": 12, ",": 13, "-": 14, ".": 15, "/": 16, "0": 17, "1": 18, "2": 19, "3": 20, "4": 21, "5": 22, "6": 23, "7": 24, "8": 25, "9": 26, ":": 27, ";": 28, "<": 29, "=": 30, ">": 31, "?": 32, "@": 33, "A": 34, "B": 35, "C": 36, "D": 37, "E": 38, "F": 39, "G": 40, "H": 41, "I": 42, "J": 43, "K": 44, "L": 45, "M": 46, "N": 47, "O": 48, "P": 49, "Q": 50, "R": 51, "S": 52, "T": 53, "U": 54, "V": 55, "W": 56, "X": 57, "Y": 58, "Z": 59, "[": 60, "\\": 61, "]": 62, "^": 63, "_": 64, "`": 65, "a": 66, "b": 67, "c": 68, "d": 69, "e": 70, "f": 71, "g": 72, "h": 73, "i": 74, "j": 75, "k": 76, "l": 77, "m": 78, "n": 79, "o": 80, "p": 81, "q": 82, "r": 83, "s": 84, "t": 85, "u": 86, "v": 87, "w": 88, "x": 89, "y": 90, "z": 91, "|": 92, "}": 93, "~": 94, "¡": 95, "¢": 96, "£": 97, "¤": 98, "¥": 99, "¦": 100, "§": 101, "¨": 102, "©": 103, "ª": 104, "«": 105, "¬": 106, "®": 107, "¯": 108, "°": 109, "±": 110, "²": 111, "³": 112, "´": 113, "µ": 114, "¶": 115, "·": 116, "¸": 117, "¹": 118, "º": 119, "»": 120, "¼": 121, "½": 122, "¾": 123, "¿": 124, "Â": 125, "Ã": 126, "Ä": 127, "Å": 128, "Æ": 129, "Ç": 130, "È": 131, "É": 132, "Ê": 133, "Ë": 134, "Ì": 135, "Í": 136, "Î": 137, "Ï": 138, "Ð": 139, "Ñ": 140, "Ö": 141, "×": 142, "Ø": 143, "Ù": 144, "Ü": 145, "à": 146, "á": 147, "â": 148, "ã": 149, "ä": 150, "å": 151, "æ": 152, "ç": 153, "è": 154, "é": 155, "ë": 156, "ì": 157, "ï": 158, "Ċ": 159, "Ġ": 160, "Ģ": 161, "ģ": 162, "Ĥ": 163, "ĥ": 164, "Ħ": 165, "ħ": 166, "Ĩ": 167, "ĩ": 168, "Ī": 169, "ī": 170, "Ĭ": 171, "ĭ": 172, "Į": 173, "į": 174, "İ": 175, "ı": 176, "IJ": 177, "ij": 178, "Ĵ": 179, "ĵ": 180, "Ķ": 181, "ķ": 182, "ĸ": 183, "Ĺ": 184, "ĺ": 185, "Ļ": 186, "ļ": 187, "Ľ": 188, "ľ": 189, "Ŀ": 190, "ŀ": 191, "Ł": 192, "ł": 193, "Ń": 194, "e</w>": 195, "d</w>": 196, "a</w>": 197, "o</w>": 198, "n</w>": 199, "±</w>": 200, "l</w>": 201, "m</w>": 202, "h</w>": 203, "r</w>": 204, "i</w>": 205, "s</w>": 206, "Z</w>": 207, "t</w>": 208, "f</w>": 209, "k</w>": 210, "y</w>": 211, "b</w>": 212, "F</w>": 213, "g</w>": 214, "7</w>": 215, "0</w>": 216, "p</w>": 217, "L</w>": 218, "H</w>": 219, "¡</w>": 220, "Ī</w>": 221, "1</w>": 222, "Ģ</w>": 223, "c</w>": 224, "ĩ</w>": 225, "6</w>": 226, "A</w>": 227, "z</w>": 228, "u</w>": 229, "S</w>": 230, "2</w>": 231, "v</w>": 232, "4</w>": 233, "M</w>": 234, "T</w>": 235, "8</w>": 236, "I</w>": 237, "N</w>": 238, "C</w>": 239, "5</w>": 240, "¹</w>": 241, "9</w>": 242, "3</w>": 243, "ī</w>": 244, "P</w>": 245, "E</w>": 246, "»</w>": 247, "V</w>": 248, "İ</w>": 249, "w</w>": 250, "J</w>": 251, "ł</w>": 252, ".</w>": 253, "K</w>": 254, "D</w>": 255, "Ķ</w>": 256, "¸</w>": 257, "B</w>": 258, "©</w>": 259, "º</w>": 260, "µ</w>": 261, "Ĥ</w>": 262, "X</w>": 263, "R</w>": 264, "O</w>": 265, "«</w>": 266, "Ļ</w>": 267, "U</w>": 268, "x</w>": 269, "[</w>": 270, "¿</w>": 271, "³</w>": 272, "ģ</w>": 273, "W</w>": 274, "§</w>": 275, "-</w>": 276, "ĸ</w>": 277, "Ħ</w>": 278, ",</w>": 279, "q</w>": 280, "ħ</w>": 281, "¨</w>": 282, "G</w>": 283, "²</w>": 284, "ĺ</w>": 285, "ª</w>": 286, "¯</w>": 287, "j</w>": 288, "]</w>": 289, "ļ</w>": 290, "Ŀ</w>": 291, "¤</w>": 292, "ŀ</w>": 293, "½</w>": 294, "IJ</w>": 295, "'</w>": 296, "Ń</w>": 297, "°</w>": 298, "ľ</w>": 299, "></w>": 300, "¶</w>": 301, "į</w>": 302, "¦</w>": 303, "|</w>": 304, "¼</w>": 305, "¢</w>": 306, "´</w>": 307, "Ĩ</w>": 308, "Q</w>": 309, "Y</w>": 310, "Ľ</w>": 311, "ĵ</w>": 312, "ij</w>": 313, "ķ</w>": 314, "Ĭ</w>": 315, "¾</w>": 316, ";</w>": 317, "(</w>": 318, "¬</w>": 319, "@</w>": 320, "ĭ</w>": 321, "Ĺ</w>": 322, "£</w>": 323, "Į</w>": 324, "#</w>": 325, "·</w>": 326, "*</w>": 327, "Ĵ</w>": 328, "®</w>": 329, ")</w>": 330, "^</w>": 331, "ı</w>": 332, "Ġ</w>": 333, "_</w>": 334, "Ł</w>": 335, "}</w>": 336, "ĥ</w>": 337, "\\</w>": 338, "¥</w>": 339, "<</w>": 340, "+</w>": 341, "=</w>": 342, "~</w>": 343, "\"</w>": 344, "!</w>": 345, "?</w>": 346, "`</w>": 347, "$</w>": 348, "Ċ</w>": 349, "/</w>": 350, "%</w>": 351, "&</w>": 352, ":</w>": 353, "Ġt": 354, "Ġth": 355, "Ġa": 356, "Ġthe</w>": 357, "in": 358, "Ġo": 359, "Ġ,</w>": 360, "Ġs": 361, "ed</w>": 362, "Ġw": 363, "er": 364, "Ġ.</w>": 365, "Ġi": 366, "re": 367, "Ġc": 368, "nd</w>": 369, "Ġf": 370, "Ġb": 371, "at": 372, "Ġof</w>": 373, "er</w>": 374, "en": 375, "ar": 376, "or": 377, "it": 378, "Ġp": 379, "Ġh": 380, "Ġand</w>": 381, "on": 382, "ing</w>": 383, "an": 384, "ro": 385, "Ġm": 386, "Ġd": 387, "es</w>": 388, "Ġin</w>": 389, "on</w>": 390, "Ġto</w>": 391, "ou": 392, "is": 393, "Ġa</w>": 394, "ic": 395, "ĠT": 396, "al": 397, "Ġl": 398, "Ġ=</w>": 399, "Ġre": 400, "Ġ\"</w>": 401, "es": 402, "ĠS": 403, "as</w>": 404, "al</w>": 405, "il": 406, "el": 407, "ion</w>": 408, "ĠA": 409, "ĠC": 410, "Ġ1": 411, "ĠĊ</w>": 412, "ur": 413, "ĠTh": 414, "Ġn": 415, "as": 416, "Ġ@": 417, "ec": 418, "om": 419, "ac": 420, "Ġe": 421, "Ġwas</w>": 422, "ĠM": 423, "or</w>": 424, "an</w>": 425, "am": 426, "en</w>": 427, "ol": 428, "Ġin": 429, "Ġg": 430, "Ġ'</w>": 431, "ĠB": 432, "ly</w>": 433, "at</w>": 434, "iv": 435, "ts</w>": 436, "ĠThe</w>": 437, "us": 438, "-@</w>": 439, "Ġ@-@</w>": 440, "is</w>": 441, "ĠI": 442, "Ġwh": 443, "ig": 444, "ĠH": 445, "Ġst": 446, "os": 447, "un": 448, "th": 449, "ĠP": 450, "Ġwit": 451, "Ġthat</w>": 452, "ir": 453, "Ġas</w>": 454, "em": 455, "Ġon</w>": 456, "ra": 457, "Ġfor</w>": 458, "ĠR": 459, "et": 460, "ow": 461, "Ġ2": 462, "id": 463, "ĠD": 464, "le</w>": 465, "Ġwith</w>": 466, "la": 467, "ent</w>": 468, "im": 469, "ĠF": 470, "ea": 471, "ion": 472, "Ġby</w>": 473, "Ġ)</w>": 474, "Ġ(</w>": 475, "Ġal": 476, "Ġcon": 477, "ent": 478, "ĠW": 479, "Ġis</w>": 480, "ere</w>": 481, "ĠG": 482, "ĠN": 483, "ĠL": 484, "Ġha": 485, "ers</w>": 486, "ri": 487, "th</w>": 488, "ted</w>": 489, "uc": 490, "ĠJ": 491, "Ġ19": 492, "ev": 493, "ul": 494, "Ġv": 495, "ce</w>": 496, "ation</w>": 497, "rom</w>": 498, "Ġbe": 499, "ĠE": 500, "in</w>": 501, "Ġthe": 502, "Ġfrom</w>": 503, "ĠO": 504, "ter</w>": 505, "Ġpro": 506, "Ġar": 507, "ad": 508, "Ġcom": 509, "ic</w>": 510, "ag": 511, "Ġhis</w>": 512, "Ġsh": 513, "Ġat</w>": 514, "ov": 515, "ies</w>": 516, "oo": 517, "pp": 518, "st": 519, "ch": 520, "Ġr": 521, "Ġ20": 522, "ay</w>": 523, "if": 524, "Ġwere</w>": 525, "Ġch": 526, "ut</w>": 527, "st</w>": 528, "ut": 529, "ds</w>": 530, "op": 531, "um": 532, "Ġit</w>": 533, "oc": 534, "ter": 535, "le": 536, "igh": 537, "ud": 538, "Ġex": 539, "ions</w>": 540, "ate</w>": 541, "ity</w>": 542, "ated</w>": 543, "Ġun": 544, "ep": 545, "qu": 546, "Ġno": 547, "ĠK": 548, "ive</w>": 549, "ist": 550, "Ġon": 551, "ame</w>": 552, "oun": 553, "ir</w>": 554, "ab": 555, "Ġâ": 556, "ing": 557, "Ġhe</w>": 558, "ld</w>": 559, "ug": 560, "ich</w>": 561, "Ġan</w>": 562, "ed": 563, "Ġk": 564, "ĠâĢ": 565, "Ġhad</w>": 566, "ve</w>": 567, "ain": 568, "Ġse": 569, "tion</w>": 570, "ore</w>": 571, "res": 572, "Ġwhich</w>": 573, "ĠIn</w>": 574, "od": 575, "ther</w>": 576, "ak": 577, "Ġsp": 578, "ar</w>": 579, "Ġy": 580, "ĠCh": 581, "ong</w>": 582, "Ġac": 583, "est</w>": 584, "ĠU": 585, "ap": 586, "ff": 587, "ally</w>": 588, "rit": 589, "ĠSt": 590, "ub": 591, "ge</w>": 592, "ber</w>": 593, "et</w>": 594, "Ġbe</w>": 595, "ear": 596, "Ġrec": 597, "ers": 598, "Ġfir": 599, "ot": 600, "Ġare</w>": 601, "Ġan": 602, "ch</w>": 603, "og": 604, "ia</w>": 605, "est": 606, "ine</w>": 607, "ill": 608, "and": 609, "el</w>": 610, "ary</w>": 611, "ew</w>": 612, "id</w>": 613, "Ġfor": 614, "Ġ;</w>": 615, "Ġcomp": 616, "ĠV": 617, "Ġinc": 618, "tr": 619, "Ġ200": 620, "Ġtheir</w>": 621, "us</w>": 622, "Ġbut</w>": 623, "ran": 624, "ical</w>": 625, "Ġfirst</w>": 626, "Ġde": 627, "Ġint": 628, "Ġro": 629, "so</w>": 630, "ĠâĢĵ</w>": 631, "Ġnot</w>": 632, "ding</w>": 633, "fter</w>": 634, "ure</w>": 635, "Ġpar": 636, "Ġ:</w>": 637, "ian</w>": 638, "Ġtw": 639, "ould</w>": 640, "Ġalso</w>": 641, "Ġits</w>": 642, "Ġwor": 643, "um</w>": 644, "Ġor</w>": 645, "ost</w>": 646, "00</w>": 647, "our": 648, "ard</w>": 649, "Ġres": 650, "mp": 651, "ue</w>": 652, "Ġab": 653, "ish</w>": 654, "Ġcont": 655, "Ġad": 656, "own</w>": 657, "all</w>": 658, "oug": 659, "Ġher</w>": 660, "ast</w>": 661, "Ġen": 662, "ome</w>": 663, "all": 664, "ded</w>": 665, "ow</w>": 666, "Ġhave</w>": 667, "Ġus": 668, "ear</w>": 669, "ack</w>": 670, "duc": 671, "ial</w>": 672, "ss": 673, "ents</w>": 674, "ain</w>": 675, "ting</w>": 676, "Ġone</w>": 677, "ess": 678, "Ġhas</w>": 679, "ight</w>": 680, "av": 681, "Ġev": 682, "out</w>": 683, "ay": 684, "ence</w>": 685, "Ġbeen</w>": 686, "ew": 687, "Ġtwo</w>": 688, "Ġcl": 689, "der</w>": 690, "ime</w>": 691, "ks</w>": 692, "ess</w>": 693, "ish": 694, ".@</w>": 695, "Ġ@.@</w>": 696, "Ġpla": 697, "Ġpl": 698, "Ġor": 699, "up</w>": 700, "ment</w>": 701, "uring</w>": 702, "oll": 703, "ĠIn": 704, "Ġthis</w>": 705, "Ġbec": 706, "Ġcomm": 707, "Ġdis": 708, "ater</w>": 709, "age</w>": 710, "Ġapp": 711, "ous</w>": 712, "ey</w>": 713, "il</w>": 714, "per": 715, "ĠAl": 716, "ional</w>": 717, "lud": 718, "ely</w>": 719, "tt": 720, "ile</w>": 721, "iz": 722, "Ġj": 723, "Ġwho</w>": 724, "Ġag": 725, "ib": 726, "Ġthey</w>": 727, "for": 728, "Ġov": 729, "ath": 730, "eg": 731, "Ġsc": 732, "ip": 733, "Ġ201": 734, "Ġ3": 735, "Ġper": 736, "ory</w>": 737, "Ġdes": 738, "ide</w>": 739, "Ġser": 740, "se</w>": 741, "ĠHe</w>": 742, "land</w>": 743, "ations</w>": 744, "ric": 745, "it</w>": 746, "res</w>": 747, "ered</w>": 748, "Ġpre": 749, "ĠSh": 750, "ance</w>": 751, "ort</w>": 752, "ant</w>": 753, ",@</w>": 754, "Ġ@,@</w>": 755, "ell</w>": 756, "ĠY": 757, "ned</w>": 758, "ell": 759, "ite</w>": 760, "Ġinclud": 761, "Ġrep": 762, "Ġafter</w>": 763, "Ġsuc": 764, "ree</w>": 765, "any</w>": 766, "im</w>": 767, "ort": 768, "Ġ18": 769, "Ġsu": 770, "ade</w>": 771, "our</w>": 772, "ĠUn": 773, "ĠIt</w>": 774, "ik": 775, "ĠMar": 776, "ember</w>": 777, "Ġ1</w>": 778, "een</w>": 779, "and</w>": 780, "Ġsec": 781, "ice</w>": 782, "Ġtime</w>": 783, "ĠAn": 784, "Ġinto</w>": 785, "Ġfin": 786, "Ġother</w>": 787, "Ġatt": 788, "ill</w>": 789, "ren": 790, "ach": 791, "ass": 792, "eral</w>": 793, "ese</w>": 794, "sh": 795, "als</w>": 796, "ition</w>": 797, "ough</w>": 798, "les</w>": 799, "amp": 800, "Ġwould</w>": 801, "Ġmore</w>": 802, "roug": 803, "rib": 804, "ery</w>": 805, "ace</w>": 806, "ĠA</w>": 807, "Ġplay": 808, "ited</w>": 809, "ked</w>": 810, "ist</w>": 811, "ied</w>": 812, "Ġ2</w>": 813, "ased</w>": 814, "ings</w>": 815, "ang": 816, "am</w>": 817, "ip</w>": 818, "Ġbo": 819, "able</w>": 820, "ty</w>": 821, "Ġchar": 822, "Ġcent": 823, "etw": 824, "ates</w>": 825, "rop": 826, "ĠI</w>": 827, "und</w>": 828, "ĠAm": 829, "ces</w>": 830, "oin": 831, "Ġinter": 832, "up": 833, "ct": 834, "one</w>": 835, "Ġtra": 836, "ant": 837, "ect": 838, "Ġall</w>": 839, "ef": 840, "Ġcons": 841, "ubl": 842, "ning</w>": 843, "ans</w>": 844, "Ġfe": 845, "ust</w>": 846, "Ġ0": 847, "Ġrem": 848, "ase</w>": 849, "ong": 850, "Ġwhen</w>": 851, "eb": 852, "ĠWh": 853, "Ġear": 854, "ever</w>": 855, "Ġover</w>": 856, "Ġkn": 857, "aus": 858, "Ġpos": 859, "ad</w>": 860, "erm": 861, "Ġshe</w>": 862, "Ġra": 863, "Ġduring</w>": 864, "ason</w>": 865, "vi": 866, "Ġexp": 867, "Ġlea": 868, "Ġel": 869, "Ġ4": 870, "Ġonly</w>": 871, "ond</w>": 872, "Ġdec": 873, "Ġacc": 874, "Ġoff": 875, "iss": 876, "Ġfl": 877, "ĠEn": 878, "ot</w>": 879, "ens": 880, "ose</w>": 881, "ake</w>": 882, "om</w>": 883, "Ġsev": 884, "ach</w>": 885, "etween</w>": 886, "ern": 887, "Ġ3</w>": 888, "Ġpr": 889, "Ġgro": 890, "ruc": 891, "Ġdi": 892, "Ġ199": 893, "ĠAr": 894, "Ġgame</w>": 895, "Ġhim</w>": 896, "ook</w>": 897, "Ġup</w>": 898, "Ġabout</w>": 899, "Ġrel": 900, "form": 901, "Ġthree</w>": 902, "att": 903, "ĠCom": 904, "Ġsa": 905, "ears</w>": 906, "Ġ5": 907, "ry</w>": 908, "Ġimp": 909, "Ġmost</w>": 910, "fer": 911, "Ġpres": 912, "Ġfil": 913, "Ġbetween</w>": 914, "Ġbeg": 915, "ph": 916, "ors</w>": 917, "Ġthan</w>": 918, "Ġrecor": 919, "ob": 920, "eric": 921, "ating</w>": 922, "Ġthroug": 923, "king</w>": 924, "Ġout</w>": 925, "Ġnum": 926, "ood</w>": 927, "ollow": 928, "act": 929, "uil": 930, "Ġcre": 931, "olog": 932, "ational</w>": 933, "Ġproduc": 934, "Ġwhile</w>": 935, "Ġlater</w>": 936, "Ġwrit": 937, "ex": 938, "Ġstar": 939, "Ġspec": 940, "ee": 941, "ished</w>": 942, "Ġreg": 943, "ision</w>": 944, "outh</w>": 945, "Ġrele": 946, "Ġass": 947, "Ġseason</w>": 948, "Ġmade</w>": 949, "ily</w>": 950, "ru": 951, "oy": 952, "tur": 953, "te</w>": 954, "Ġqu": 955, "Ġmov": 956, "ury</w>": 957, "ĠAmeric": 958, "ement</w>": 959, "cc": 960, "ound</w>": 961, "Ġlar": 962, "Ġform": 963, "ect</w>": 964, "Ġdef": 965, "Ġmus": 966, "ĠPar": 967, "Ġme": 968, "Ġsub": 969, "way</w>": 970, "op</w>": 971, "oh": 972, "eld</w>": 973, "ie</w>": 974, "emp": 975, "ames</w>": 976, "ern</w>": 977, "Ġnor": 978, "ived</w>": 979, "evel": 980, "Ġsuch</w>": 981, "ards</w>": 982, "Ġind": 983, "ike</w>": 984, "Ġgen": 985, "ert": 986, "Ġyear</w>": 987, "Ġused</w>": 988, "Ġnew</w>": 989, "Ġ5</w>": 990, "Ġalb": 991, "sp": 992, "yp": 993, "Ġwith": 994, "Ġwhere</w>": 995, "ics</w>": 996, "ĠThis</w>": 997, "Ġthem</w>": 998, "wn</w>": 999 }, "merges": [ "Ġ t", "Ġt h", "Ġ a", "Ġth e</w>", "i n", "Ġ o", "Ġ ,</w>", "Ġ s", "e d</w>", "Ġ w", "e r", "Ġ .</w>", "Ġ i", "r e", "Ġ c", "n d</w>", "Ġ f", "Ġ b", "a t", "Ġo f</w>", "e r</w>", "e n", "a r", "o r", "i t", "Ġ p", "Ġ h", "Ġa nd</w>", "o n", "in g</w>", "a n", "r o", "Ġ m", "Ġ d", "e s</w>", "Ġi n</w>", "o n</w>", "Ġt o</w>", "o u", "i s", "Ġ a</w>", "i c", "Ġ T", "a l", "Ġ l", "Ġ =</w>", "Ġ re", "Ġ \"</w>", "e s", "Ġ S", "a s</w>", "a l</w>", "i l", "e l", "i on</w>", "Ġ A", "Ġ C", "Ġ 1", "Ġ Ċ</w>", "u r", "ĠT h", "Ġ n", "a s", "Ġ @", "e c", "o m", "a c", "Ġ e", "Ġw as</w>", "Ġ M", "o r</w>", "a n</w>", "a m", "e n</w>", "o l", "Ġ in", "Ġ g", "Ġ '</w>", "Ġ B", "l y</w>", "a t</w>", "i v", "t s</w>", "ĠTh e</w>", "u s", "- @</w>", "Ġ@ -@</w>", "i s</w>", "Ġ I", "Ġw h", "i g", "Ġ H", "Ġs t", "o s", "u n", "t h", "Ġ P", "Ġw it", "Ġth at</w>", "i r", "Ġa s</w>", "e m", "Ġo n</w>", "r a", "Ġf or</w>", "Ġ R", "e t", "o w", "Ġ 2", "i d", "Ġ D", "l e</w>", "Ġwit h</w>", "l a", "en t</w>", "i m", "Ġ F", "e a", "i on", "Ġb y</w>", "Ġ )</w>", "Ġ (</w>", "Ġa l", "Ġc on", "en t", "Ġ W", "Ġi s</w>", "er e</w>", "Ġ G", "Ġ N", "Ġ L", "Ġh a", "er s</w>", "r i", "t h</w>", "t ed</w>", "u c", "Ġ J", "Ġ1 9", "e v", "u l", "Ġ v", "c e</w>", "at ion</w>", "ro m</w>", "Ġb e", "Ġ E", "i n</w>", "Ġth e", "Ġf rom</w>", "Ġ O", "t er</w>", "Ġp ro", "Ġa r", "a d", "Ġc om", "i c</w>", "a g", "Ġh is</w>", "Ġs h", "Ġa t</w>", "o v", "i es</w>", "o o", "p p", "s t", "c h", "Ġ r", "Ġ2 0", "a y</w>", "i f", "Ġw ere</w>", "Ġc h", "u t</w>", "s t</w>", "u t", "d s</w>", "o p", "u m", "Ġi t</w>", "o c", "t er", "l e", "ig h", "u d", "Ġe x", "ion s</w>", "at e</w>", "it y</w>", "at ed</w>", "Ġ un", "e p", "q u", "Ġn o", "Ġ K", "iv e</w>", "is t", "Ġo n", "am e</w>", "ou n", "i r</w>", "a b", "Ġ â", "in g", "Ġh e</w>", "l d</w>", "u g", "ic h</w>", "Ġa n</w>", "e d", "Ġ k", "Ġâ Ģ", "Ġha d</w>", "v e</w>", "a in", "Ġs e", "t ion</w>", "or e</w>", "re s", "Ġwh ich</w>", "ĠI n</w>", "o d", "th er</w>", "a k", "Ġs p", "a r</w>", "Ġ y", "ĠC h", "on g</w>", "Ġa c", "es t</w>", "Ġ U", "a p", "f f", "al ly</w>", "r it", "ĠS t", "u b", "g e</w>", "b er</w>", "e t</w>", "Ġb e</w>", "e ar", "Ġre c", "er s", "Ġf ir", "o t", "Ġar e</w>", "Ġa n", "c h</w>", "o g", "i a</w>", "es t", "in e</w>", "il l", "an d", "e l</w>", "ar y</w>", "e w</w>", "i d</w>", "Ġf or", "Ġ ;</w>", "Ġcom p", "Ġ V", "Ġin c", "t r", "Ġ20 0", "Ġthe ir</w>", "u s</w>", "Ġb ut</w>", "r an", "ic al</w>", "Ġfir st</w>", "Ġd e", "Ġin t", "Ġ ro", "s o</w>", "ĠâĢ ĵ</w>", "Ġno t</w>", "d ing</w>", "f ter</w>", "ur e</w>", "Ġp ar", "Ġ :</w>", "i an</w>", "Ġt w", "ou ld</w>", "Ġal so</w>", "Ġi ts</w>", "Ġw or", "u m</w>", "Ġo r</w>", "os t</w>", "0 0</w>", "ou r", "ar d</w>", "Ġre s", "m p", "u e</w>", "Ġa b", "is h</w>", "Ġcon t", "Ġa d", "ow n</w>", "al l</w>", "ou g", "Ġh er</w>", "as t</w>", "Ġ en", "om e</w>", "al l", "d ed</w>", "o w</w>", "Ġha ve</w>", "Ġ us", "ea r</w>", "ac k</w>", "d uc", "i al</w>", "s s", "en ts</w>", "a in</w>", "t ing</w>", "Ġon e</w>", "es s", "Ġh as</w>", "igh t</w>", "a v", "Ġe v", "ou t</w>", "a y", "en ce</w>", "Ġbe en</w>", "e w", "Ġtw o</w>", "Ġc l", "d er</w>", "im e</w>", "k s</w>", "es s</w>", "is h", ". @</w>", "Ġ@ .@</w>", "Ġp la", "Ġp l", "Ġo r", "u p</w>", "m ent</w>", "ur ing</w>", "ol l", "ĠI n", "Ġth is</w>", "Ġb ec", "Ġcom m", "Ġd is", "at er</w>", "ag e</w>", "Ġa pp", "ou s</w>", "e y</w>", "i l</w>", "p er", "ĠA l", "ion al</w>", "l ud", "el y</w>", "t t", "il e</w>", "i z", "Ġ j", "Ġwh o</w>", "Ġa g", "i b", "Ġthe y</w>", "f or", "Ġo v", "at h", "e g", "Ġs c", "i p", "Ġ20 1", "Ġ 3", "Ġp er", "or y</w>", "Ġd es", "id e</w>", "Ġs er", "s e</w>", "ĠH e</w>", "la nd</w>", "at ions</w>", "r ic", "i t</w>", "re s</w>", "er ed</w>", "Ġp re", "ĠS h", "an ce</w>", "or t</w>", "an t</w>", ", @</w>", "Ġ@ ,@</w>", "el l</w>", "Ġ Y", "n ed</w>", "el l", "it e</w>", "Ġinc lud", "Ġre p", "Ġa fter</w>", "Ġs uc", "re e</w>", "an y</w>", "i m</w>", "or t", "Ġ1 8", "Ġs u", "ad e</w>", "ou r</w>", "ĠU n", "ĠI t</w>", "i k", "ĠM ar", "em ber</w>", "Ġ 1</w>", "e en</w>", "a nd</w>", "Ġs ec", "ic e</w>", "Ġt ime</w>", "ĠA n", "Ġint o</w>", "Ġf in", "Ġo ther</w>", "Ġa tt", "il l</w>", "re n", "ac h", "as s", "er al</w>", "es e</w>", "s h", "al s</w>", "it ion</w>", "oug h</w>", "l es</w>", "am p", "Ġw ould</w>", "Ġm ore</w>", "ro ug", "ri b", "er y</w>", "ac e</w>", "Ġ A</w>", "Ġpla y", "it ed</w>", "k ed</w>", "is t</w>", "i ed</w>", "Ġ 2</w>", "as ed</w>", "ing s</w>", "an g", "a m</w>", "i p</w>", "Ġb o", "ab le</w>", "t y</w>", "Ġch ar", "Ġc ent", "et w", "at es</w>", "ro p", "Ġ I</w>", "u nd</w>", "ĠA m", "c es</w>", "o in", "Ġin ter", "u p", "c t", "on e</w>", "Ġt ra", "an t", "ec t", "Ġal l</w>", "e f", "Ġcon s", "ub l", "n ing</w>", "an s</w>", "Ġf e", "us t</w>", "Ġ 0", "Ġre m", "as e</w>", "on g", "Ġwh en</w>", "e b", "ĠW h", "Ġe ar", "ev er</w>", "Ġov er</w>", "Ġk n", "a us", "Ġp os", "a d</w>", "er m", "Ġsh e</w>", "Ġ ra", "Ġd uring</w>", "as on</w>", "v i", "Ġex p", "Ġl ea", "Ġ el", "Ġ 4", "Ġon ly</w>", "o nd</w>", "Ġd ec", "Ġac c", "Ġo ff", "is s", "Ġf l", "ĠE n", "o t</w>", "en s", "os e</w>", "ak e</w>", "o m</w>", "Ġs ev", "ac h</w>", "etw een</w>", "er n", "Ġ 3</w>", "Ġp r", "Ġg ro", "r uc", "Ġd i", "Ġ19 9", "ĠA r", "Ġg ame</w>", "Ġh im</w>", "oo k</w>", "Ġ up</w>", "Ġab out</w>", "Ġre l", "for m", "Ġth ree</w>", "at t", "ĠC om", "Ġs a", "ear s</w>", "Ġ 5", "r y</w>", "Ġi mp", "Ġm ost</w>", "f er", "Ġp res", "Ġf il", "Ġb etween</w>", "Ġbe g", "p h", "or s</w>", "Ġth an</w>", "Ġrec or", "o b", "er ic", "at ing</w>", "Ġth roug", "k ing</w>", "Ġo ut</w>", "Ġn um", "oo d</w>", "oll ow", "ac t", "u il", "Ġc re", "ol og", "at ional</w>", "Ġpro duc", "Ġwh ile</w>", "Ġl ater</w>", "Ġw rit", "e x", "Ġst ar", "Ġsp ec", "e e", "ish ed</w>", "Ġre g", "is ion</w>", "ou th</w>", "Ġre le", "Ġa ss", "Ġse ason</w>", "Ġm ade</w>", "il y</w>", "r u", "o y", "t ur", "t e</w>", "Ġ qu", "Ġm ov", "ur y</w>", "ĠAm eric", "em ent</w>", "c c", "ou nd</w>", "Ġl ar", "Ġfor m", "ec t</w>", "Ġde f", "Ġm us", "ĠP ar", "Ġm e", "Ġs ub", "w ay</w>", "o p</w>", "o h", "el d</w>", "i e</w>", "em p", "am es</w>", "er n</w>", "Ġn or", "iv ed</w>", "ev el", "Ġsuc h</w>", "ar ds</w>", "Ġin d", "ik e</w>", "Ġg en", "er t", "Ġy ear</w>", "Ġus ed</w>", "Ġn ew</w>", "Ġ 5</w>", "Ġal b", "s p", "y p", "Ġwit h", "Ġwh ere</w>", "ic s</w>", "ĠTh is</w>", "Ġthe m</w>", "w n</w>" ] } }
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<|startoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"special": true
},
{
"id": 1,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"special": true
}
],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "NFC"
},
{
"type": "Replace",
"pattern": {
"Regex": "\\s+"
},
"content": " "
},
{
"type": "Lowercase"
}
]
},
"pre_tokenizer": {
"type": "Sequence",
"pretokenizers": [
{
"type": "Split",
"pattern": {
"Regex": "'s|'t|'re|'ve|'m|'ll|'d|[\\p{L}]+|[\\p{N}]|[^\\s\\p{L}\\p{N}]+"
},
"behavior": "Removed",
"invert": true
},
{
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": true
}
]
},
"post_processor": {
"type": "RobertaProcessing",
"sep": [
"<|endoftext|>",
1
],
"cls": [
"<|startoftext|>",
0
],
"trim_offsets": false,
"add_prefix_space": false
},
"decoder": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": "<|endoftext|>",
"continuing_subword_prefix": "",
"end_of_word_suffix": "</w>",
"fuse_unk": false,
"byte_fallback": false,
"vocab": {
"<|startoftext|>": 0,
"<|endoftext|>": 1,
"!": 2,
"\"": 3,
"#": 4,
"$": 5,
"%": 6,
"&": 7,
"'": 8,
"(": 9,
")": 10,