colbert-atto-9k-16d / tokenizer.json
davidmezzetti's picture
Initial model
969ef90
Raw History Blame Contribute Delete
22.2 kB
{ "version": "1.0", "truncation": { "direction": "Right", "max_length": 399, "strategy": "LongestFirst", "stride": 0 }, "padding": { "strategy": "BatchLongest", "direction": "Right", "pad_to_multiple_of": null, "pad_id": 4, "pad_type_id": 0, "pad_token": "[MASK]" }, "added_tokens": [ { "id": 0, "content": "[PAD]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "[UNK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "[CLS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "[SEP]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "[MASK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1000, "content": "[Q] ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 1001, "content": "[D] ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false } ], "normalizer": { "type": "BertNormalizer", "clean_text": true, "handle_chinese_chars": true, "strip_accents": null, "lowercase": true }, "pre_tokenizer": { "type": "BertPreTokenizer" }, "post_processor": { "type": "TemplateProcessing", "single": [ { "SpecialToken": { "id": "[CLS]", "type_id": 0 } }, { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "[SEP]", "type_id": 0 } } ], "pair": [ { "SpecialToken": { "id": "[CLS]", "type_id": 0 } }, { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "[SEP]", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } }, { "SpecialToken": { "id": "[SEP]", "type_id": 1 } } ], "special_tokens": { "[CLS]": { "id": "[CLS]", "ids": [ 2 ], "tokens": [ "[CLS]" ] }, "[SEP]": { "id": "[SEP]", "ids": [ 3 ], "tokens": [ "[SEP]" ] } } }, "decoder": { "type": "WordPiece", "prefix": "##", "cleanup": true }, "model": { "type": "WordPiece", "unk_token": "[UNK]", "continuing_subword_prefix": "##", "max_input_chars_per_word": 100, "vocab": { "[PAD]": 0, "[UNK]": 1, "[CLS]": 2, "[SEP]": 3, "[MASK]": 4, "!": 5, "\"": 6, "#": 7, "$": 8, "%": 9, "&": 10, "'": 11, "(": 12, ")": 13, "*": 14, "+": 15, ",": 16, "-": 17, ".": 18, "/": 19, "0": 20, "1": 21, "2": 22, "3": 23, "4": 24, "5": 25, "6": 26, "7": 27, "8": 28, "9": 29, ":": 30, ";": 31, "<": 32, "=": 33, ">": 34, "?": 35, "@": 36, "[": 37, "\\": 38, "]": 39, "^": 40, "_": 41, "`": 42, "a": 43, "b": 44, "c": 45, "d": 46, "e": 47, "f": 48, "g": 49, "h": 50, "i": 51, "j": 52, "k": 53, "l": 54, "m": 55, "n": 56, "o": 57, "p": 58, "q": 59, "r": 60, "s": 61, "t": 62, "u": 63, "v": 64, "w": 65, "x": 66, "y": 67, "z": 68, "{": 69, "|": 70, "}": 71, "~": 72, "##0": 73, "##4": 74, "##3": 75, "##2": 76, "##5": 77, "##7": 78, "##a": 79, "##l": 80, "##f": 81, "##s": 82, "##h": 83, "##t": 84, "##e": 85, "##r": 86, "##c": 87, "##k": 88, "##8": 89, "##i": 90, "##y": 91, "##v": 92, "##x": 93, "##o": 94, "##m": 95, "##n": 96, "##d": 97, "##z": 98, "##w": 99, "##u": 100, "##p": 101, "##6": 102, "##9": 103, "##1": 104, "##g": 105, "##b": 106, "##j": 107, "##q": 108, "##he": 109, "##er": 110, "the": 111, "##in": 112, "##on": 113, "##an": 114, "##or": 115, "##ed": 116, "##es": 117, "##en": 118, "##at": 119, "in": 120, "##ar": 121, "##al": 122, "of": 123, "an": 124, "##is": 125, "##it": 126, "##ic": 127, "##as": 128, "##ing": 129, "and": 130, "##ion": 131, "##ou": 132, "to": 133, "##om": 134, "##re": 135, "##il": 136, "##el": 137, "##ent": 138, "##ol": 139, "##am": 140, "##ec": 141, "##st": 142, "##et": 143, "##ur": 144, "##ac": 145, "19": 146, "st": 147, "##ad": 148, "##ro": 149, "th": 150, "##le": 151, "##iv": 152, "on": 153, "was": 154, "##ers": 155, "for": 156, "20": 157, "##ation": 158, "##ir": 159, "##ig": 160, "re": 161, "##ot": 162, "as": 163, "##id": 164, "##ter": 165, "##ay": 166, "al": 167, "##ly": 168, "##em": 169, "is": 170, "##ch": 171, "##ist": 172, "##th": 173, "fr": 174, "##im": 175, "##us": 176, "##ith": 177, "##ow": 178, "##ut": 179, "ch": 180, "with": 181, "##and": 182, "he": 183, "##ul": 184, "by": 185, "##ag": 186, "at": 187, "con": 188, "##ts": 189, "be": 190, "##un": 191, "##ce": 192, "##os": 193, "##um": 194, "##op": 195, "##oun": 196, "com": 197, "un": 198, "wh": 199, "##oc": 200, "##ain": 201, "de": 202, "##her": 203, "pro": 204, "##ian": 205, "##ver": 206, "from": 207, "##ct": 208, "##est": 209, "##ap": 210, "##ri": 211, "se": 212, "pl": 213, "##av": 214, "##ab": 215, "that": 216, "##ess": 217, "##all": 218, "##ber": 219, "##ia": 220, "##ity": 221, "ar": 222, "##ies": 223, "##od": 224, "ne": 225, "##ect": 226, "ex": 227, "##ate": 228, "it": 229, "le": 230, "or": 231, "en": 232, "##if": 233, "pr": 234, "##art": 235, "sp": 236, "his": 237, "201": 238, "##ard": 239, "su": 240, "##ill": 241, "##ud": 242, "##ak": 243, "##ip": 244, "##igh": 245, "tr": 246, "##our": 247, "##ov": 248, "##ary": 249, "sh": 250, "200": 251, "##ich": 252, "##ated": 253, "##ort": 254, "##ish": 255, "##se": 256, "##mer": 257, "##ment": 258, "##ive": 259, "##fer": 260, "##ge": 261, "##ld": 262, "##ear": 263, "##ial": 264, "##og": 265, "##ere": 266, "##pp": 267, "##ong": 268, "##ie": 269, "18": 270, "cl": 271, "##ant": 272, "are": 273, "##ell": 274, "##ue": 275, "##ub": 276, "##ks": 277, "##ast": 278, "##ine": 279, "wor": 280, "comp": 281, "br": 282, "##ces": 283, "##ust": 284, "##ound": 285, "##ame": 286, "af": 287, "were": 288, "##ions": 289, "mar": 290, "##so": 291, "gr": 292, "##gh": 293, "##ical": 294, "##ore": 295, "play": 296, "##ign": 297, "##ost": 298, "new": 299, "also": 300, "##end": 301, "rec": 302, "ad": 303, "##ord": 304, "##pt": 305, "which": 306, "ser": 307, "sc": 308, "##tern": 309, "##ack": 310, "##uc": 311, "##qu": 312, "not": 313, "res": 314, "fir": 315, "ac": 316, "##ther": 317, "col": 318, "##ight": 319, "##age": 320, "inc": 321, "##ican": 322, "cent": 323, "##per": 324, "##ure": 325, "199": 326, "##ember": 327, "##ass": 328, "##der": 329, "man": 330, "first": 331, "##ire": 332, "her": 333, "##ous": 334, "whe": 335, "##ra": 336, "202": 337, "this": 338, "##man": 339, "car": 340, "had": 341, "##feren": 342, "##iz": 343, "part": 344, "cont": 345, "ab": 346, "she": 347, "##ount": 348, "comm": 349, "##ang": 350, "##ry": 351, "##ational": 352, "te": 353, "ind": 354, "##hed": 355, "pe": 356, "##ough": 357, "##ans": 358, "##ations": 359, "##ric": 360, "##ond": 361, "##ory": 362, "##orn": 363, "fil": 364, "year": 365, "amer": 366, "##land": 367, "##ice": 368, "##mp": 369, "el": 370, "##ok": 371, "int": 372, "tw": 373, "##ave": 374, "rep": 375, "##ru": 376, "##ome": 377, "##amp": 378, "has": 379, "go": 380, "after": 381, "all": 382, "rel": 383, "##ide": 384, "co": 385, "##ib": 386, "##ater": 387, "##ition": 388, "##out": 389, "its": 390, "per": 391, "referen": 392, "ag": 393, "us": 394, "one": 395, "##ey": 396, "##ach": 397, "par": 398, "their": 399, "pol": 400, "bo": 401, "des": 402, "film": 403, "##act": 404, "references": 405, "ro": 406, "sch": 407, "##ace": 408, "can": 409, "app": 410, "198": 411, "##ens": 412, "##ates": 413, "fe": 414, "##wn": 415, "who": 416, "##ime": 417, "but": 418, "##ited": 419, "##ree": 420, "american": 421, "dis": 422, "##ual": 423, "##ult": 424, "jo": 425, "##own": 426, "##rit": 427, "##ail": 428, "##ople": 429, "##ubl": 430, "##aw": 431, "##ath": 432, "act": 433, "##ff": 434, "##ivers": 435, "##ik": 436, "##lud": 437, "ed": 438, "fin": 439, "##00": 440, "off": 441, "two": 442, "##ence": 443, "##ball": 444, "##ents": 445, "##ors": 446, "##gr": 447, "##les": 448, "##ild": 449, "##ool": 450, "mus": 451, "ass": 452, "197": 453, "dec": 454, "they": 455, "##ite": 456, "##ance": 457, "includ": 458, "people": 459, "bec": 460, "##ould": 461, "reg": 462, "##ug": 463, "ph": 464, "other": 465, "fo": 466, "##uring": 467, "we": 468, "##oll": 469, "sec": 470, "##iss": 471, "##ade": 472, "##ward": 473, "bas": 474, "##ied": 475, "eng": 476, "bl": 477, "lin": 478, "##ile": 479, "count": 480, "pre": 481, "ev": 482, "##ury": 483, "##hip": 484, "me": 485, "fl": 486, "serv": 487, "gen": 488, "have": 489, "up": 490, "##for": 491, "##io": 492, "##te": 493, "##we": 494, "##ings": 495, "univers": 496, "may": 497, "sou": 498, "##hn": 499, "min": 500, "##ason": 501, "bu": 502, "##old": 503, "over": 504, "196": 505, "hist": 506, "school": 507, "out": 508, "nor": 509, "##ely": 510, "been": 511, "##ics": 512, "dist": 513, "pres": 514, "##eld": 515, "am": 516, "war": 517, "sy": 518, "mon": 519, "loc": 520, "ma": 521, "##ose": 522, "##able": 523, "during": 524, "publ": 525, "##ft": 526, "ear": 527, "##ished": 528, "exp": 529, "kn": 530, "time": 531, "##als": 532, "foot": 533, "art": 534, "##ision": 535, "##gan": 536, "med": 537, "##az": 538, "national": 539, "work": 540, "##ton": 541, "att": 542, "win": 543, "##vel": 544, "##urn": 545, "##ral": 546, "cons": 547, "under": 548, "char": 549, "team": 550, "acc": 551, "dr": 552, "##rib": 553, "university": 554, "elect": 555, "prod": 556, "##ames": 557, "when": 558, "##olog": 559, "world": 560, "united": 561, "##ict": 562, "##ock": 563, "17": 564, "##ale": 565, "bet": 566, "##ah": 567, "194": 568, "there": 569, "century": 570, "bro": 571, "##ists": 572, "gre": 573, "##way": 574, "est": 575, "gu": 576, "into": 577, "##one": 578, "stud": 579, "football": 580, "##int": 581, "##eas": 582, "##cc": 583, "195": 584, "season": 585, "##ction": 586, "##oup": 587, "no": 588, "##ick": 589, "##anc": 590, "state": 591, "po": 592, "born": 593, "sub": 594, "##ev": 595, "##au": 596, "##ake": 597, "spec": 598, "##eral": 599, "sing": 600, "players": 601, "found": 602, "##ix": 603, "south": 604, "record": 605, "##ious": 606, "##rap": 607, "##ily": 608, "##ons": 609, "##ife": 610, "inter": 611, "##ternal": 612, "alb": 613, "nov": 614, "##oth": 615, "city": 616, "trans": 617, "ret": 618, "##let": 619, "states": 620, "air": 621, "##ep": 622, "form": 623, "pop": 624, "##ank": 625, "dep": 626, "##ince": 627, "high": 628, "him": 629, "##emb": 630, "##ier": 631, "will": 632, "thr": 633, "more": 634, "##ollow": 635, "##oci": 636, "##ative": 637, "10": 638, "so": 639, "second": 640, "again": 641, "##att": 642, "follow": 643, "would": 644, "def": 645, "##hes": 646, "later": 647, "##vern": 648, "##form": 649, "prov": 650, "##ork": 651, "##ouse": 652, "bir": 653, "most": 654, "fam": 655, "dire": 656, "where": 657, "imp": 658, "##lish": 659, "ap": 660, "##ities": 661, "external": 662, "years": 663, "##ood": 664, "##ina": 665, "193": 666, "##ating": 667, "inst": 668, "16": 669, "links": 670, "##air": 671, "##up": 672, "end": 673, "##ween": 674, "##res": 675, "liv": 676, "three": 677, "##hy": 678, "govern": 679, "between": 680, "qu": 681, "wom": 682, "group": 683, "##ular": 684, "15": 685, "##ann": 686, "##ement": 687, "cap": 688, "rem": 689, "only": 690, "north": 691, "rev": 692, "writ": 693, "met": 694, "jun": 695, "##son": 696, "cr": 697, "how": 698, "##enn": 699, "ter": 700, "some": 701, "about": 702, "##ional": 703, "music": 704, "##ague": 705, "award": 706, "releas": 707, "song": 708, "direct": 709, "##ograp": 710, "sa": 711, "la": 712, "mov": 713, "ann": 714, "inv": 715, "bar": 716, "through": 717, "album": 718, "sur": 719, "ge": 720, "sm": 721, "##ific": 722, "compan": 723, "champ": 724, "##enc": 725, "ger": 726, "op": 727, "12": 728, "aust": 729, "##lo": 730, "while": 731, "##iam": 732, "##ery": 733, "series": 734, "##rict": 735, "day": 736, "192": 737, "list": 738, "##aus": 739, "##orts": 740, "id": 741, "arm": 742, "set": 743, "back": 744, "four": 745, "care": 746, "made": 747, "cre": 748, "##sp": 749, "num": 750, "history": 751, "offic": 752, "polit": 753, "john": 754, "##ounc": 755, "jan": 756, "##ss": 757, "then": 758, "county": 759, "##ally": 760, "bel": 761, "them": 762, "league": 763, "##ress": 764, "##itt": 765, "see": 766, "##ident": 767, "such": 768, "lar": 769, "intern": 770, "##ths": 771, "##fore": 772, "##erson": 773, "lead": 774, "oper": 775, "##ph": 776, "house": 777, "known": 778, "##uss": 779, "mod": 780, "sw": 781, "supp": 782, "191": 783, "##ese": 784, "##0s": 785, "##ield": 786, "brit": 787, "##ley": 788, "estab": 789, "##und": 790, "##ures": 791, "men": 792, "##ten": 793, "##velop": 794, "used": 795, "call": 796, "ju": 797, "mat": 798, "commun": 799, "women": 800, "death": 801, "prof": 802, "##are": 803, "##ys": 804, "add": 805, "11": 806, "##oss": 807, "including": 808, "desc": 809, "club": 810, "div": 811, "##ket": 812, "than": 813, "##ern": 814, "##ism": 815, "mil": 816, "##ather": 817, "films": 818, "##iew": 819, "life": 820, "main": 821, "german": 822, "became": 823, "well": 824, "develop": 825, "tel": 826, "do": 827, "##til": 828, "const": 829, "##ourn": 830, "district": 831, "##ope": 832, "class": 833, "build": 834, "##fl": 835, "general": 836, "##ins": 837, "##ower": 838, "public": 839, "rece": 840, "##erv": 841, "being": 842, "##ural": 843, "##els": 844, "early": 845, "##ont": 846, "sign": 847, "##bs": 848, "##its": 849, "before": 850, "##ai": 851, "champions": 852, "str": 853, "cor": 854, "against": 855, "har": 856, "21": 857, "show": 858, "##ments": 859, "international": 860, "aug": 861, "##ase": 862, "##ise": 863, "oct": 864, "##ection": 865, "14": 866, "orig": 867, "##ert": 868, "west": 869, "em": 870, "both": 871, "march": 872, "these": 873, "town": 874, "dem": 875, "family": 876, "##red": 877, "law": 878, "13": 879, "career": 880, "sept": 881, "##uary": 882, "align": 883, "##urch": 884, "coll": 885, "##amed": 886, "colle": 887, "##other": 888, "##ived": 889, "leg": 890, "sl": 891, "august": 892, "cro": 893, "##ister": 894, "##ike": 895, "chr": 896, "##iver": 897, "##eng": 898, "im": 899, "##hen": 900, "september": 901, "##ober": 902, "games": 903, "many": 904, "open": 905, "following": 906, "##yp": 907, "long": 908, "former": 909, "cour": 910, "cal": 911, "##min": 912, "design": 913, "##co": 914, "october": 915, "##ke": 916, "##ians": 917, "austral": 918, "origin": 919, "##ution": 920, "##ected": 921, "##ained": 922, "##ning": 923, "red": 924, "var": 925, "progr": 926, "game": 927, "british": 928, "##ole": 929, "king": 930, "##ril": 931, "january": 932, "##stem": 933, "organ": 934, "own": 935, "produc": 936, "##ull": 937, "##ae": 938, "eur": 939, "contr": 940, "since": 941, "sim": 942, "until": 943, "##ever": 944, "government": 945, "sever": 946, "june": 947, "##lin": 948, "run": 949, "##ner": 950, "july": 951, "memb": 952, "##ene": 953, "land": 954, "york": 955, "val": 956, "miss": 957, "associ": 958, "won": 959, "system": 960, "company": 961, "##ized": 962, "cur": 963, "##led": 964, "appear": 965, "living": 966, "##ology": 967, "april": 968, "star": 969, "##ty": 970, "lo": 971, "english": 972, "##by": 973, "november": 974, "members": 975, "vill": 976, "av": 977, "arch": 978, "##ind": 979, "port": 980, "number": 981, "##xt": 982, "based": 983, "head": 984, "ep": 985, "championship": 986, "december": 987, "##ica": 988, "maj": 989, "child": 990, "name": 991, "released": 992, "educ": 993, "ext": 994, "however": 995, "played": 996, "bus": 997, "##ash": 998, "canad": 999 } } }
{
"version": "1.0",
"truncation": {
"direction": "Right",
"max_length": 399,
"strategy": "LongestFirst",
"stride": 0
},
"padding": {
"strategy": "BatchLongest",
"direction": "Right",
"pad_to_multiple_of": null,
"pad_id": 4,
"pad_type_id": 0,
"pad_token": "[MASK]"
},
"added_tokens": [
{
"id": 0,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "[UNK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "[CLS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "[SEP]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 4,
"content": "[MASK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1000,
"content": "[Q] ",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"special": false
},
{
"id": 1001,
"content": "[D] ",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"special": false
}
],
"normalizer": {
"type": "BertNormalizer",
"clean_text": true,
"handle_chinese_chars": true,
"strip_accents": null,
"lowercase": true
},
"pre_tokenizer": {
"type": "BertPreTokenizer"
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"SpecialToken": {
"id": "[CLS]",
"type_id": 0
}
},