colbert-atto-840-8d / tokenizer.json
davidmezzetti's picture
Initial model
3674079
Raw History Blame Contribute Delete
7.6 kB
{
"version": "1.0",
"truncation": {
"direction": "Right",
"max_length": 449,
"strategy": "LongestFirst",
"stride": 0
},
"padding": {
"strategy": "BatchLongest",
"direction": "Right",
"pad_to_multiple_of": null,
"pad_id": 4,
"pad_type_id": 0,
"pad_token": "[MASK]"
},
"added_tokens": [
{
"id": 0,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "[UNK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "[CLS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "[SEP]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 4,
"content": "[MASK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 250,
"content": "[Q] ",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"special": false
},
{
"id": 251,
"content": "[D] ",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"special": false
}
],
"normalizer": {
"type": "BertNormalizer",
"clean_text": true,
"handle_chinese_chars": true,
"strip_accents": null,
"lowercase": true
},
"pre_tokenizer": {
"type": "BertPreTokenizer"
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"SpecialToken": {
"id": "[CLS]",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[SEP]",
"type_id": 0
}
}
],
"pair": [
{
"SpecialToken": {
"id": "[CLS]",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[SEP]",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
},
{
"SpecialToken": {
"id": "[SEP]",
"type_id": 1
}
}
],
"special_tokens": {
"[CLS]": {
"id": "[CLS]",
"ids": [
2
],
"tokens": [
"[CLS]"
]
},
"[SEP]": {
"id": "[SEP]",
"ids": [
3
],
"tokens": [
"[SEP]"
]
}
}
},
"decoder": {
"type": "WordPiece",
"prefix": "##",
"cleanup": true
},
"model": {
"type": "WordPiece",
"unk_token": "[UNK]",
"continuing_subword_prefix": "##",
"max_input_chars_per_word": 100,
"vocab": {
"[PAD]": 0,
"[UNK]": 1,
"[CLS]": 2,
"[SEP]": 3,
"[MASK]": 4,
"!": 5,
"\"": 6,
"#": 7,
"$": 8,
"%": 9,
"&": 10,
"'": 11,
"(": 12,
")": 13,
"*": 14,
"+": 15,
",": 16,
"-": 17,
".": 18,
"/": 19,
"0": 20,
"1": 21,
"2": 22,
"3": 23,
"4": 24,
"5": 25,
"6": 26,
"7": 27,
"8": 28,
"9": 29,
":": 30,
";": 31,
"<": 32,
"=": 33,
">": 34,
"?": 35,
"@": 36,
"[": 37,
"\\": 38,
"]": 39,
"^": 40,
"_": 41,
"`": 42,
"a": 43,
"b": 44,
"c": 45,
"d": 46,
"e": 47,
"f": 48,
"g": 49,
"h": 50,
"i": 51,
"j": 52,
"k": 53,
"l": 54,
"m": 55,
"n": 56,
"o": 57,
"p": 58,
"q": 59,
"r": 60,
"s": 61,
"t": 62,
"u": 63,
"v": 64,
"w": 65,
"x": 66,
"y": 67,
"z": 68,
"{": 69,
"|": 70,
"}": 71,
"~": 72,
"##n": 73,
"##j": 74,
"##a": 75,
"##e": 76,
"##y": 77,
"##3": 78,
"##1": 79,
"##9": 80,
"##0": 81,
"##d": 82,
"##l": 83,
"##s": 84,
"##t": 85,
"##h": 86,
"##u": 87,
"##p": 88,
"##r": 89,
"##o": 90,
"##i": 91,
"##v": 92,
"##c": 93,
"##b": 94,
"##z": 95,
"##g": 96,
"##w": 97,
"##k": 98,
"##m": 99,
"##x": 100,
"##f": 101,
"##7": 102,
"##2": 103,
"##5": 104,
"##8": 105,
"##4": 106,
"##6": 107,
"##q": 108,
"##he": 109,
"##er": 110,
"the": 111,
"##in": 112,
"##on": 113,
"##an": 114,
"##or": 115,
"##ed": 116,
"##es": 117,
"##en": 118,
"##at": 119,
"in": 120,
"##ar": 121,
"##al": 122,
"of": 123,
"an": 124,
"##is": 125,
"##it": 126,
"##ic": 127,
"##as": 128,
"##ing": 129,
"and": 130,
"##ion": 131,
"##ou": 132,
"to": 133,
"##om": 134,
"##re": 135,
"##il": 136,
"##el": 137,
"##ent": 138,
"##ol": 139,
"##am": 140,
"##ec": 141,
"##st": 142,
"##et": 143,
"##ur": 144,
"##ac": 145,
"19": 146,
"st": 147,
"##ad": 148,
"##ro": 149,
"th": 150,
"##le": 151,
"##iv": 152,
"on": 153,
"was": 154,
"##ers": 155,
"for": 156,
"20": 157,
"##ation": 158,
"##ir": 159,
"##ig": 160,
"re": 161,
"##ot": 162,
"as": 163,
"##id": 164,
"##ter": 165,
"##ay": 166,
"al": 167,
"##ly": 168,
"##em": 169,
"is": 170,
"##ch": 171,
"##ist": 172,
"##th": 173,
"fr": 174,
"##im": 175,
"##us": 176,
"##ith": 177,
"##ow": 178,
"##ut": 179,
"ch": 180,
"with": 181,
"##and": 182,
"he": 183,
"##ul": 184,
"by": 185,
"##ag": 186,
"at": 187,
"con": 188,
"##ts": 189,
"be": 190,
"##un": 191,
"##ce": 192,
"##os": 193,
"##um": 194,
"##op": 195,
"##oun": 196,
"com": 197,
"un": 198,
"wh": 199,
"##oc": 200,
"##ain": 201,
"de": 202,
"##her": 203,
"pro": 204,
"##ian": 205,
"##ver": 206,
"from": 207,
"##ct": 208,
"##est": 209,
"##ap": 210,
"##ri": 211,
"se": 212,
"pl": 213,
"##av": 214,
"##ab": 215,
"that": 216,
"##ess": 217,
"##all": 218,
"##ber": 219,
"##ia": 220,
"##ity": 221,
"ar": 222,
"##ies": 223,
"##od": 224,
"ne": 225,
"##ect": 226,
"ex": 227,
"##ate": 228,
"it": 229,
"le": 230,
"or": 231,
"en": 232,
"##if": 233,
"pr": 234,
"##art": 235,
"sp": 236,
"his": 237,
"201": 238,
"##ard": 239,
"su": 240,
"##ill": 241,
"##ud": 242,
"##ak": 243,
"##ip": 244,
"##igh": 245,
"tr": 246,
"##our": 247,
"##ov": 248,
"##ary": 249
}
}
}