{ "version": "1.0", "truncation": { "direction": "Right", "max_length": 449, "strategy": "LongestFirst", "stride": 0 }, "padding": { "strategy": "BatchLongest", "direction": "Right", "pad_to_multiple_of": null, "pad_id": 4, "pad_type_id": 0, "pad_token": "[MASK]" }, "added_tokens": [ { "id": 0, "content": "[PAD]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "[UNK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "[CLS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "[SEP]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "[MASK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 250, "content": "[Q] ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 251, "content": "[D] ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false } ], "normalizer": { "type": "BertNormalizer", "clean_text": true, "handle_chinese_chars": true, "strip_accents": null, "lowercase": true }, "pre_tokenizer": { "type": "BertPreTokenizer" }, "post_processor": { "type": "TemplateProcessing", "single": [ { "SpecialToken": { "id": "[CLS]", "type_id": 0 } }, { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "[SEP]", "type_id": 0 } } ], "pair": [ { "SpecialToken": { "id": "[CLS]", "type_id": 0 } }, { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "[SEP]", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } }, { "SpecialToken": { "id": "[SEP]", "type_id": 1 } } ], "special_tokens": { "[CLS]": { "id": "[CLS]", "ids": [ 2 ], "tokens": [ "[CLS]" ] }, "[SEP]": { "id": "[SEP]", "ids": [ 3 ], "tokens": [ "[SEP]" ] } } }, "decoder": { "type": "WordPiece", "prefix": "##", "cleanup": true }, "model": { "type": "WordPiece", "unk_token": "[UNK]", "continuing_subword_prefix": "##", "max_input_chars_per_word": 100, "vocab": { "[PAD]": 0, "[UNK]": 1, "[CLS]": 2, "[SEP]": 3, "[MASK]": 4, "!": 5, "\"": 6, "#": 7, "$": 8, "%": 9, "&": 10, "'": 11, "(": 12, ")": 13, "*": 14, "+": 15, ",": 16, "-": 17, ".": 18, "/": 19, "0": 20, "1": 21, "2": 22, "3": 23, "4": 24, "5": 25, "6": 26, "7": 27, "8": 28, "9": 29, ":": 30, ";": 31, "<": 32, "=": 33, ">": 34, "?": 35, "@": 36, "[": 37, "\\": 38, "]": 39, "^": 40, "_": 41, "`": 42, "a": 43, "b": 44, "c": 45, "d": 46, "e": 47, "f": 48, "g": 49, "h": 50, "i": 51, "j": 52, "k": 53, "l": 54, "m": 55, "n": 56, "o": 57, "p": 58, "q": 59, "r": 60, "s": 61, "t": 62, "u": 63, "v": 64, "w": 65, "x": 66, "y": 67, "z": 68, "{": 69, "|": 70, "}": 71, "~": 72, "##n": 73, "##j": 74, "##a": 75, "##e": 76, "##y": 77, "##3": 78, "##1": 79, "##9": 80, "##0": 81, "##d": 82, "##l": 83, "##s": 84, "##t": 85, "##h": 86, "##u": 87, "##p": 88, "##r": 89, "##o": 90, "##i": 91, "##v": 92, "##c": 93, "##b": 94, "##z": 95, "##g": 96, "##w": 97, "##k": 98, "##m": 99, "##x": 100, "##f": 101, "##7": 102, "##2": 103, "##5": 104, "##8": 105, "##4": 106, "##6": 107, "##q": 108, "##he": 109, "##er": 110, "the": 111, "##in": 112, "##on": 113, "##an": 114, "##or": 115, "##ed": 116, "##es": 117, "##en": 118, "##at": 119, "in": 120, "##ar": 121, "##al": 122, "of": 123, "an": 124, "##is": 125, "##it": 126, "##ic": 127, "##as": 128, "##ing": 129, "and": 130, "##ion": 131, "##ou": 132, "to": 133, "##om": 134, "##re": 135, "##il": 136, "##el": 137, "##ent": 138, "##ol": 139, "##am": 140, "##ec": 141, "##st": 142, "##et": 143, "##ur": 144, "##ac": 145, "19": 146, "st": 147, "##ad": 148, "##ro": 149, "th": 150, "##le": 151, "##iv": 152, "on": 153, "was": 154, "##ers": 155, "for": 156, "20": 157, "##ation": 158, "##ir": 159, "##ig": 160, "re": 161, "##ot": 162, "as": 163, "##id": 164, "##ter": 165, "##ay": 166, "al": 167, "##ly": 168, "##em": 169, "is": 170, "##ch": 171, "##ist": 172, "##th": 173, "fr": 174, "##im": 175, "##us": 176, "##ith": 177, "##ow": 178, "##ut": 179, "ch": 180, "with": 181, "##and": 182, "he": 183, "##ul": 184, "by": 185, "##ag": 186, "at": 187, "con": 188, "##ts": 189, "be": 190, "##un": 191, "##ce": 192, "##os": 193, "##um": 194, "##op": 195, "##oun": 196, "com": 197, "un": 198, "wh": 199, "##oc": 200, "##ain": 201, "de": 202, "##her": 203, "pro": 204, "##ian": 205, "##ver": 206, "from": 207, "##ct": 208, "##est": 209, "##ap": 210, "##ri": 211, "se": 212, "pl": 213, "##av": 214, "##ab": 215, "that": 216, "##ess": 217, "##all": 218, "##ber": 219, "##ia": 220, "##ity": 221, "ar": 222, "##ies": 223, "##od": 224, "ne": 225, "##ect": 226, "ex": 227, "##ate": 228, "it": 229, "le": 230, "or": 231, "en": 232, "##if": 233, "pr": 234, "##art": 235, "sp": 236, "his": 237, "201": 238, "##ard": 239, "su": 240, "##ill": 241, "##ud": 242, "##ak": 243, "##ip": 244, "##igh": 245, "tr": 246, "##our": 247, "##ov": 248, "##ary": 249 } } }