samdei-type-base / tokenizer.json
mengsay's picture
Model folders in the transformers format (Blenderbot classes, tokenizer, generation config)
0420834 verified
Raw History Blame Contribute Delete
9.26 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<pad>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "</s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "<unk>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 228,
"content": "<src_pad>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 229,
"content": "<src_s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 230,
"content": "</src>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 231,
"content": "<src_unk>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "Split",
"pattern": {
"Regex": "[\\s\\S]"
},
"behavior": "Isolated",
"invert": false
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "</src>",
"type_id": 0
}
}
],
"pair": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "</src>",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
},
{
"SpecialToken": {
"id": "</src>",
"type_id": 1
}
}
],
"special_tokens": {
"</src>": {
"id": "</src>",
"ids": [
230
],
"tokens": [
"</src>"
]
}
}
},
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "๎€€"
},
"content": ""
},
{
"type": "Fuse"
}
]
},
"model": {
"type": "WordLevel",
"vocab": {
"<pad>": 0,
"<s>": 1,
"</s>": 2,
"<unk>": 3,
"๎€€ ": 4,
"๎€€!": 5,
"๎€€\"": 6,
"๎€€#": 7,
"๎€€$": 8,
"๎€€%": 9,
"๎€€&": 10,
"๎€€'": 11,
"๎€€(": 12,
"๎€€)": 13,
"๎€€*": 14,
"๎€€+": 15,
"๎€€,": 16,
"๎€€-": 17,
"๎€€.": 18,
"๎€€/": 19,
"๎€€0": 20,
"๎€€1": 21,
"๎€€2": 22,
"๎€€3": 23,
"๎€€4": 24,
"๎€€5": 25,
"๎€€6": 26,
"๎€€7": 27,
"๎€€8": 28,
"๎€€9": 29,
"๎€€:": 30,
"๎€€;": 31,
"๎€€<": 32,
"๎€€=": 33,
"๎€€>": 34,
"๎€€?": 35,
"๎€€@": 36,
"๎€€A": 37,
"๎€€B": 38,
"๎€€C": 39,
"๎€€D": 40,
"๎€€E": 41,
"๎€€F": 42,
"๎€€G": 43,
"๎€€H": 44,
"๎€€I": 45,
"๎€€J": 46,
"๎€€K": 47,
"๎€€L": 48,
"๎€€M": 49,
"๎€€N": 50,
"๎€€O": 51,
"๎€€P": 52,
"๎€€Q": 53,
"๎€€R": 54,
"๎€€S": 55,
"๎€€T": 56,
"๎€€U": 57,
"๎€€V": 58,
"๎€€W": 59,
"๎€€X": 60,
"๎€€Y": 61,
"๎€€Z": 62,
"๎€€[": 63,
"๎€€]": 64,
"๎€€^": 65,
"๎€€_": 66,
"๎€€a": 67,
"๎€€b": 68,
"๎€€c": 69,
"๎€€d": 70,
"๎€€e": 71,
"๎€€f": 72,
"๎€€g": 73,
"๎€€h": 74,
"๎€€i": 75,
"๎€€j": 76,
"๎€€k": 77,
"๎€€l": 78,
"๎€€m": 79,
"๎€€n": 80,
"๎€€o": 81,
"๎€€p": 82,
"๎€€q": 83,
"๎€€r": 84,
"๎€€s": 85,
"๎€€t": 86,
"๎€€u": 87,
"๎€€v": 88,
"๎€€w": 89,
"๎€€x": 90,
"๎€€y": 91,
"๎€€z": 92,
"๎€€|": 93,
"๎€€~": 94,
"๎€€ยก": 95,
"๎€€ยซ": 96,
"๎€€ยฎ": 97,
"๎€€ยฐ": 98,
"๎€€ยท": 99,
"๎€€ยป": 100,
"๎€€ร—": 101,
"๎€€ร ": 102,
"๎€€รก": 103,
"๎€€รง": 104,
"๎€€รจ": 105,
"๎€€รฉ": 106,
"๎€€รญ": 107,
"๎€€ว”": 108,
"๎€€เธ": 109,
"๎€€เธ—": 110,
"๎€€เธ™": 111,
"๎€€เธ ": 112,
"๎€€เธข": 113,
"๎€€เธฃ": 114,
"๎€€เธฉ": 115,
"๎€€เธฒ": 116,
"๎€€เธด": 117,
"๎€€เน€": 118,
"๎€€เน„": 119,
"๎€€เน’": 120,
"๎€€เน•": 121,
"๎€€เบฅ": 122,
"๎€€เบง": 123,
"๎€€เบช": 124,
"๎€€เบฒ": 125,
"๎€€แž€": 126,
"๎€€แž": 127,
"๎€€แž‚": 128,
"๎€€แžƒ": 129,
"๎€€แž„": 130,
"๎€€แž…": 131,
"๎€€แž†": 132,
"๎€€แž‡": 133,
"๎€€แžˆ": 134,
"๎€€แž‰": 135,
"๎€€แžŠ": 136,
"๎€€แž‹": 137,
"๎€€แžŒ": 138,
"๎€€แž": 139,
"๎€€แžŽ": 140,
"๎€€แž": 141,
"๎€€แž": 142,
"๎€€แž‘": 143,
"๎€€แž’": 144,
"๎€€แž“": 145,
"๎€€แž”": 146,
"๎€€แž•": 147,
"๎€€แž–": 148,
"๎€€แž—": 149,
"๎€€แž˜": 150,
"๎€€แž™": 151,
"๎€€แžš": 152,
"๎€€แž›": 153,
"๎€€แžœ": 154,
"๎€€แž": 155,
"๎€€แžž": 156,
"๎€€แžŸ": 157,
"๎€€แž ": 158,
"๎€€แžก": 159,
"๎€€แžข": 160,
"๎€€แžฅ": 161,
"๎€€แžฆ": 162,
"๎€€แžง": 163,
"๎€€แžฉ": 164,
"๎€€แžช": 165,
"๎€€แžซ": 166,
"๎€€แžฌ": 167,
"๎€€แžญ": 168,
"๎€€แžฎ": 169,
"๎€€แžฏ": 170,
"๎€€แžฐ": 171,
"๎€€แžฑ": 172,
"๎€€แžฒ": 173,
"๎€€แžณ": 174,
"๎€€แžถ": 175,
"๎€€แžท": 176,
"๎€€แžธ": 177,
"๎€€แžน": 178,
"๎€€แžบ": 179,
"๎€€แžป": 180,
"๎€€แžผ": 181,
"๎€€แžฝ": 182,
"๎€€แžพ": 183,
"๎€€แžฟ": 184,
"๎€€แŸ€": 185,
"๎€€แŸ": 186,
"๎€€แŸ‚": 187,
"๎€€แŸƒ": 188,
"๎€€แŸ„": 189,
"๎€€แŸ…": 190,
"๎€€แŸ†": 191,
"๎€€แŸ‡": 192,
"๎€€แŸˆ": 193,
"๎€€แŸ‰": 194,
"๎€€แŸŠ": 195,
"๎€€แŸ‹": 196,
"๎€€แŸŒ": 197,
"๎€€แŸ": 198,
"๎€€แŸŽ": 199,
"๎€€แŸ": 200,
"๎€€แŸ": 201,
"๎€€แŸ‘": 202,
"๎€€แŸ’": 203,
"๎€€แŸ”": 204,
"๎€€แŸ•": 205,
"๎€€แŸ–": 206,
"๎€€แŸ—": 207,
"๎€€แŸ˜": 208,
"๎€€แŸ›": 209,
"๎€€โ€‘": 210,
"๎€€โ€“": 211,
"๎€€โ€”": 212,
"๎€€โ€˜": 213,
"๎€€โ€™": 214,
"๎€€โ€œ": 215,
"๎€€โ€": 216,
"๎€€โ€ข": 217,
"๎€€โ€ฆ": 218,
"๎€€โ€น": 219,
"๎€€โ€บ": 220,
"๎€€โ†’": 221,
"๎€€โˆ’": 222,
"๎€€โ—": 223,
"๎€€๏ฟฝ": 224,
"๎€€๐Ÿ˜€": 225,
"๎€€๐Ÿ˜›": 226,
"๎€€๐Ÿ™‚": 227,
"<src_pad>": 228,
"<src_s>": 229,
"</src>": 230,
"<src_unk>": 231,
" ": 232,
"!": 233,
"\"": 234,
"#": 235,
"$": 236,
"%": 237,
"&": 238,
"'": 239,
"(": 240,
")": 241,
"*": 242,
"+": 243,
",": 244,
"-": 245,
".": 246,
"/": 247,
"0": 248,
"1": 249,
"2": 250,
"3": 251,
"4": 252,
"5": 253,
"6": 254,
"7": 255,
"8": 256,
"9": 257,
":": 258,
";": 259,
"<": 260,
"=": 261,
">": 262,
"?": 263,
"@": 264,
"A": 265,
"B": 266,
"C": 267,
"D": 268,
"E": 269,
"F": 270,
"G": 271,
"H": 272,
"I": 273,
"J": 274,
"K": 275,
"L": 276,
"M": 277,
"N": 278,
"O": 279,
"P": 280,
"Q": 281,
"R": 282,
"S": 283,
"T": 284,
"U": 285,
"V": 286,
"W": 287,
"X": 288,
"Y": 289,
"Z": 290,
"[": 291,
"]": 292,
"^": 293,
"_": 294,
"a": 295,
"b": 296,
"c": 297,
"d": 298,
"e": 299,
"f": 300,
"g": 301,
"h": 302,
"i": 303,
"j": 304,
"k": 305,
"l": 306,
"m": 307,
"n": 308,
"o": 309,
"p": 310,
"q": 311,
"r": 312,
"s": 313,
"t": 314,
"u": 315,
"v": 316,
"w": 317,
"x": 318,
"y": 319,
"z": 320,
"|": 321,
"~": 322,
"ยก": 323,
"ยซ": 324,
"ยฎ": 325,
"ยฐ": 326,
"ยท": 327,
"ยป": 328,
"ร—": 329,
"ร ": 330,
"รก": 331,
"รง": 332,
"รจ": 333,
"รฉ": 334,
"รญ": 335,
"ว”": 336,
"ษ“": 337,
"ษ™": 338,
"เธ": 339,
"เธ—": 340,
"เธ™": 341,
"เธ ": 342,
"เธข": 343,
"เธฃ": 344,
"เธฉ": 345,
"เธฒ": 346,
"เธด": 347,
"เน€": 348,
"เน„": 349,
"เน’": 350,
"เน•": 351,
"เบฅ": 352,
"เบง": 353,
"เบช": 354,
"เบฒ": 355,
"แž€": 356,
"แž": 357,
"แž‚": 358,
"แžƒ": 359,
"แž„": 360,
"แž…": 361,
"แž†": 362,
"แž‡": 363,
"แžˆ": 364,
"แž‰": 365,
"แžŠ": 366,
"แž‹": 367,
"แžŒ": 368,
"แžŽ": 369,
"แž": 370,
"แž": 371,
"แž‘": 372,
"แž’": 373,
"แž“": 374,
"แž”": 375,
"แž•": 376,
"แž–": 377,
"แž—": 378,
"แž˜": 379,
"แž™": 380,
"แžš": 381,
"แž›": 382,
"แžœ": 383,
"แžŸ": 384,
"แž ": 385,
"แžก": 386,
"แžข": 387,
"แžง": 388,
"แžซ": 389,
"แžฌ": 390,
"แžฏ": 391,
"แžฒ": 392,
"แžถ": 393,
"แžท": 394,
"แžธ": 395,
"แžน": 396,
"แžบ": 397,
"แžป": 398,
"แžผ": 399,
"แžฝ": 400,
"แžพ": 401,
"แžฟ": 402,
"แŸ€": 403,
"แŸ": 404,
"แŸ‚": 405,
"แŸƒ": 406,
"แŸ„": 407,
"แŸ…": 408,
"แŸ†": 409,
"แŸ‡": 410,
"แŸˆ": 411,
"แŸ‰": 412,
"แŸŠ": 413,
"แŸ‹": 414,
"แŸŒ": 415,
"แŸ": 416,
"แŸ": 417,
"แŸ": 418,
"แŸ’": 419,
"แŸ”": 420,
"แŸ•": 421,
"แŸ–": 422,
"แŸ—": 423,
"แŸ˜": 424,
"แŸ›": 425,
"โ€‘": 426,
"โ€“": 427,
"โ€”": 428,
"โ€˜": 429,
"โ€™": 430,
"โ€œ": 431,
"โ€": 432,
"โ€ข": 433,
"โ€ฆ": 434,
"โ€น": 435,
"โ€บ": 436,
"โ†’": 437,
"โˆ’": 438,
"โ—": 439,
"๏ฟฝ": 440,
"๐Ÿ˜€": 441,
"๐Ÿ˜›": 442,
"๐Ÿ™‚": 443
},
"unk_token": "<src_unk>"
}
}