samdei-type-base / channel /tokenizer.json
mengsay's picture
Model folders in the transformers format (Blenderbot classes, tokenizer, generation config)
0420834 verified
Raw History Blame Contribute Delete
9.22 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<pad>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "</s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "<unk>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 216,
"content": "<src_pad>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 217,
"content": "<src_s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 218,
"content": "</src>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 219,
"content": "<src_unk>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "Split",
"pattern": {
"Regex": "[\\s\\S]"
},
"behavior": "Isolated",
"invert": false
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "</src>",
"type_id": 0
}
}
],
"pair": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "</src>",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
},
{
"SpecialToken": {
"id": "</src>",
"type_id": 1
}
}
],
"special_tokens": {
"</src>": {
"id": "</src>",
"ids": [
218
],
"tokens": [
"</src>"
]
}
}
},
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "๎€€"
},
"content": ""
},
{
"type": "Fuse"
}
]
},
"model": {
"type": "WordLevel",
"vocab": {
"<pad>": 0,
"<s>": 1,
"</s>": 2,
"<unk>": 3,
"๎€€ ": 4,
"๎€€!": 5,
"๎€€\"": 6,
"๎€€#": 7,
"๎€€$": 8,
"๎€€%": 9,
"๎€€&": 10,
"๎€€'": 11,
"๎€€(": 12,
"๎€€)": 13,
"๎€€*": 14,
"๎€€+": 15,
"๎€€,": 16,
"๎€€-": 17,
"๎€€.": 18,
"๎€€/": 19,
"๎€€0": 20,
"๎€€1": 21,
"๎€€2": 22,
"๎€€3": 23,
"๎€€4": 24,
"๎€€5": 25,
"๎€€6": 26,
"๎€€7": 27,
"๎€€8": 28,
"๎€€9": 29,
"๎€€:": 30,
"๎€€;": 31,
"๎€€<": 32,
"๎€€=": 33,
"๎€€>": 34,
"๎€€?": 35,
"๎€€@": 36,
"๎€€A": 37,
"๎€€B": 38,
"๎€€C": 39,
"๎€€D": 40,
"๎€€E": 41,
"๎€€F": 42,
"๎€€G": 43,
"๎€€H": 44,
"๎€€I": 45,
"๎€€J": 46,
"๎€€K": 47,
"๎€€L": 48,
"๎€€M": 49,
"๎€€N": 50,
"๎€€O": 51,
"๎€€P": 52,
"๎€€Q": 53,
"๎€€R": 54,
"๎€€S": 55,
"๎€€T": 56,
"๎€€U": 57,
"๎€€V": 58,
"๎€€W": 59,
"๎€€X": 60,
"๎€€Y": 61,
"๎€€Z": 62,
"๎€€[": 63,
"๎€€]": 64,
"๎€€^": 65,
"๎€€_": 66,
"๎€€a": 67,
"๎€€b": 68,
"๎€€c": 69,
"๎€€d": 70,
"๎€€e": 71,
"๎€€f": 72,
"๎€€g": 73,
"๎€€h": 74,
"๎€€i": 75,
"๎€€j": 76,
"๎€€k": 77,
"๎€€l": 78,
"๎€€m": 79,
"๎€€n": 80,
"๎€€o": 81,
"๎€€p": 82,
"๎€€q": 83,
"๎€€r": 84,
"๎€€s": 85,
"๎€€t": 86,
"๎€€u": 87,
"๎€€v": 88,
"๎€€w": 89,
"๎€€x": 90,
"๎€€y": 91,
"๎€€z": 92,
"๎€€|": 93,
"๎€€~": 94,
"๎€€ยก": 95,
"๎€€ยซ": 96,
"๎€€ยฎ": 97,
"๎€€ยฐ": 98,
"๎€€ยท": 99,
"๎€€ยป": 100,
"๎€€ร—": 101,
"๎€€ร ": 102,
"๎€€รก": 103,
"๎€€รง": 104,
"๎€€รจ": 105,
"๎€€รฉ": 106,
"๎€€รญ": 107,
"๎€€ว”": 108,
"๎€€ษ“": 109,
"๎€€ษ™": 110,
"๎€€เธ": 111,
"๎€€เธ—": 112,
"๎€€เธ™": 113,
"๎€€เธ ": 114,
"๎€€เธข": 115,
"๎€€เธฃ": 116,
"๎€€เธฉ": 117,
"๎€€เธฒ": 118,
"๎€€เธด": 119,
"๎€€เน€": 120,
"๎€€เน„": 121,
"๎€€เน’": 122,
"๎€€เน•": 123,
"๎€€เบฅ": 124,
"๎€€เบง": 125,
"๎€€เบช": 126,
"๎€€เบฒ": 127,
"๎€€แž€": 128,
"๎€€แž": 129,
"๎€€แž‚": 130,
"๎€€แžƒ": 131,
"๎€€แž„": 132,
"๎€€แž…": 133,
"๎€€แž†": 134,
"๎€€แž‡": 135,
"๎€€แžˆ": 136,
"๎€€แž‰": 137,
"๎€€แžŠ": 138,
"๎€€แž‹": 139,
"๎€€แžŒ": 140,
"๎€€แžŽ": 141,
"๎€€แž": 142,
"๎€€แž": 143,
"๎€€แž‘": 144,
"๎€€แž’": 145,
"๎€€แž“": 146,
"๎€€แž”": 147,
"๎€€แž•": 148,
"๎€€แž–": 149,
"๎€€แž—": 150,
"๎€€แž˜": 151,
"๎€€แž™": 152,
"๎€€แžš": 153,
"๎€€แž›": 154,
"๎€€แžœ": 155,
"๎€€แžŸ": 156,
"๎€€แž ": 157,
"๎€€แžก": 158,
"๎€€แžข": 159,
"๎€€แžง": 160,
"๎€€แžซ": 161,
"๎€€แžฌ": 162,
"๎€€แžฏ": 163,
"๎€€แžฒ": 164,
"๎€€แžถ": 165,
"๎€€แžท": 166,
"๎€€แžธ": 167,
"๎€€แžน": 168,
"๎€€แžบ": 169,
"๎€€แžป": 170,
"๎€€แžผ": 171,
"๎€€แžฝ": 172,
"๎€€แžพ": 173,
"๎€€แžฟ": 174,
"๎€€แŸ€": 175,
"๎€€แŸ": 176,
"๎€€แŸ‚": 177,
"๎€€แŸƒ": 178,
"๎€€แŸ„": 179,
"๎€€แŸ…": 180,
"๎€€แŸ†": 181,
"๎€€แŸ‡": 182,
"๎€€แŸˆ": 183,
"๎€€แŸ‰": 184,
"๎€€แŸŠ": 185,
"๎€€แŸ‹": 186,
"๎€€แŸŒ": 187,
"๎€€แŸ": 188,
"๎€€แŸ": 189,
"๎€€แŸ": 190,
"๎€€แŸ’": 191,
"๎€€แŸ”": 192,
"๎€€แŸ•": 193,
"๎€€แŸ–": 194,
"๎€€แŸ—": 195,
"๎€€แŸ˜": 196,
"๎€€แŸ›": 197,
"๎€€โ€‘": 198,
"๎€€โ€“": 199,
"๎€€โ€”": 200,
"๎€€โ€˜": 201,
"๎€€โ€™": 202,
"๎€€โ€œ": 203,
"๎€€โ€": 204,
"๎€€โ€ข": 205,
"๎€€โ€ฆ": 206,
"๎€€โ€น": 207,
"๎€€โ€บ": 208,
"๎€€โ†’": 209,
"๎€€โˆ’": 210,
"๎€€โ—": 211,
"๎€€๏ฟฝ": 212,
"๎€€๐Ÿ˜€": 213,
"๎€€๐Ÿ˜›": 214,
"๎€€๐Ÿ™‚": 215,
"<src_pad>": 216,
"<src_s>": 217,
"</src>": 218,
"<src_unk>": 219,
" ": 220,
"!": 221,
"\"": 222,
"#": 223,
"$": 224,
"%": 225,
"&": 226,
"'": 227,
"(": 228,
")": 229,
"*": 230,
"+": 231,
",": 232,
"-": 233,
".": 234,
"/": 235,
"0": 236,
"1": 237,
"2": 238,
"3": 239,
"4": 240,
"5": 241,
"6": 242,
"7": 243,
"8": 244,
"9": 245,
":": 246,
";": 247,
"<": 248,
"=": 249,
">": 250,
"?": 251,
"@": 252,
"A": 253,
"B": 254,
"C": 255,
"D": 256,
"E": 257,
"F": 258,
"G": 259,
"H": 260,
"I": 261,
"J": 262,
"K": 263,
"L": 264,
"M": 265,
"N": 266,
"O": 267,
"P": 268,
"Q": 269,
"R": 270,
"S": 271,
"T": 272,
"U": 273,
"V": 274,
"W": 275,
"X": 276,
"Y": 277,
"Z": 278,
"[": 279,
"]": 280,
"^": 281,
"_": 282,
"a": 283,
"b": 284,
"c": 285,
"d": 286,
"e": 287,
"f": 288,
"g": 289,
"h": 290,
"i": 291,
"j": 292,
"k": 293,
"l": 294,
"m": 295,
"n": 296,
"o": 297,
"p": 298,
"q": 299,
"r": 300,
"s": 301,
"t": 302,
"u": 303,
"v": 304,
"w": 305,
"x": 306,
"y": 307,
"z": 308,
"|": 309,
"~": 310,
"ยก": 311,
"ยซ": 312,
"ยฎ": 313,
"ยฐ": 314,
"ยท": 315,
"ยป": 316,
"ร—": 317,
"ร ": 318,
"รก": 319,
"รง": 320,
"รจ": 321,
"รฉ": 322,
"รญ": 323,
"ว”": 324,
"เธ": 325,
"เธ—": 326,
"เธ™": 327,
"เธ ": 328,
"เธข": 329,
"เธฃ": 330,
"เธฉ": 331,
"เธฒ": 332,
"เธด": 333,
"เน€": 334,
"เน„": 335,
"เน’": 336,
"เน•": 337,
"เบฅ": 338,
"เบง": 339,
"เบช": 340,
"เบฒ": 341,
"แž€": 342,
"แž": 343,
"แž‚": 344,
"แžƒ": 345,
"แž„": 346,
"แž…": 347,
"แž†": 348,
"แž‡": 349,
"แžˆ": 350,
"แž‰": 351,
"แžŠ": 352,
"แž‹": 353,
"แžŒ": 354,
"แž": 355,
"แžŽ": 356,
"แž": 357,
"แž": 358,
"แž‘": 359,
"แž’": 360,
"แž“": 361,
"แž”": 362,
"แž•": 363,
"แž–": 364,
"แž—": 365,
"แž˜": 366,
"แž™": 367,
"แžš": 368,
"แž›": 369,
"แžœ": 370,
"แž": 371,
"แžž": 372,
"แžŸ": 373,
"แž ": 374,
"แžก": 375,
"แžข": 376,
"แžฅ": 377,
"แžฆ": 378,
"แžง": 379,
"แžฉ": 380,
"แžช": 381,
"แžซ": 382,
"แžฌ": 383,
"แžญ": 384,
"แžฎ": 385,
"แžฏ": 386,
"แžฐ": 387,
"แžฑ": 388,
"แžฒ": 389,
"แžณ": 390,
"แžถ": 391,
"แžท": 392,
"แžธ": 393,
"แžน": 394,
"แžบ": 395,
"แžป": 396,
"แžผ": 397,
"แžฝ": 398,
"แžพ": 399,
"แžฟ": 400,
"แŸ€": 401,
"แŸ": 402,
"แŸ‚": 403,
"แŸƒ": 404,
"แŸ„": 405,
"แŸ…": 406,
"แŸ†": 407,
"แŸ‡": 408,
"แŸˆ": 409,
"แŸ‰": 410,
"แŸŠ": 411,
"แŸ‹": 412,
"แŸŒ": 413,
"แŸ": 414,
"แŸŽ": 415,
"แŸ": 416,
"แŸ": 417,
"แŸ‘": 418,
"แŸ’": 419,
"แŸ”": 420,
"แŸ•": 421,
"แŸ–": 422,
"แŸ—": 423,
"แŸ˜": 424,
"แŸ›": 425,
"โ€‘": 426,
"โ€“": 427,
"โ€”": 428,
"โ€˜": 429,
"โ€™": 430,
"โ€œ": 431,
"โ€": 432,
"โ€ข": 433,
"โ€ฆ": 434,
"โ€น": 435,
"โ€บ": 436,
"โ†’": 437,
"โˆ’": 438,
"โ—": 439,
"๏ฟฝ": 440,
"๐Ÿ˜€": 441,
"๐Ÿ˜›": 442,
"๐Ÿ™‚": 443
},
"unk_token": "<src_unk>"
}
}