Intelligence_OS_1p5B / tokenizer.json
shreyansh-programmer's picture
Upload tokenizer.json with huggingface_hub
380889d verified
Raw History Blame Contribute Delete
27.1 kB
{ "target_vocab_size": 512, "special_tokens": [ "<pad>", "<s>", "</s>", "<unk>" ], "merges": [ { "p0": 105, "p1": 118, "id": 260 }, { "p0": 119, "p1": 36, "id": 261 }, { "p0": 105, "p1": 36, "id": 262 }, { "p0": 109, "p1": 114, "id": 263 }, { "p0": 101, "p1": 120, "id": 264 }, { "p0": 105, "p1": 114, "id": 265 }, { "p0": 101, "p1": 114, "id": 266 }, { "p0": 115, "p1": 114, "id": 267 }, { "p0": 101, "p1": 112, "id": 268 }, { "p0": 62, "p1": 36, "id": 269 }, { "p0": 125, "p1": 36, "id": 270 }, { "p0": 104, "p1": 36, "id": 271 }, { "p0": 101, "p1": 118, "id": 272 }, { "p0": 115, "p1": 118, "id": 273 }, { "p0": 109, "p1": 267, "id": 274 }, { "p0": 118, "p1": 115, "id": 275 }, { "p0": 105, "p1": 112, "id": 276 }, { "p0": 263, "p1": 107, "id": 277 }, { "p0": 50, "p1": 14, "id": 278 }, { "p0": 109, "p1": 103, "id": 279 }, { "p0": 120, "p1": 108, "id": 280 }, { "p0": 278, "p1": 14, "id": 281 }, { "p0": 265, "p1": 103, "id": 282 }, { "p0": 69, "p1": 77, "id": 283 }, { "p0": 36, "p1": 119, "id": 284 }, { "p0": 266, "p1": 271, "id": 285 }, { "p0": 48, "p1": 36, "id": 286 }, { "p0": 264, "p1": 274, "id": 287 }, { "p0": 121, "p1": 120, "id": 288 }, { "p0": 103, "p1": 115, "id": 289 }, { "p0": 101, "p1": 103, "id": 290 }, { "p0": 89, "p1": 119, "id": 291 }, { "p0": 109, "p1": 261, "id": 292 }, { "p0": 109, "p1": 107, "id": 293 }, { "p0": 113, "p1": 116, "id": 294 }, { "p0": 291, "p1": 260, "id": 295 }, { "p0": 295, "p1": 269, "id": 296 }, { "p0": 283, "p1": 269, "id": 297 }, { "p0": 119, "p1": 120, "id": 298 }, { "p0": 281, "p1": 296, "id": 299 }, { "p0": 36, "p1": 116, "id": 300 }, { "p0": 112, "p1": 105, "id": 301 }, { "p0": 265, "p1": 120, "id": 302 }, { "p0": 120, "p1": 109, "id": 303 }, { "p0": 106, "p1": 273, "id": 304 }, { "p0": 67, "p1": 14, "id": 305 }, { "p0": 305, "p1": 297, "id": 306 }, { "p0": 276, "p1": 112, "id": 307 }, { "p0": 264, "p1": 36, "id": 308 }, { "p0": 121, "p1": 118, "id": 309 }, { "p0": 277, "p1": 36, "id": 310 }, { "p0": 289, "p1": 294, "id": 311 }, { "p0": 105, "p1": 104, "id": 312 }, { "p0": 105, "p1": 261, "id": 313 }, { "p0": 117, "p1": 121, "id": 314 }, { "p0": 279, "p1": 109, "id": 315 }, { "p0": 120, "p1": 115, "id": 316 }, { "p0": 268, "p1": 36, "id": 317 }, { "p0": 105, "p1": 113, "id": 318 }, { "p0": 91, "p1": 108, "id": 319 }, { "p0": 106, "p1": 315, "id": 320 }, { "p0": 116, "p1": 260, "id": 321 }, { "p0": 115, "p1": 106, "id": 322 }, { "p0": 311, "p1": 288, "id": 323 }, { "p0": 304, "p1": 113, "id": 324 }, { "p0": 36, "p1": 113, "id": 325 }, { "p0": 121, "p1": 112, "id": 326 }, { "p0": 299, "p1": 319, "id": 327 }, { "p0": 327, "p1": 308, "id": 328 }, { "p0": 328, "p1": 292, "id": 329 }, { "p0": 120, "p1": 307, "id": 330 }, { "p0": 330, "p1": 293, "id": 331 }, { "p0": 331, "p1": 282, "id": 332 }, { "p0": 36, "p1": 104, "id": 333 }, { "p0": 109, "p1": 280, "id": 334 }, { "p0": 264, "p1": 120, "id": 335 }, { "p0": 50, "p1": 36, "id": 336 }, { "p0": 119, "p1": 262, "id": 337 }, { "p0": 118, "p1": 105, "id": 338 }, { "p0": 113, "p1": 109, "id": 339 }, { "p0": 122, "p1": 262, "id": 340 }, { "p0": 113, "p1": 36, "id": 341 }, { "p0": 120, "p1": 118, "id": 342 }, { "p0": 262, "p1": 322, "id": 343 }, { "p0": 105, "p1": 119, "id": 344 }, { "p0": 301, "p1": 272, "id": 345 }, { "p0": 345, "p1": 114, "id": 346 }, { "p0": 105, "p1": 103, "id": 347 }, { "p0": 275, "p1": 121, "id": 348 }, { "p0": 101, "p1": 107, "id": 349 }, { "p0": 312, "p1": 49, "id": 350 }, { "p0": 105, "p1": 124, "id": 351 }, { "p0": 260, "p1": 107, "id": 352 }, { "p0": 112, "p1": 115, "id": 353 }, { "p0": 317, "p1": 263, "id": 354 }, { "p0": 101, "p1": 119, "id": 355 }, { "p0": 332, "p1": 105, "id": 356 }, { "p0": 266, "p1": 36, "id": 357 }, { "p0": 112, "p1": 266, "id": 358 }, { "p0": 358, "p1": 107, "id": 359 }, { "p0": 359, "p1": 121, "id": 360 }, { "p0": 360, "p1": 349, "id": 361 }, { "p0": 361, "p1": 262, "id": 362 }, { "p0": 119, "p1": 105, "id": 363 }, { "p0": 264, "p1": 262, "id": 364 }, { "p0": 105, "p1": 106, "id": 365 }, { "p0": 365, "p1": 320, "id": 366 }, { "p0": 286, "p1": 285, "id": 367 }, { "p0": 260, "p1": 261, "id": 368 }, { "p0": 335, "p1": 302, "id": 369 }, { "p0": 369, "p1": 274, "id": 370 }, { "p0": 314, "p1": 260, "id": 371 }, { "p0": 371, "p1": 270, "id": 372 }, { "p0": 270, "p1": 285, "id": 373 }, { "p0": 120, "p1": 309, "id": 374 }, { "p0": 343, "p1": 36, "id": 375 }, { "p0": 351, "p1": 321, "id": 376 }, { "p0": 105, "p1": 271, "id": 377 }, { "p0": 103, "p1": 108, "id": 378 }, { "p0": 116, "p1": 115, "id": 379 }, { "p0": 323, "p1": 287, "id": 380 }, { "p0": 103, "p1": 268, "id": 381 }, { "p0": 381, "p1": 103, "id": 382 }, { "p0": 382, "p1": 326, "id": 383 }, { "p0": 101, "p1": 104, "id": 384 }, { "p0": 265, "p1": 352, "id": 385 }, { "p0": 115, "p1": 104, "id": 386 }, { "p0": 121, "p1": 114, "id": 387 }, { "p0": 103, "p1": 109, "id": 388 }, { "p0": 122, "p1": 260, "id": 389 }, { "p0": 339, "p1": 126, "id": 390 }, { "p0": 36, "p1": 272, "id": 391 }, { "p0": 306, "p1": 81, "id": 392 }, { "p0": 303, "p1": 320, "id": 393 }, { "p0": 393, "p1": 354, "id": 394 }, { "p0": 115, "p1": 288, "id": 395 }, { "p0": 395, "p1": 36, "id": 396 }, { "p0": 261, "p1": 280, "id": 397 }, { "p0": 300, "p1": 260, "id": 398 }, { "p0": 300, "p1": 275, "id": 399 } ], "vocab": { "0": [ 60, 112, 97, 100, 62 ], "1": [ 60, 115, 62 ], "2": [ 60, 47, 115, 62 ], "3": [ 60, 117, 110, 107, 62 ], "4": [ 0 ], "5": [ 1 ], "6": [ 2 ], "7": [ 3 ], "8": [ 4 ], "9": [ 5 ], "10": [ 6 ], "11": [ 7 ], "12": [ 8 ], "13": [ 9 ], "14": [ 10 ], "15": [ 11 ], "16": [ 12 ], "17": [ 13 ], "18": [ 14 ], "19": [ 15 ], "20": [ 16 ], "21": [ 17 ], "22": [ 18 ], "23": [ 19 ], "24": [ 20 ], "25": [ 21 ], "26": [ 22 ], "27": [ 23 ], "28": [ 24 ], "29": [ 25 ], "30": [ 26 ], "31": [ 27 ], "32": [ 28 ], "33": [ 29 ], "34": [ 30 ], "35": [ 31 ], "36": [ 32 ], "37": [ 33 ], "38": [ 34 ], "39": [ 35 ], "40": [ 36 ], "41": [ 37 ], "42": [ 38 ], "43": [ 39 ], "44": [ 40 ], "45": [ 41 ], "46": [ 42 ], "47": [ 43 ], "48": [ 44 ], "49": [ 45 ], "50": [ 46 ], "51": [ 47 ], "52": [ 48 ], "53": [ 49 ], "54": [ 50 ], "55": [ 51 ], "56": [ 52 ], "57": [ 53 ], "58": [ 54 ], "59": [ 55 ], "60": [ 56 ], "61": [ 57 ], "62": [ 58 ], "63": [ 59 ], "64": [ 60 ], "65": [ 61 ], "66": [ 62 ], "67": [ 63 ], "68": [ 64 ], "69": [ 65 ], "70": [ 66 ], "71": [ 67 ], "72": [ 68 ], "73": [ 69 ], "74": [ 70 ], "75": [ 71 ], "76": [ 72 ], "77": [ 73 ], "78": [ 74 ], "79": [ 75 ], "80": [ 76 ], "81": [ 77 ], "82": [ 78 ], "83": [ 79 ], "84": [ 80 ], "85": [ 81 ], "86": [ 82 ], "87": [ 83 ], "88": [ 84 ], "89": [ 85 ], "90": [ 86 ], "91": [ 87 ], "92": [ 88 ], "93": [ 89 ], "94": [ 90 ], "95": [ 91 ], "96": [ 92 ], "97": [ 93 ], "98": [ 94 ], "99": [ 95 ], "100": [ 96 ], "101": [ 97 ], "102": [ 98 ], "103": [ 99 ], "104": [ 100 ], "105": [ 101 ], "106": [ 102 ], "107": [ 103 ], "108": [ 104 ], "109": [ 105 ], "110": [ 106 ], "111": [ 107 ], "112": [ 108 ], "113": [ 109 ], "114": [ 110 ], "115": [ 111 ], "116": [ 112 ], "117": [ 113 ], "118": [ 114 ], "119": [ 115 ], "120": [ 116 ], "121": [ 117 ], "122": [ 118 ], "123": [ 119 ], "124": [ 120 ], "125": [ 121 ], "126": [ 122 ], "127": [ 123 ], "128": [ 124 ], "129": [ 125 ], "130": [ 126 ], "131": [ 127 ], "132": [ 128 ], "133": [ 129 ], "134": [ 130 ], "135": [ 131 ], "136": [ 132 ], "137": [ 133 ], "138": [ 134 ], "139": [ 135 ], "140": [ 136 ], "141": [ 137 ], "142": [ 138 ], "143": [ 139 ], "144": [ 140 ], "145": [ 141 ], "146": [ 142 ], "147": [ 143 ], "148": [ 144 ], "149": [ 145 ], "150": [ 146 ], "151": [ 147 ], "152": [ 148 ], "153": [ 149 ], "154": [ 150 ], "155": [ 151 ], "156": [ 152 ], "157": [ 153 ], "158": [ 154 ], "159": [ 155 ], "160": [ 156 ], "161": [ 157 ], "162": [ 158 ], "163": [ 159 ], "164": [ 160 ], "165": [ 161 ], "166": [ 162 ], "167": [ 163 ], "168": [ 164 ], "169": [ 165 ], "170": [ 166 ], "171": [ 167 ], "172": [ 168 ], "173": [ 169 ], "174": [ 170 ], "175": [ 171 ], "176": [ 172 ], "177": [ 173 ], "178": [ 174 ], "179": [ 175 ], "180": [ 176 ], "181": [ 177 ], "182": [ 178 ], "183": [ 179 ], "184": [ 180 ], "185": [ 181 ], "186": [ 182 ], "187": [ 183 ], "188": [ 184 ], "189": [ 185 ], "190": [ 186 ], "191": [ 187 ], "192": [ 188 ], "193": [ 189 ], "194": [ 190 ], "195": [ 191 ], "196": [ 192 ], "197": [ 193 ], "198": [ 194 ], "199": [ 195 ], "200": [ 196 ], "201": [ 197 ], "202": [ 198 ], "203": [ 199 ], "204": [ 200 ], "205": [ 201 ], "206": [ 202 ], "207": [ 203 ], "208": [ 204 ], "209": [ 205 ], "210": [ 206 ], "211": [ 207 ], "212": [ 208 ], "213": [ 209 ], "214": [ 210 ], "215": [ 211 ], "216": [ 212 ], "217": [ 213 ], "218": [ 214 ], "219": [ 215 ], "220": [ 216 ], "221": [ 217 ], "222": [ 218 ], "223": [ 219 ], "224": [ 220 ], "225": [ 221 ], "226": [ 222 ], "227": [ 223 ], "228": [ 224 ], "229": [ 225 ], "230": [ 226 ], "231": [ 227 ], "232": [ 228 ], "233": [ 229 ], "234": [ 230 ], "235": [ 231 ], "236": [ 232 ], "237": [ 233 ], "238": [ 234 ], "239": [ 235 ], "240": [ 236 ], "241": [ 237 ], "242": [ 238 ], "243": [ 239 ], "244": [ 240 ], "245": [ 241 ], "246": [ 242 ], "247": [ 243 ], "248": [ 244 ], "249": [ 245 ], "250": [ 246 ], "251": [ 247 ], "252": [ 248 ], "253": [ 249 ], "254": [ 250 ], "255": [ 251 ], "256": [ 252 ], "257": [ 253 ], "258": [ 254 ], "259": [ 255 ], "260": [ 101, 114 ], "261": [ 115, 32 ], "262": [ 101, 32 ], "263": [ 105, 110 ], "264": [ 97, 116 ], "265": [ 101, 110 ], "266": [ 97, 110 ], "267": [ 111, 110 ], "268": [ 97, 108 ], "269": [ 58, 32 ], "270": [ 121, 32 ], "271": [ 100, 32 ], "272": [ 97, 114 ], "273": [ 111, 114 ], "274": [ 105, 111, 110 ], "275": [ 114, 111 ], "276": [ 101, 108 ], "277": [ 105, 110, 103 ], "278": [ 46, 10 ], "279": [ 105, 99 ], "280": [ 116, 104 ], "281": [ 46, 10, 10 ], "282": [ 101, 110, 99 ], "283": [ 65, 73 ], "284": [ 32, 115 ], "285": [ 97, 110, 100, 32 ], "286": [ 44, 32 ], "287": [ 97, 116, 105, 111, 110 ], "288": [ 117, 116 ], "289": [ 99, 111 ], "290": [ 97, 99 ], "291": [ 85, 115 ], "292": [ 105, 115, 32 ], "293": [ 105, 103 ], "294": [ 109, 112 ], "295": [ 85, 115, 101, 114 ], "296": [ 85, 115, 101, 114, 58, 32 ], "297": [ 65, 73, 58, 32 ], "298": [ 115, 116 ], "299": [ 46, 10, 10, 85, 115, 101, 114, 58, 32 ], "300": [ 32, 112 ], "301": [ 108, 101 ], "302": [ 101, 110, 116 ], "303": [ 116, 105 ], "304": [ 102, 111, 114 ], "305": [ 63, 10 ], "306": [ 63, 10, 65, 73, 58, 32 ], "307": [ 101, 108, 108 ], "308": [ 97, 116, 32 ], "309": [ 117, 114 ], "310": [ 105, 110, 103, 32 ], "311": [ 99, 111, 109, 112 ], "312": [ 101, 100 ], "313": [ 101, 115, 32 ], "314": [ 113, 117 ], "315": [ 105, 99, 105 ], "316": [ 116, 111 ], "317": [ 97, 108, 32 ], "318": [ 101, 109 ], "319": [ 87, 104 ], "320": [ 102, 105, 99, 105 ], "321": [ 112, 101, 114 ], "322": [ 111, 102 ], "323": [ 99, 111, 109, 112, 117, 116 ], "324": [ 102, 111, 114, 109 ], "325": [ 32, 109 ], "326": [ 117, 108 ], "327": [ 46, 10, 10, 85, 115, 101, 114, 58, 32, 87, 104 ], "328": [ 46, 10, 10, 85, 115, 101, 114, 58, 32, 87, 104, 97, 116, 32 ], "329": [ 46, 10, 10, 85, 115, 101, 114, 58, 32, 87, 104, 97, 116, 32, 105, 115, 32 ], "330": [ 116, 101, 108, 108 ], "331": [ 116, 101, 108, 108, 105, 103 ], "332": [ 116, 101, 108, 108, 105, 103, 101, 110, 99 ], "333": [ 32, 100 ], "334": [ 105, 116, 104 ], "335": [ 97, 116, 116 ], "336": [ 46, 32 ], "337": [ 115, 101, 32 ], "338": [ 114, 101 ], "339": [ 109, 105 ], "340": [ 118, 101, 32 ], "341": [ 109, 32 ], "342": [ 116, 114 ], "343": [ 101, 32, 111, 102 ], "344": [ 101, 115 ], "345": [ 108, 101, 97, 114 ], "346": [ 108, 101, 97, 114, 110 ], "347": [ 101, 99 ], "348": [ 114, 111, 117 ], "349": [ 97, 103 ], "350": [ 101, 100, 45 ], "351": [ 101, 120 ], "352": [ 101, 114, 103 ], "353": [ 108, 111 ], "354": [ 97, 108, 32, 105, 110 ], "355": [ 97, 115 ], "356": [ 116, 101, 108, 108, 105, 103, 101, 110, 99, 101 ], "357": [ 97, 110, 32 ], "358": [ 108, 97, 110 ], "359": [ 108, 97, 110, 103 ], "360": [ 108, 97, 110, 103, 117 ], "361": [ 108, 97, 110, 103, 117, 97, 103 ], "362": [ 108, 97, 110, 103, 117, 97, 103, 101, 32 ], "363": [ 115, 101 ], "364": [ 97, 116, 101, 32 ], "365": [ 101, 102 ], "366": [ 101, 102, 102, 105, 99, 105 ], "367": [ 44, 32, 97, 110, 100, 32 ], "368": [ 101, 114, 115, 32 ], "369": [ 97, 116, 116, 101, 110, 116 ], "370": [ 97, 116, 116, 101, 110, 116, 105, 111, 110 ], "371": [ 113, 117, 101, 114 ], "372": [ 113, 117, 101, 114, 121, 32 ], "373": [ 121, 32, 97, 110, 100, 32 ], "374": [ 116, 117, 114 ], "375": [ 101, 32, 111, 102, 32 ], "376": [ 101, 120, 112, 101, 114 ], "377": [ 101, 100, 32 ], "378": [ 99, 104 ], "379": [ 112, 111 ], "380": [ 99, 111, 109, 112, 117, 116, 97, 116, 105, 111, 110 ], "381": [ 99, 97, 108 ], "382": [ 99, 97, 108, 99 ], "383": [ 99, 97, 108, 99, 117, 108 ], "384": [ 97, 100 ], "385": [ 101, 110, 101, 114, 103 ], "386": [ 111, 100 ], "387": [ 117, 110 ], "388": [ 99, 105 ], "389": [ 118, 101, 114 ], "390": [ 109, 105, 122 ], "391": [ 32, 97, 114 ], "392": [ 63, 10, 65, 73, 58, 32, 77 ], "393": [ 116, 105, 102, 105, 99, 105 ], "394": [ 116, 105, 102, 105, 99, 105, 97, 108, 32, 105, 110 ], "395": [ 111, 117, 116 ], "396": [ 111, 117, 116, 32 ], "397": [ 115, 32, 116, 104 ], "398": [ 32, 112, 101, 114 ], "399": [ 32, 112, 114, 111 ] } }
{
"target_vocab_size": 512,
"special_tokens": [
"<pad>",
"<s>",
"</s>",
"<unk>"
],
"merges": [
{
"p0": 105,
"p1": 118,
"id": 260
},
{
"p0": 119,
"p1": 36,
"id": 261
},
{
"p0": 105,
"p1": 36,
"id": 262
},
{
"p0": 109,
"p1": 114,
"id": 263
},
{
"p0": 101,
"p1": 120,
"id": 264
},
{
"p0": 105,
"p1": 114,
"id": 265
},
{
"p0": 101,
"p1": 114,
"id": 266
},
{
"p0": 115,
"p1": 114,
"id": 267
},
{
"p0": 101,
"p1": 112,
"id": 268
},
{
"p0": 62,
"p1": 36,
"id": 269
},
{
"p0": 125,
"p1": 36,
"id": 270
},
{
"p0": 104,
"p1": 36,
"id": 271
},
{
"p0": 101,
"p1": 118,
"id": 272
},
{
"p0": 115,
"p1": 118,
"id": 273
},
{
"p0": 109,
"p1": 267,
"id": 274
},
{
"p0": 118,
"p1": 115,
"id": 275
},
{
"p0": 105,
"p1": 112,
"id": 276
},
{
"p0": 263,
"p1": 107,
"id": 277
},
{