ANLP-A1 / tokenizers /byte_bpe_cipher_new_only_train_split.json
BashCache's picture
Upload folder using huggingface_hub
76bf290 verified
Raw History Blame Contribute Delete
12.5 kB
{"vocab_size": 512, "merges": [[[45, 116], 260], [[89, 22], 261], [[40, 114], 262], [[91, 26], 263], [[87, 101], 264], [[91, 20], 265], [[57, 22], 266], [[47, 112], 267], [[22, 72], 268], [[22, 70], 269], [[69, 26], 270], [[67, 116], 271], [[54, 114], 272], [[65, 112], 273], [[69, 20], 274], [[73, 101], 275], [[40, 94], 276], [[74, 98], 277], [[74, 92], 278], [[57, 42], 279], [[60, 60], 280], [[62, 40], 281], [[46, 112], 282], [[44, 116], 283], [[90, 26], 284], [[86, 101], 285], [[90, 20], 286], [[39, 22], 287], [[41, 114], 288], [[71, 22], 289], [[36, 112], 290], [[38, 116], 291], [[87, 70], 292], [[96, 26], 293], [[96, 20], 294], [[51, 114], 295], [[91, 72], 296], [[41, 66], 297], [[87, 72], 298], [[91, 70], 299], [[95, 92], 300], [[95, 98], 301], [[88, 22], 302], [[56, 22], 303], [[268, 94], 304], [[87, 98], 305], [[269, 45], 306], [[49, 66], 307], [[47, 66], 308], [[45, 38], 309], [[57, 68], 310], [[40, 64], 311], [[74, 93], 312], [[87, 55], 313], [[89, 68], 314], [[74, 99], 315], [[87, 92], 316], [[62, 112], 317], [[57, 114], 318], [[74, 26], 319], [[60, 116], 320], [[74, 20], 321], [[96, 91], 322], [[276, 261], 323], [[277, 262], 324], [[278, 263], 325], [[279, 260], 326], [[97, 98], 327], [[97, 70], 328], [[51, 38], 329], [[49, 40], 330], [[43, 38], 331], [[39, 66], 332], [[44, 36], 333], [[280, 265], 334], [[36, 64], 335], [[281, 266], 336], [[304, 264], 337], [[49, 38], 338], [[97, 92], 339], [[306, 267], 340], [[85, 96], 341], [[36, 36], 342], [[22, 91], 343], [[87, 94], 344], [[85, 98], 345], [[53, 96], 346], [[22, 93], 347], [[99, 51], 348], [[97, 72], 349], [[51, 60], 350], [[40, 282], 351], [[68, 93], 352], [[89, 284], 353], [[57, 286], 354], [[50, 64], 355], [[47, 283], 356], [[87, 288], 357], [[57, 116], 358], [[39, 38], 359], [[83, 101], 360], [[59, 112], 361], [[45, 66], 362], [[79, 26], 363], [[95, 70], 364], [[95, 72], 365], [[99, 68], 366], [[79, 20], 367], [[91, 285], 368], [[60, 114], 369], [[22, 99], 370], [[37, 66], 371], [[101, 336], 372], [[96, 85], 373], [[91, 98], 374], [[68, 99], 375], [[91, 92], 376], [[67, 68], 377], [[70, 101], 378], [[51, 66], 379], [[85, 99], 380], [[36, 38], 381], [[68, 87], 382], [[91, 51], 383], [[87, 96], 384], [[68, 89], 385], [[91, 302], 386], [[97, 84], 387], [[69, 72], 388], [[22, 85], 389], [[93, 55], 390], [[67, 40], 391], [[39, 116], 392], [[37, 112], 393], [[45, 303], 394], [[50, 114], 395], [[97, 20], 396], [[47, 38], 397], [[49, 64], 398], [[97, 26], 399], [[91, 55], 400], [[53, 98], 401], [[97, 73], 402], [[97, 71], 403], [[37, 61], 404], [[44, 112], 405], [[53, 68], 406], [[97, 90], 407], [[67, 88], 408], [[46, 116], 409], [[55, 47], 410], [[39, 58], 411], [[43, 114], 412], [[88, 26], 413], [[57, 43], 414], [[49, 39], 415], [[95, 89], 416], [[57, 96], 417], [[37, 38], 418], [[50, 405], 419], [[87, 51], 420], [[85, 93], 421], [[40, 36], 422], [[85, 68], 423], [[64, 45], 424], [[408, 20], 425], [[50, 36], 426], [[94, 93], 427], [[45, 22], 428], [[75, 72], 429], [[62, 41], 430], [[69, 93], 431], [[37, 409], 432], [[60, 67], 433], [[51, 36], 434], [[51, 67], 435], [[87, 26], 436], [[87, 20], 437], [[316, 288], 438], [[310, 20], 439], [[66, 57], 440], [[49, 116], 441], [[307, 286], 442], [[89, 20], 443], [[51, 112], 444], [[342, 283], 445], [[36, 114], 446], [[47, 116], 447], [[39, 41], 448], [[89, 26], 449], [[305, 284], 450], [[42, 114], 451], [[89, 96], 452], [[61, 96], 453], [[42, 260], 454], [[45, 112], 455], [[311, 112], 456], [[341, 285], 457], [[37, 291], 458], [[77, 22], 459], [[98, 93], 460], [[69, 99], 461], [[308, 116], 462], [[61, 322], 463], [[49, 114], 464], [[314, 26], 465], [[38, 91], 466], [[331, 59], 467], [[296, 101], 468], [[313, 114], 469], [[98, 99], 470], [[50, 290], 471], [[67, 294], 472], [[45, 114], 473], [[94, 99], 474], [[95, 87], 475], [[44, 290], 476], [[50, 63], 477], [[22, 73], 478], [[43, 291], 479], [[49, 43], 480], [[91, 94], 481], [[297, 443], 482], [[69, 70], 483], [[373, 101], 484], [[300, 451], 485], [[22, 71], 486], [[301, 449], 487], [[95, 99], 488], [[333, 447], 489], [[22, 88], 490], [[61, 294], 491], [[85, 72], 492], [[346, 302], 493], [[75, 70], 494], [[329, 303], 495], [[97, 95], 496], [[45, 43], 497], [[339, 101], 498], [[89, 98], 499], [[25, 454], 500], [[65, 60], 501], [[22, 86], 502], [[57, 393], 503], [[62, 392], 504], [[74, 72], 505], [[41, 40], 506], [[40, 61], 507], [[40, 396], 508], [[95, 93], 509], [[22, 69], 510], [[22, 89], 511]], "byte_vocab": {"4": [0], "5": [1], "6": [2], "7": [3], "8": [4], "9": [5], "10": [6], "11": [7], "12": [8], "13": [9], "14": [10], "15": [11], "16": [12], "17": [13], "18": [14], "19": [15], "20": [16], "21": [17], "22": [18], "23": [19], "24": [20], "25": [21], "26": [22], "27": [23], "28": [24], "29": [25], "30": [26], "31": [27], "32": [28], "33": [29], "34": [30], "35": [31], "36": [32], "37": [33], "38": [34], "39": [35], "40": [36], "41": [37], "42": [38], "43": [39], "44": [40], "45": [41], "46": [42], "47": [43], "48": [44], "49": [45], "50": [46], "51": [47], "52": [48], "53": [49], "54": [50], "55": [51], "56": [52], "57": [53], "58": [54], "59": [55], "60": [56], "61": [57], "62": [58], "63": [59], "64": [60], "65": [61], "66": [62], "67": [63], "68": [64], "69": [65], "70": [66], "71": [67], "72": [68], "73": [69], "74": [70], "75": [71], "76": [72], "77": [73], "78": [74], "79": [75], "80": [76], "81": [77], "82": [78], "83": [79], "84": [80], "85": [81], "86": [82], "87": [83], "88": [84], "89": [85], "90": [86], "91": [87], "92": [88], "93": [89], "94": [90], "95": [91], "96": [92], "97": [93], "98": [94], "99": [95], "100": [96], "101": [97], "102": [98], "103": [99], "104": [100], "105": [101], "106": [102], "107": [103], "108": [104], "109": [105], "110": [106], "111": [107], "112": [108], "113": [109], "114": [110], "115": [111], "116": [112], "117": [113], "118": [114], "119": [115], "120": [116], "121": [117], "122": [118], "123": [119], "124": [120], "125": [121], "126": [122], "127": [123], "128": [124], "129": [125], "130": [126], "131": [127], "132": [128], "133": [129], "134": [130], "135": [131], "136": [132], "137": [133], "138": [134], "139": [135], "140": [136], "141": [137], "142": [138], "143": [139], "144": [140], "145": [141], "146": [142], "147": [143], "148": [144], "149": [145], "150": [146], "151": [147], "152": [148], "153": [149], "154": [150], "155": [151], "156": [152], "157": [153], "158": [154], "159": [155], "160": [156], "161": [157], "162": [158], "163": [159], "164": [160], "165": [161], "166": [162], "167": [163], "168": [164], "169": [165], "170": [166], "171": [167], "172": [168], "173": [169], "174": [170], "175": [171], "176": [172], "177": [173], "178": [174], "179": [175], "180": [176], "181": [177], "182": [178], "183": [179], "184": [180], "185": [181], "186": [182], "187": [183], "188": [184], "189": [185], "190": [186], "191": [187], "192": [188], "193": [189], "194": [190], "195": [191], "196": [192], "197": [193], "198": [194], "199": [195], "200": [196], "201": [197], "202": [198], "203": [199], "204": [200], "205": [201], "206": [202], "207": [203], "208": [204], "209": [205], "210": [206], "211": [207], "212": [208], "213": [209], "214": [210], "215": [211], "216": [212], "217": [213], "218": [214], "219": [215], "220": [216], "221": [217], "222": [218], "223": [219], "224": [220], "225": [221], "226": [222], "227": [223], "228": [224], "229": [225], "230": [226], "231": [227], "232": [228], "233": [229], "234": [230], "235": [231], "236": [232], "237": [233], "238": [234], "239": [235], "240": [236], "241": [237], "242": [238], "243": [239], "244": [240], "245": [241], "246": [242], "247": [243], "248": [244], "249": [245], "250": [246], "251": [247], "252": [248], "253": [249], "254": [250], "255": [251], "256": [252], "257": [253], "258": [254], "259": [255], "260": [41, 112], "261": [85, 18], "262": [36, 110], "263": [87, 22], "264": [83, 97], "265": [87, 16], "266": [53, 18], "267": [43, 108], "268": [18, 68], "269": [18, 66], "270": [65, 22], "271": [63, 112], "272": [50, 110], "273": [61, 108], "274": [65, 16], "275": [69, 97], "276": [36, 90], "277": [70, 94], "278": [70, 88], "279": [53, 38], "280": [56, 56], "281": [58, 36], "282": [42, 108], "283": [40, 112], "284": [86, 22], "285": [82, 97], "286": [86, 16], "287": [35, 18], "288": [37, 110], "289": [67, 18], "290": [32, 108], "291": [34, 112], "292": [83, 66], "293": [92, 22], "294": [92, 16], "295": [47, 110], "296": [87, 68], "297": [37, 62], "298": [83, 68], "299": [87, 66], "300": [91, 88], "301": [91, 94], "302": [84, 18], "303": [52, 18], "304": [18, 68, 90], "305": [83, 94], "306": [18, 66, 41], "307": [45, 62], "308": [43, 62], "309": [41, 34], "310": [53, 64], "311": [36, 60], "312": [70, 89], "313": [83, 51], "314": [85, 64], "315": [70, 95], "316": [83, 88], "317": [58, 108], "318": [53, 110], "319": [70, 22], "320": [56, 112], "321": [70, 16], "322": [92, 87], "323": [36, 90, 85, 18], "324": [70, 94, 36, 110], "325": [70, 88, 87, 22], "326": [53, 38, 41, 112], "327": [93, 94], "328": [93, 66], "329": [47, 34], "330": [45, 36], "331": [39, 34], "332": [35, 62], "333": [40, 32], "334": [56, 56, 87, 16], "335": [32, 60], "336": [58, 36, 53, 18], "337": [18, 68, 90, 83, 97], "338": [45, 34], "339": [93, 88], "340": [18, 66, 41, 43, 108], "341": [81, 92], "342": [32, 32], "343": [18, 87], "344": [83, 90], "345": [81, 94], "346": [49, 92], "347": [18, 89], "348": [95, 47], "349": [93, 68], "350": [47, 56], "351": [36, 42, 108], "352": [64, 89], "353": [85, 86, 22], "354": [53, 86, 16], "355": [46, 60], "356": [43, 40, 112], "357": [83, 37, 110], "358": [53, 112], "359": [35, 34], "360": [79, 97], "361": [55, 108], "362": [41, 62], "363": [75, 22], "364": [91, 66], "365": [91, 68], "366": [95, 64], "367": [75, 16], "368": [87, 82, 97], "369": [56, 110], "370": [18, 95], "371": [33, 62], "372": [97, 58, 36, 53, 18], "373": [92, 81], "374": [87, 94], "375": [64, 95], "376": [87, 88], "377": [63, 64], "378": [66, 97], "379": [47, 62], "380": [81, 95], "381": [32, 34], "382": [64, 83], "383": [87, 47], "384": [83, 92], "385": [64, 85], "386": [87, 84, 18], "387": [93, 80], "388": [65, 68], "389": [18, 81], "390": [89, 51], "391": [63, 36], "392": [35, 112], "393": [33, 108], "394": [41, 52, 18], "395": [46, 110], "396": [93, 16], "397": [43, 34], "398": [45, 60], "399": [93, 22], "400": [87, 51], "401": [49, 94], "402": [93, 69], "403": [93, 67], "404": [33, 57], "405": [40, 108], "406": [49, 64], "407": [93, 86], "408": [63, 84], "409": [42, 112], "410": [51, 43], "411": [35, 54], "412": [39, 110], "413": [84, 22], "414": [53, 39], "415": [45, 35], "416": [91, 85], "417": [53, 92], "418": [33, 34], "419": [46, 40, 108], "420": [83, 47], "421": [81, 89], "422": [36, 32], "423": [81, 64], "424": [60, 41], "425": [63, 84, 16], "426": [46, 32], "427": [90, 89], "428": [41, 18], "429": [71, 68], "430": [58, 37], "431": [65, 89], "432": [33, 42, 112], "433": [56, 63], "434": [47, 32], "435": [47, 63], "436": [83, 22], "437": [83, 16], "438": [83, 88, 37, 110], "439": [53, 64, 16], "440": [62, 53], "441": [45, 112], "442": [45, 62, 86, 16], "443": [85, 16], "444": [47, 108], "445": [32, 32, 40, 112], "446": [32, 110], "447": [43, 112], "448": [35, 37], "449": [85, 22], "450": [83, 94, 86, 22], "451": [38, 110], "452": [85, 92], "453": [57, 92], "454": [38, 41, 112], "455": [41, 108], "456": [36, 60, 108], "457": [81, 92, 82, 97], "458": [33, 34, 112], "459": [73, 18], "460": [94, 89], "461": [65, 95], "462": [43, 62, 112], "463": [57, 92, 87], "464": [45, 110], "465": [85, 64, 22], "466": [34, 87], "467": [39, 34, 55], "468": [87, 68, 97], "469": [83, 51, 110], "470": [94, 95], "471": [46, 32, 108], "472": [63, 92, 16], "473": [41, 110], "474": [90, 95], "475": [91, 83], "476": [40, 32, 108], "477": [46, 59], "478": [18, 69], "479": [39, 34, 112], "480": [45, 39], "481": [87, 90], "482": [37, 62, 85, 16], "483": [65, 66], "484": [92, 81, 97], "485": [91, 88, 38, 110], "486": [18, 67], "487": [91, 94, 85, 22], "488": [91, 95], "489": [40, 32, 43, 112], "490": [18, 84], "491": [57, 92, 16], "492": [81, 68], "493": [49, 92, 84, 18], "494": [71, 66], "495": [47, 34, 52, 18], "496": [93, 91], "497": [41, 39], "498": [93, 88, 97], "499": [85, 94], "500": [21, 38, 41, 112], "501": [61, 56], "502": [18, 82], "503": [53, 33, 108], "504": [58, 35, 112], "505": [70, 68], "506": [37, 36], "507": [36, 57], "508": [36, 93, 16], "509": [91, 89], "510": [18, 65], "511": [18, 85]}}