Download plain_tokenizer.json from nuxsh/a1nlp: direct link, hf CLI and curl.
- Browser
- Download file 10.2 kB
-
https://huggingface.co/nuxsh/a1nlp/resolve/main/plain_tokenizer.json
- Command line
-
hf download hf://nuxsh/a1nlp/plain_tokenizer.json
-
curl -L -o plain_tokenizer.json https://huggingface.co/nuxsh/a1nlp/resolve/main/plain_tokenizer.json
10.2 kB
| {"vocab": {"<pad>": 0, "<bos>": 1, "<eos>": 2, "<unk>": 3, "00": 4, "01": 5, "02": 6, "03": 7, "04": 8, "05": 9, "06": 10, "07": 11, "08": 12, "09": 13, "0a": 14, "0b": 15, "0c": 16, "0d": 17, "0e": 18, "0f": 19, "10": 20, "11": 21, "12": 22, "13": 23, "14": 24, "15": 25, "16": 26, "17": 27, "18": 28, "19": 29, "1a": 30, "1b": 31, "1c": 32, "1d": 33, "1e": 34, "1f": 35, "20": 36, "21": 37, "22": 38, "23": 39, "24": 40, "25": 41, "26": 42, "27": 43, "28": 44, "29": 45, "2a": 46, "2b": 47, "2c": 48, "2d": 49, "2e": 50, "2f": 51, "30": 52, "31": 53, "32": 54, "33": 55, "34": 56, "35": 57, "36": 58, "37": 59, "38": 60, "39": 61, "3a": 62, "3b": 63, "3c": 64, "3d": 65, "3e": 66, "3f": 67, "40": 68, "41": 69, "42": 70, "43": 71, "44": 72, "45": 73, "46": 74, "47": 75, "48": 76, "49": 77, "4a": 78, "4b": 79, "4c": 80, "4d": 81, "4e": 82, "4f": 83, "50": 84, "51": 85, "52": 86, "53": 87, "54": 88, "55": 89, "56": 90, "57": 91, "58": 92, "59": 93, "5a": 94, "5b": 95, "5c": 96, "5d": 97, "5e": 98, "5f": 99, "60": 100, "61": 101, "62": 102, "63": 103, "64": 104, "65": 105, "66": 106, "67": 107, "68": 108, "69": 109, "6a": 110, "6b": 111, "6c": 112, "6d": 113, "6e": 114, "6f": 115, "70": 116, "71": 117, "72": 118, "73": 119, "74": 120, "75": 121, "76": 122, "77": 123, "78": 124, "79": 125, "7a": 126, "7b": 127, "7c": 128, "7d": 129, "7e": 130, "7f": 131, "80": 132, "81": 133, "82": 134, "83": 135, "84": 136, "85": 137, "86": 138, "87": 139, "88": 140, "89": 141, "8a": 142, "8b": 143, "8c": 144, "8d": 145, "8e": 146, "8f": 147, "90": 148, "91": 149, "92": 150, "93": 151, "94": 152, "95": 153, "96": 154, "97": 155, "98": 156, "99": 157, "9a": 158, "9b": 159, "9c": 160, "9d": 161, "9e": 162, "9f": 163, "a0": 164, "a1": 165, "a2": 166, "a3": 167, "a4": 168, "a5": 169, "a6": 170, "a7": 171, "a8": 172, "a9": 173, "aa": 174, "ab": 175, "ac": 176, "ad": 177, "ae": 178, "af": 179, "b0": 180, "b1": 181, "b2": 182, "b3": 183, "b4": 184, "b5": 185, "b6": 186, "b7": 187, "b8": 188, "b9": 189, "ba": 190, "bb": 191, "bc": 192, "bd": 193, "be": 194, "bf": 195, "c0": 196, "c1": 197, "c2": 198, "c3": 199, "c4": 200, "c5": 201, "c6": 202, "c7": 203, "c8": 204, "c9": 205, "ca": 206, "cb": 207, "cc": 208, "cd": 209, "ce": 210, "cf": 211, "d0": 212, "d1": 213, "d2": 214, "d3": 215, "d4": 216, "d5": 217, "d6": 218, "d7": 219, "d8": 220, "d9": 221, "da": 222, "db": 223, "dc": 224, "dd": 225, "de": 226, "df": 227, "e0": 228, "e1": 229, "e2": 230, "e3": 231, "e4": 232, "e5": 233, "e6": 234, "e7": 235, "e8": 236, "e9": 237, "ea": 238, "eb": 239, "ec": 240, "ed": 241, "ee": 242, "ef": 243, "f0": 244, "f1": 245, "f2": 246, "f3": 247, "f4": 248, "f5": 249, "f6": 250, "f7": 251, "f8": 252, "f9": 253, "fa": 254, "fb": 255, "fc": 256, "fd": 257, "fe": 258, "ff": 259, "6520": 260, "7320": 261, "7468": 262, "6420": 263, "6e20": 264, "6572": 265, "7420": 266, "74686520": 267, "696e": 268, "616e": 269, "656420": 270, "7920": 271, "6f72": 272, "6172": 273, "616c": 274, "6f20": 275, "6620": 276, "7265": 277, "6f6e": 278, "7469": 279, "6f6620": 280, "656e": 281, "6120": 282, "6720": 283, "616e6420": 284, "657220": 285, "6f6e20": 286, "696e20": 287, "696e6720": 288, "6f75": 289, "746f20": 290, "7374": 291, "617320": 292, "6174": 293, "657320": 294, "6869": 295, "726f": 296, "6d20": 297, "7369": 298, "6368": 299, "616e20": 300, "7269": 301, "656c": 302, "686520": 303, "617420": 304, "616c20": 305, "746820": 306, "6c69": 307, "6163": 308, "666f72": 309, "737420": 310, "7572": 311, "77617320": 312, "6f6d": 313, "6563": 314, "696c": 315, "6561": 316, "6469": 317, "616d": 318, "6972": 319, "6f6c": 320, "656e20": 321, "6974": 322, "6f662074686520": 323, "54686520": 324, "6963": 325, "6f77": 326, "6c7920": 327, "617469": 328, "6573": 329, "656d": 330, "6c20": 331, "6b20": 332, "7769": 333, "756e": 334, "6768": 335, "6c6520": 336, "7368": 337, "636820": 338, "697320": 339, "7261": 340, "6574": 341, "7468617420": 342, "65726520": 343, "6173": 344, "6576": 345, "627920": 346, "666f7220": 347, "6861": 348, "636f6e": 349, "656e74": 350, "7769746820": 351, "656e7420": 352, "7375": 353, "6167": 354, "6164": 355, "706c": 356, "696e2074686520": 357, "766520": 358, "6f70": 359, "617220": 360, "6465": 361, "6f7220": 362, "6265": 363, "6f756e": 364, "636520": 365, "65727320": 366, "6f76": 367, "756c": 368, "6170": 369, "6973": 370, "6275": 371, "74656420": 372, "70726f": 373, "68697320": 374, "6174696f6e20": 375, "636f6d": 376, "746865": 377, "7765726520": 378, "706f": 379, "726f6d20": 380, "66726f6d20": 381, "636c": 382, "7669": 383, "6162": 384, "6578": 385, "74696f6e20": 386, "7370": 387, "7768": 388, "617920": 389, "697420": 390, "776869": 391, "6f63": 392, "7573": 393, "6f6e7320": 394, "746f2074686520": 395, "7720": 396, "747320": 397, "696d": 398, "7020": 399, "74657220": 400, "647320": 401, "6174656420": 402, "7175": 403, "7275": 404, "61746520": 405, "706572": 406, "6e6f": 407, "746572": 408, "6f7572": 409, "64656420": 410, "656172": 411, "707265": 412, "6f6f": 413, "696e672074686520": 414, "6179": 415, "61696e": 416, "68616420": 417, "6666": 418, "7468657220": 419, "5374": 420, "697220": 421, "6f756768": 422, "617474": 423, "706172": 424, "616c6c20": 425, "696e67": 426, "62657220": 427, "6c6420": 428, "776869636820": 429, "736520": 430, "756d": 431, "65642074686520": 432, "64696e6720": 433, "6320": 434, "6f6e6520": 435, "616d6520": 436, "726563": 437, "7365": 438, "646520": 439, "6475": 440, "6d6f": 441, "496e20": 442, "616c69": 443, "7320616e6420": 444, "7472": 445, "6f6e6720": 446, "6e65": 447, "6f757420": 448, "61726520": 449, "7765": 450, "706f72": 451, "736f20": 452, "666972": 453, "6d616e": 454, "656420746f20": 455, "6775": 456, "6f6e2074686520": 457, "6c65": 458, "616b": 459, "656420627920": 460, "6669": 461, "747572": 462, "74696d": 463, "736820": 464, "626520": 465, "61727920": 466, "7363": 467, "676520": 468, "64657220": 469, "747920": 470, "4368": 471, "616c6c7920": 472, "6c6f77": 473, "746520": 474, "73696f6e20": 475, "4120": 476, "6976": 477, "646572": 478, "69747920": 479, "69747320": 480, "737472": 481, "6570": 482, "616e642074686520": 483, "6920": 484, "696e6520": 485, "67687420": 486, "776f72": 487, "746865697220": 488, "6369": 489, "6f776e20": 490, "736869": 491, "62757420": 492, "6e6f7420": 493, "616c736f20": 494, "6c6561": 495, "6674657220": 496, "5468": 497, "7477": 498, "6f756e6420": 499}, "merges": [["65", "20"], ["73", "20"], ["74", "68"], ["64", "20"], ["6e", "20"], ["65", "72"], ["74", "20"], ["7468", "6520"], ["69", "6e"], ["61", "6e"], ["65", "6420"], ["79", "20"], ["6f", "72"], ["61", "72"], ["61", "6c"], ["6f", "20"], ["66", "20"], ["72", "65"], ["6f", "6e"], ["74", "69"], ["6f", "6620"], ["65", "6e"], ["61", "20"], ["67", "20"], ["616e", "6420"], ["6572", "20"], ["6f", "6e20"], ["69", "6e20"], ["696e", "6720"], ["6f", "75"], ["74", "6f20"], ["73", "74"], ["61", "7320"], ["61", "74"], ["65", "7320"], ["68", "69"], ["72", "6f"], ["6d", "20"], ["73", "69"], ["63", "68"], ["61", "6e20"], ["72", "69"], ["65", "6c"], ["68", "6520"], ["61", "7420"], ["616c", "20"], ["7468", "20"], ["6c", "69"], ["61", "63"], ["66", "6f72"], ["73", "7420"], ["75", "72"], ["77", "617320"], ["6f", "6d"], ["65", "63"], ["69", "6c"], ["65", "61"], ["64", "69"], ["61", "6d"], ["69", "72"], ["6f", "6c"], ["65", "6e20"], ["69", "74"], ["6f6620", "74686520"], ["54", "686520"], ["69", "63"], ["6f", "77"], ["6c", "7920"], ["61", "7469"], ["65", "73"], ["65", "6d"], ["6c", "20"], ["6b", "20"], ["77", "69"], ["75", "6e"], ["67", "68"], ["6c", "6520"], ["73", "68"], ["6368", "20"], ["69", "7320"], ["72", "61"], ["65", "74"], ["7468", "617420"], ["6572", "6520"], ["61", "73"], ["65", "76"], ["62", "7920"], ["666f72", "20"], ["68", "61"], ["63", "6f6e"], ["656e", "74"], ["7769", "746820"], ["656e", "7420"], ["73", "75"], ["61", "67"], ["61", "64"], ["70", "6c"], ["696e20", "74686520"], ["76", "6520"], ["6f", "70"], ["6172", "20"], ["64", "65"], ["6f72", "20"], ["62", "65"], ["6f75", "6e"], ["63", "6520"], ["6572", "7320"], ["6f", "76"], ["75", "6c"], ["61", "70"], ["69", "73"], ["62", "75"], ["74", "656420"], ["70", "726f"], ["6869", "7320"], ["617469", "6f6e20"], ["63", "6f6d"], ["7468", "65"], ["77", "65726520"], ["70", "6f"], ["726f", "6d20"], ["66", "726f6d20"], ["63", "6c"], ["76", "69"], ["61", "62"], ["65", "78"], ["7469", "6f6e20"], ["73", "70"], ["77", "68"], ["61", "7920"], ["69", "7420"], ["77", "6869"], ["6f", "63"], ["75", "73"], ["6f6e", "7320"], ["746f20", "74686520"], ["77", "20"], ["74", "7320"], ["69", "6d"], ["70", "20"], ["74", "657220"], ["64", "7320"], ["6174", "656420"], ["71", "75"], ["72", "75"], ["6174", "6520"], ["70", "6572"], ["6e", "6f"], ["74", "6572"], ["6f75", "72"], ["64", "656420"], ["65", "6172"], ["70", "7265"], ["6f", "6f"], ["696e6720", "74686520"], ["61", "79"], ["61", "696e"], ["6861", "6420"], ["66", "66"], ["7468", "657220"], ["53", "74"], ["6972", "20"], ["6f75", "6768"], ["6174", "74"], ["70", "6172"], ["616c", "6c20"], ["696e", "67"], ["62", "657220"], ["6c", "6420"], ["776869", "636820"], ["73", "6520"], ["75", "6d"], ["656420", "74686520"], ["64", "696e6720"], ["63", "20"], ["6f6e", "6520"], ["616d", "6520"], ["7265", "63"], ["73", "65"], ["64", "6520"], ["64", "75"], ["6d", "6f"], ["49", "6e20"], ["616c", "69"], ["7320", "616e6420"], ["74", "72"], ["6f6e", "6720"], ["6e", "65"], ["6f75", "7420"], ["6172", "6520"], ["77", "65"], ["70", "6f72"], ["73", "6f20"], ["66", "6972"], ["6d", "616e"], ["656420", "746f20"], ["67", "75"], ["6f6e20", "74686520"], ["6c", "65"], ["61", "6b"], ["656420", "627920"], ["66", "69"], ["74", "7572"], ["7469", "6d"], ["7368", "20"], ["62", "6520"], ["6172", "7920"], ["73", "63"], ["67", "6520"], ["64", "657220"], ["74", "7920"], ["43", "68"], ["616c", "6c7920"], ["6c", "6f77"], ["74", "6520"], ["7369", "6f6e20"], ["41", "20"], ["69", "76"], ["64", "6572"], ["6974", "7920"], ["6974", "7320"], ["7374", "72"], ["65", "70"], ["616e6420", "74686520"], ["69", "20"], ["696e", "6520"], ["6768", "7420"], ["77", "6f72"], ["746865", "697220"], ["63", "69"], ["6f77", "6e20"], ["73", "6869"], ["6275", "7420"], ["6e6f", "7420"], ["616c", "736f20"], ["6c", "6561"], ["66", "74657220"], ["54", "68"], ["74", "77"], ["6f756e", "6420"]]} |