{ "format": "anlp-byte-bpe", "version": 2, "special_tokens": [ "[PAD]", "[UNK]", "[BOS]", "[EOS]" ], "requested_vocab_size": 1000, "min_frequency": 2, "merges": [ [ 45, 116 ], [ 91, 26 ], [ 89, 22 ], [ 40, 114 ], [ 87, 101 ], [ 91, 20 ], [ 47, 112 ], [ 57, 22 ], [ 22, 72 ], [ 22, 70 ], [ 69, 26 ], [ 54, 114 ], [ 69, 20 ], [ 67, 116 ], [ 65, 112 ], [ 73, 101 ], [ 40, 94 ], [ 74, 92 ], [ 74, 98 ], [ 60, 60 ], [ 57, 42 ], [ 62, 40 ], [ 46, 112 ], [ 90, 26 ], [ 44, 116 ], [ 71, 22 ], [ 39, 22 ], [ 41, 114 ], [ 90, 20 ], [ 86, 101 ], [ 36, 112 ], [ 51, 114 ], [ 87, 70 ], [ 96, 26 ], [ 96, 20 ], [ 38, 116 ], [ 87, 72 ], [ 41, 66 ], [ 91, 72 ], [ 91, 70 ], [ 92, 101 ], [ 95, 98 ], [ 88, 22 ], [ 56, 22 ], [ 268, 94 ], [ 49, 66 ], [ 87, 98 ], [ 269, 45 ], [ 45, 38 ], [ 47, 66 ], [ 57, 68 ], [ 40, 64 ], [ 74, 93 ], [ 87, 55 ], [ 74, 99 ], [ 89, 68 ], [ 62, 112 ], [ 57, 114 ], [ 74, 26 ], [ 60, 116 ], [ 74, 20 ], [ 96, 91 ], [ 277, 261 ], [ 276, 262 ], [ 278, 263 ], [ 280, 260 ], [ 43, 38 ], [ 97, 70 ], [ 51, 38 ], [ 97, 98 ], [ 279, 265 ], [ 44, 36 ], [ 49, 40 ], [ 304, 264 ], [ 39, 66 ], [ 95, 92 ], [ 281, 267 ], [ 36, 64 ], [ 49, 38 ], [ 307, 266 ], [ 36, 36 ], [ 85, 96 ], [ 87, 94 ], [ 85, 98 ], [ 87, 92 ], [ 22, 91 ], [ 53, 96 ], [ 22, 93 ], [ 99, 51 ], [ 97, 72 ], [ 51, 60 ], [ 89, 283 ], [ 40, 282 ], [ 50, 64 ], [ 68, 93 ], [ 47, 284 ], [ 83, 101 ], [ 39, 38 ], [ 57, 288 ], [ 79, 26 ], [ 59, 112 ], [ 87, 287 ], [ 57, 116 ], [ 70, 101 ], [ 45, 66 ], [ 95, 72 ], [ 60, 114 ], [ 22, 99 ], [ 79, 20 ], [ 91, 289 ], [ 99, 68 ], [ 37, 66 ], [ 68, 99 ], [ 95, 70 ], [ 91, 98 ], [ 96, 85 ], [ 67, 68 ], [ 51, 66 ], [ 36, 38 ], [ 85, 99 ], [ 68, 87 ], [ 91, 51 ], [ 91, 302 ], [ 87, 96 ], [ 97, 84 ], [ 68, 89 ], [ 69, 72 ], [ 93, 55 ], [ 67, 40 ], [ 37, 112 ], [ 50, 114 ], [ 22, 85 ], [ 49, 64 ], [ 39, 116 ], [ 45, 303 ], [ 53, 98 ], [ 91, 55 ], [ 47, 38 ], [ 97, 26 ], [ 37, 61 ], [ 44, 112 ], [ 97, 20 ], [ 97, 71 ], [ 53, 68 ], [ 97, 73 ], [ 46, 116 ], [ 97, 90 ], [ 88, 20 ], [ 88, 26 ], [ 43, 114 ], [ 97, 92 ], [ 39, 58 ], [ 37, 38 ], [ 50, 400 ], [ 95, 89 ], [ 55, 47 ], [ 57, 43 ], [ 49, 39 ], [ 45, 22 ], [ 57, 96 ], [ 36, 62 ], [ 91, 92 ], [ 85, 68 ], [ 85, 93 ], [ 101, 336 ], [ 51, 36 ], [ 60, 67 ], [ 67, 407 ], [ 64, 45 ], [ 37, 405 ], [ 49, 116 ], [ 51, 67 ], [ 87, 51 ], [ 40, 36 ], [ 50, 36 ], [ 94, 93 ], [ 69, 93 ], [ 87, 20 ], [ 75, 72 ], [ 305, 288 ], [ 340, 284 ], [ 42, 114 ], [ 51, 112 ], [ 47, 116 ], [ 45, 112 ], [ 36, 114 ], [ 66, 57 ], [ 89, 20 ], [ 61, 96 ], [ 344, 287 ], [ 310, 20 ], [ 77, 22 ], [ 306, 283 ], [ 87, 26 ], [ 89, 96 ], [ 311, 112 ], [ 37, 295 ], [ 89, 26 ], [ 39, 41 ], [ 341, 289 ], [ 42, 260 ], [ 309, 116 ], [ 98, 93 ], [ 326, 59 ], [ 313, 114 ], [ 61, 321 ], [ 69, 99 ], [ 315, 26 ], [ 49, 114 ], [ 98, 99 ], [ 298, 101 ], [ 67, 294 ], [ 97, 300 ], [ 94, 99 ], [ 95, 87 ], [ 50, 290 ], [ 38, 91 ], [ 45, 114 ], [ 44, 290 ], [ 49, 43 ], [ 22, 73 ], [ 50, 63 ], [ 61, 294 ], [ 41, 40 ], [ 85, 72 ], [ 43, 295 ], [ 95, 300 ], [ 297, 447 ], [ 335, 441 ], [ 22, 88 ], [ 331, 443 ], [ 69, 70 ], [ 91, 94 ], [ 375, 101 ], [ 301, 457 ], [ 346, 302 ], [ 95, 99 ], [ 75, 70 ], [ 328, 303 ], [ 22, 71 ], [ 97, 95 ], [ 22, 86 ], [ 45, 43 ], [ 54, 62 ], [ 57, 389 ], [ 65, 60 ], [ 74, 72 ], [ 25, 460 ], [ 89, 98 ], [ 22, 69 ], [ 53, 272 ], [ 40, 61 ], [ 62, 393 ], [ 74, 70 ], [ 95, 93 ], [ 40, 401 ], [ 91, 87 ], [ 22, 89 ], [ 22, 75 ], [ 66, 67 ], [ 87, 71 ], [ 91, 89 ], [ 51, 273 ], [ 36, 274 ], [ 22, 98 ], [ 91, 91 ], [ 87, 275 ], [ 348, 444 ], [ 39, 64 ], [ 44, 62 ], [ 40, 274 ], [ 95, 71 ], [ 85, 70 ], [ 60, 61 ], [ 41, 55 ], [ 47, 273 ], [ 91, 73 ], [ 22, 87 ], [ 45, 39 ], [ 384, 101 ], [ 312, 101 ], [ 51, 37 ], [ 22, 92 ], [ 73, 98 ], [ 90, 93 ], [ 73, 99 ], [ 85, 92 ], [ 87, 285 ], [ 41, 39 ], [ 57, 272 ], [ 55, 37 ], [ 44, 38 ], [ 91, 71 ], [ 87, 95 ], [ 67, 61 ], [ 269, 390 ], [ 99, 93 ], [ 51, 282 ], [ 44, 49 ], [ 97, 97 ], [ 91, 271 ], [ 56, 64 ], [ 36, 61 ], [ 49, 286 ], [ 91, 275 ], [ 45, 286 ], [ 85, 270 ], [ 268, 398 ], [ 40, 38 ], [ 95, 96 ], [ 38, 42 ], [ 63, 66 ], [ 64, 39 ], [ 99, 99 ], [ 67, 96 ], [ 45, 40 ], [ 367, 408 ], [ 91, 57 ], [ 91, 285 ], [ 95, 73 ], [ 73, 93 ], [ 41, 64 ], [ 94, 26 ], [ 54, 43 ], [ 87, 93 ], [ 96, 87 ], [ 95, 85 ], [ 36, 39 ], [ 45, 64 ], [ 91, 85 ], [ 347, 409 ], [ 45, 266 ], [ 91, 101 ], [ 69, 73 ], [ 40, 65 ], [ 69, 71 ], [ 22, 74 ], [ 99, 20 ], [ 22, 68 ], [ 50, 39 ], [ 89, 270 ], [ 94, 264 ], [ 48, 114 ], [ 91, 88 ], [ 55, 43 ], [ 89, 99 ], [ 50, 38 ], [ 44, 65 ], [ 342, 101 ], [ 90, 99 ], [ 87, 73 ], [ 22, 84 ], [ 87, 97 ], [ 95, 94 ], [ 36, 290 ], [ 419, 72 ], [ 66, 61 ], [ 381, 282 ], [ 53, 74 ], [ 345, 557 ], [ 75, 98 ], [ 97, 74 ], [ 343, 26 ], [ 99, 291 ], [ 47, 60 ], [ 36, 45 ], [ 87, 74 ], [ 85, 85 ], [ 51, 295 ], [ 88, 93 ], [ 397, 40 ], [ 22, 90 ], [ 36, 316 ], [ 68, 91 ], [ 49, 65 ], [ 378, 112 ], [ 416, 334 ], [ 425, 116 ], [ 49, 55 ], [ 56, 36 ], [ 37, 36 ], [ 87, 88 ], [ 91, 96 ], [ 53, 294 ], [ 276, 20 ], [ 69, 87 ], [ 94, 20 ], [ 37, 116 ], [ 44, 456 ], [ 57, 64 ], [ 90, 87 ], [ 281, 116 ], [ 45, 56 ], [ 44, 60 ], [ 46, 61 ], [ 70, 93 ], [ 496, 293 ], [ 75, 92 ], [ 36, 65 ], [ 53, 320 ], [ 87, 86 ], [ 51, 40 ], [ 40, 49 ], [ 88, 370 ], [ 95, 68 ], [ 95, 74 ], [ 44, 274 ], [ 51, 62 ], [ 70, 70 ], [ 395, 20 ], [ 93, 409 ], [ 51, 319 ], [ 44, 61 ], [ 70, 72 ], [ 57, 98 ], [ 22, 459 ], [ 87, 300 ], [ 99, 408 ], [ 376, 20 ], [ 96, 99 ], [ 371, 116 ], [ 22, 104 ], [ 41, 471 ], [ 69, 89 ], [ 91, 95 ], [ 96, 93 ], [ 40, 267 ], [ 45, 53 ], [ 514, 291 ], [ 41, 38 ], [ 280, 112 ], [ 69, 363 ], [ 58, 327 ], [ 91, 74 ], [ 353, 112 ], [ 88, 387 ], [ 22, 102 ], [ 51, 64 ], [ 97, 96 ], [ 88, 99 ], [ 96, 70 ], [ 54, 316 ], [ 49, 360 ], [ 50, 61 ], [ 97, 94 ], [ 94, 262 ], [ 69, 92 ], [ 60, 265 ], [ 91, 291 ], [ 87, 68 ], [ 93, 293 ], [ 38, 41 ], [ 73, 560 ], [ 87, 89 ], [ 44, 64 ], [ 95, 275 ], [ 44, 357 ], [ 74, 87 ], [ 97, 68 ], [ 278, 101 ], [ 70, 99 ], [ 22, 117 ], [ 36, 116 ], [ 45, 55 ], [ 51, 61 ], [ 91, 68 ], [ 63, 510 ], [ 39, 39 ], [ 69, 98 ], [ 47, 67 ], [ 98, 85 ], [ 22, 123 ], [ 97, 75 ], [ 22, 95 ], [ 73, 566 ], [ 85, 318 ], [ 95, 285 ], [ 51, 63 ], [ 62, 61 ], [ 58, 522 ], [ 72, 93 ], [ 49, 36 ], [ 292, 101 ], [ 90, 89 ], [ 43, 66 ], [ 96, 89 ], [ 95, 95 ], [ 41, 286 ], [ 75, 94 ], [ 91, 317 ], [ 267, 72 ], [ 106, 98 ], [ 41, 68 ], [ 51, 47 ], [ 63, 60 ], [ 88, 72 ], [ 106, 92 ], [ 44, 57 ], [ 45, 65 ], [ 262, 70 ], [ 95, 86 ], [ 22, 97 ], [ 97, 69 ], [ 43, 60 ], [ 22, 103 ], [ 22, 105 ], [ 98, 26 ], [ 89, 92 ], [ 43, 547 ], [ 74, 91 ], [ 74, 361 ], [ 54, 42 ], [ 99, 26 ], [ 95, 88 ], [ 99, 85 ], [ 98, 20 ], [ 44, 46 ], [ 49, 37 ], [ 413, 323 ], [ 429, 322 ], [ 91, 300 ], [ 57, 294 ], [ 75, 71 ], [ 406, 339 ], [ 98, 87 ], [ 349, 101 ], [ 40, 39 ], [ 41, 43 ], [ 36, 67 ], [ 44, 39 ], [ 73, 92 ], [ 367, 291 ], [ 36, 57 ], [ 345, 101 ], [ 427, 324 ], [ 71, 55 ], [ 411, 333 ], [ 22, 118 ], [ 61, 563 ], [ 63, 38 ], [ 75, 96 ], [ 36, 49 ], [ 41, 47 ], [ 38, 62 ], [ 73, 68 ], [ 98, 91 ], [ 347, 293 ], [ 61, 40 ], [ 40, 290 ], [ 41, 42 ], [ 57, 274 ], [ 98, 356 ], [ 87, 85 ], [ 22, 119 ], [ 22, 121 ], [ 41, 65 ], [ 53, 312 ], [ 95, 97 ], [ 36, 46 ], [ 39, 43 ], [ 56, 38 ], [ 85, 91 ], [ 384, 424 ], [ 88, 70 ], [ 61, 329 ], [ 67, 57 ], [ 74, 275 ], [ 22, 120 ], [ 41, 59 ], [ 84, 87 ], [ 63, 61 ], [ 49, 56 ], [ 47, 295 ], [ 54, 65 ], [ 75, 73 ], [ 93, 20 ], [ 49, 266 ], [ 84, 89 ], [ 72, 99 ], [ 67, 60 ], [ 94, 366 ], [ 22, 124 ], [ 98, 89 ], [ 57, 355 ], [ 66, 53 ], [ 50, 57 ], [ 60, 38 ], [ 94, 91 ], [ 58, 42 ], [ 22, 127 ], [ 38, 37 ], [ 68, 85 ], [ 22, 129 ], [ 314, 475 ], [ 96, 359 ], [ 49, 47 ], [ 40, 60 ], [ 48, 66 ], [ 346, 88 ], [ 46, 114 ], [ 51, 46 ], [ 56, 65 ], [ 58, 43 ], [ 63, 67 ], [ 74, 85 ], [ 36, 53 ], [ 50, 65 ], [ 37, 67 ], [ 53, 38 ], [ 12, 290 ], [ 40, 62 ], [ 30, 686 ], [ 45, 61 ], [ 49, 51 ], [ 54, 47 ], [ 42, 42 ], [ 87, 271 ], [ 22, 130 ], [ 43, 51 ], [ 61, 41 ], [ 74, 285 ], [ 39, 47 ], [ 61, 272 ], [ 67, 67 ], [ 38, 412 ], [ 97, 93 ], [ 49, 63 ], [ 89, 93 ], [ 43, 365 ], [ 64, 38 ], [ 74, 351 ], [ 756, 261 ], [ 328, 56 ], [ 49, 59 ], [ 342, 26 ], [ 454, 70 ], [ 98, 94 ], [ 98, 264 ], [ 49, 61 ], [ 68, 26 ], [ 74, 89 ], [ 50, 66 ], [ 751, 263 ], [ 22, 125 ], [ 57, 47 ], [ 38, 266 ], [ 332, 116 ], [ 28, 707 ], [ 96, 86 ], [ 96, 73 ], [ 276, 735 ], [ 343, 101 ], [ 8, 705 ], [ 91, 93 ], [ 279, 296 ], [ 22, 94 ], [ 50, 41 ], [ 570, 112 ], [ 43, 273 ], [ 55, 114 ], [ 278, 632 ], [ 44, 41 ], [ 308, 265 ], [ 91, 97 ], [ 432, 62 ], [ 61, 66 ], [ 830, 277 ], [ 39, 65 ], [ 85, 89 ], [ 22, 126 ], [ 51, 44 ], [ 40, 53 ], [ 299, 261 ], [ 533, 329 ], [ 95, 91 ], [ 350, 824 ], [ 98, 262 ], [ 41, 56 ], [ 281, 659 ], [ 22, 131 ], [ 55, 329 ], [ 431, 96 ], [ 298, 263 ], [ 42, 299 ], [ 85, 262 ], [ 311, 260 ], [ 43, 680 ], [ 51, 43 ], [ 280, 332 ], [ 281, 57 ], [ 44, 651 ], [ 313, 266 ], [ 84, 356 ], [ 74, 464 ], [ 74, 264 ], [ 315, 264 ], [ 374, 318 ], [ 51, 337 ], [ 89, 72 ], [ 862, 279 ], [ 54, 63 ], [ 509, 523 ], [ 36, 60 ], [ 39, 360 ], [ 84, 93 ], [ 97, 88 ], [ 888, 582 ], [ 50, 60 ], [ 98, 96 ], [ 68, 71 ], [ 695, 114 ], [ 45, 47 ], [ 54, 266 ], [ 59, 43 ], [ 373, 477 ], [ 51, 444 ], [ 518, 526 ], [ 663, 26 ], [ 36, 418 ], [ 40, 299 ], [ 87, 261 ], [ 879, 644 ], [ 48, 266 ], [ 91, 75 ], [ 48, 362 ], [ 94, 87 ], [ 338, 112 ], [ 46, 38 ], [ 89, 85 ], [ 40, 66 ], [ 74, 315 ], [ 364, 320 ], [ 59, 266 ], [ 91, 99 ], [ 433, 319 ], [ 46, 678 ], [ 297, 333 ], [ 48, 39 ], [ 85, 95 ], [ 432, 316 ] ], "training_metadata": { "dataset_sha256": "a68deecf98a7b4911a6a1594e32d66dc8c4a1614f13eaf76bbf40a0845ffd6d3", "train_indices_sha256": "f1f4d74fed4345d743a0f6de93e2de24ebfd16133016ca44400fd8a898ed4504", "example_unit": "line", "window_bytes": null, "role": "source" }, "input_mode": "cipher_bits", "pretokenizer": "whole" }