anlp-assignment-1 / C2 /source_tokenizer.json
jupist's picture
Upload verified C2 checkpoint
acf755c verified
Raw History Blame Contribute Delete
24.6 kB
{
"format": "anlp-byte-bpe",
"version": 2,
"special_tokens": [
"[PAD]",
"[UNK]",
"[BOS]",
"[EOS]"
],
"requested_vocab_size": 1000,
"min_frequency": 2,
"merges": [
[
45,
116
],
[
91,
26
],
[
89,
22
],
[
40,
114
],
[
87,
101
],
[
91,
20
],
[
47,
112
],
[
57,
22
],
[
22,
72
],
[
22,
70
],
[
69,
26
],
[
54,
114
],
[
69,
20
],
[
67,
116
],
[
65,
112
],
[
73,
101
],
[
40,
94
],
[
74,
92
],
[
74,
98
],
[
60,
60
],
[
57,
42
],
[
62,
40
],
[
46,
112
],
[
90,
26
],
[
44,
116
],
[
71,
22
],
[
39,
22
],
[
41,
114
],
[
90,
20
],
[
86,
101
],
[
36,
112
],
[
51,
114
],
[
87,
70
],
[
96,
26
],
[
96,
20
],
[
38,
116
],
[
87,
72
],
[
41,
66
],
[
91,
72
],
[
91,
70
],
[
92,
101
],
[
95,
98
],
[
88,
22
],
[
56,
22
],
[
268,
94
],
[
49,
66
],
[
87,
98
],
[
269,
45
],
[
45,
38
],
[
47,
66
],
[
57,
68
],
[
40,
64
],
[
74,
93
],
[
87,
55
],
[
74,
99
],
[
89,
68
],
[
62,
112
],
[
57,
114
],
[
74,
26
],
[
60,
116
],
[
74,
20
],
[
96,
91
],
[
277,
261
],
[
276,
262
],
[
278,
263
],
[
280,
260
],
[
43,
38
],
[
97,
70
],
[
51,
38
],
[
97,
98
],
[
279,
265
],
[
44,
36
],
[
49,
40
],
[
304,
264
],
[
39,
66
],
[
95,
92
],
[
281,
267
],
[
36,
64
],
[
49,
38
],
[
307,
266
],
[
36,
36
],
[
85,
96
],
[
87,
94
],
[
85,
98
],
[
87,
92
],
[
22,
91
],
[
53,
96
],
[
22,
93
],
[
99,
51
],
[
97,
72
],
[
51,
60
],
[
89,
283
],
[
40,
282
],
[
50,
64
],
[
68,
93
],
[
47,
284
],
[
83,
101
],
[
39,
38
],
[
57,
288
],
[
79,
26
],
[
59,
112
],
[
87,
287
],
[
57,
116
],
[
70,
101
],
[
45,
66
],
[
95,
72
],
[
60,
114
],
[
22,
99
],
[
79,
20
],
[
91,
289
],
[
99,
68
],
[
37,
66
],
[
68,
99
],
[
95,
70
],
[
91,
98
],
[
96,
85
],
[
67,
68
],
[
51,
66
],
[
36,
38
],
[
85,
99
],
[
68,
87
],
[
91,
51
],
[
91,
302
],
[
87,
96
],
[
97,
84
],
[
68,
89
],
[
69,
72
],
[
93,
55
],
[
67,
40
],
[
37,
112
],
[
50,
114
],
[
22,
85
],
[
49,
64
],
[
39,
116
],
[
45,
303
],
[
53,
98
],
[
91,
55
],
[
47,
38
],
[
97,
26
],
[
37,
61
],
[
44,
112
],
[
97,
20
],
[
97,
71
],
[
53,
68
],
[
97,
73
],
[
46,
116
],
[
97,
90
],
[
88,
20
],
[
88,
26
],
[
43,
114
],
[
97,
92
],
[
39,
58
],
[
37,
38
],
[
50,
400
],
[
95,
89
],
[
55,
47
],
[
57,
43
],
[
49,
39
],
[
45,
22
],
[
57,
96
],
[
36,
62
],
[
91,
92
],
[
85,
68
],
[
85,
93
],
[
101,
336
],
[
51,
36
],
[
60,
67
],
[
67,
407
],
[
64,
45
],
[
37,
405
],
[
49,
116
],
[
51,
67
],
[
87,
51
],
[
40,
36
],
[
50,
36
],
[
94,
93
],
[
69,
93
],
[
87,
20
],
[
75,
72
],
[
305,
288
],
[
340,
284
],
[
42,
114
],
[
51,
112
],
[
47,
116
],
[
45,
112
],
[
36,
114
],
[
66,
57
],
[
89,
20
],
[
61,
96
],
[
344,
287
],
[
310,
20
],
[
77,
22
],
[
306,
283
],
[
87,
26
],
[
89,
96
],
[
311,
112
],
[
37,
295
],
[
89,
26
],
[
39,
41
],
[
341,
289
],
[
42,
260
],
[
309,
116
],
[
98,
93
],
[
326,
59
],
[
313,
114
],
[
61,
321
],
[
69,
99
],
[
315,
26
],
[
49,
114
],
[
98,
99
],
[
298,
101
],
[
67,
294
],
[
97,
300
],
[
94,
99
],
[
95,
87
],
[
50,
290
],
[
38,
91
],
[
45,
114
],
[
44,
290
],
[
49,
43
],
[
22,
73
],
[
50,
63
],
[
61,
294
],
[
41,
40
],
[
85,
72
],
[
43,
295
],
[
95,
300
],
[
297,
447
],
[
335,
441
],
[
22,
88
],
[
331,
443
],
[
69,
70
],
[
91,
94
],
[
375,
101
],
[
301,
457
],
[
346,
302
],
[
95,
99
],
[
75,
70
],
[
328,
303
],
[
22,
71
],
[
97,
95
],
[
22,
86
],
[
45,
43
],
[
54,
62
],
[
57,
389
],
[
65,
60
],
[
74,
72
],
[
25,
460
],
[
89,
98
],
[
22,
69
],
[
53,
272
],
[
40,
61
],
[
62,
393
],
[
74,
70
],
[
95,
93
],
[
40,
401
],
[
91,
87
],
[
22,
89
],
[
22,
75
],
[
66,
67
],
[
87,
71
],
[
91,
89
],
[
51,
273
],
[
36,
274
],
[
22,
98
],
[
91,
91
],
[
87,
275
],
[
348,
444
],
[
39,
64
],
[
44,
62
],
[
40,
274
],
[
95,
71
],
[
85,
70
],
[
60,
61
],
[
41,
55
],
[
47,
273
],
[
91,
73
],
[
22,
87
],
[
45,
39
],
[
384,
101
],
[
312,
101
],
[
51,
37
],
[
22,
92
],
[
73,
98
],
[
90,
93
],
[
73,
99
],
[
85,
92
],
[
87,
285
],
[
41,
39
],
[
57,
272
],
[
55,
37
],
[
44,
38
],
[
91,
71
],
[
87,
95
],
[
67,
61
],
[
269,
390
],
[
99,
93
],
[
51,
282
],
[
44,
49
],
[
97,
97
],
[
91,
271
],
[
56,
64
],
[
36,
61
],
[
49,
286
],
[
91,
275
],
[
45,
286
],
[
85,
270
],
[
268,
398
],
[
40,
38
],
[
95,
96
],
[
38,
42
],
[
63,
66
],
[
64,
39
],
[
99,
99
],
[
67,
96
],
[
45,
40
],
[
367,
408
],
[
91,
57
],
[
91,
285
],
[
95,
73
],
[
73,
93
],
[
41,
64
],
[
94,
26
],
[
54,
43
],
[
87,
93
],
[
96,
87
],
[
95,
85
],
[
36,
39
],
[
45,
64
],
[
91,
85
],
[
347,
409
],
[
45,
266
],
[
91,
101
],
[
69,
73
],
[
40,
65
],
[
69,
71
],
[
22,
74
],
[
99,
20
],
[
22,
68
],
[
50,
39
],
[
89,
270
],
[
94,
264
],
[
48,
114
],
[
91,
88
],
[
55,
43
],
[
89,
99
],
[
50,
38
],
[
44,
65
],
[
342,
101
],
[
90,
99
],
[
87,
73
],
[
22,
84
],
[
87,
97
],
[
95,
94
],
[
36,
290
],
[
419,
72
],
[
66,
61
],
[
381,
282
],
[
53,
74
],
[
345,
557
],
[
75,
98
],
[
97,
74
],
[
343,
26
],
[
99,
291
],
[
47,
60
],
[
36,
45
],
[
87,
74
],
[
85,
85
],
[
51,
295
],
[
88,
93
],
[
397,
40
],
[
22,
90
],
[
36,
316
],
[
68,
91
],
[
49,
65
],
[
378,
112
],
[
416,
334
],
[
425,
116
],
[
49,
55
],
[
56,
36
],
[
37,
36
],
[
87,
88
],
[
91,
96
],
[
53,
294
],
[
276,
20
],
[
69,
87
],
[
94,
20
],
[
37,
116
],
[
44,
456
],
[
57,
64
],
[
90,
87
],
[
281,
116
],
[
45,
56
],
[
44,
60
],
[
46,
61
],
[
70,
93
],
[
496,
293
],
[
75,
92
],
[
36,
65
],
[
53,
320
],
[
87,
86
],
[
51,
40
],
[
40,
49
],
[
88,
370
],
[
95,
68
],
[
95,
74
],
[
44,
274
],
[
51,
62
],
[
70,
70
],
[
395,
20
],
[
93,
409
],
[
51,
319
],
[
44,
61
],
[
70,
72
],
[
57,
98
],
[
22,
459
],
[
87,
300
],
[
99,
408
],
[
376,
20
],
[
96,
99
],
[
371,
116
],
[
22,
104
],
[
41,
471
],
[
69,
89
],
[
91,
95
],
[
96,
93
],
[
40,
267
],
[
45,
53
],
[
514,
291
],
[
41,
38
],
[
280,
112
],
[
69,
363
],
[
58,
327
],
[
91,
74
],
[
353,
112
],
[
88,
387
],
[
22,
102
],
[
51,
64
],
[
97,
96
],
[
88,
99
],
[
96,
70
],
[
54,
316
],
[
49,
360
],
[
50,
61
],
[
97,
94
],
[
94,
262
],
[
69,
92
],
[
60,
265
],
[
91,
291
],
[
87,
68
],
[
93,
293
],
[
38,
41
],
[
73,
560
],
[
87,
89
],
[
44,
64
],
[
95,
275
],
[
44,
357
],
[
74,
87
],
[
97,
68
],
[
278,
101
],
[
70,
99
],
[
22,
117
],
[
36,
116
],
[
45,
55
],
[
51,
61
],
[
91,
68
],
[
63,
510
],
[
39,
39
],
[
69,
98
],
[
47,
67
],
[
98,
85
],
[
22,
123
],
[
97,
75
],
[
22,
95
],
[
73,
566
],
[
85,
318
],
[
95,
285
],
[
51,
63
],
[
62,
61
],
[
58,
522
],
[
72,
93
],
[
49,
36
],
[
292,
101
],
[
90,
89
],
[
43,
66
],
[
96,
89
],
[
95,
95
],
[
41,
286
],
[
75,
94
],
[
91,
317
],
[
267,
72
],
[
106,
98
],
[
41,
68
],
[
51,
47
],
[
63,
60
],
[
88,
72
],
[
106,
92
],
[
44,
57
],
[
45,
65
],
[
262,
70
],
[
95,
86
],
[
22,
97
],
[
97,
69
],
[
43,
60
],
[
22,
103
],
[
22,
105
],
[
98,
26
],
[
89,
92
],
[
43,
547
],
[
74,
91
],
[
74,
361
],
[
54,
42
],
[
99,
26
],
[
95,
88
],
[
99,
85
],
[
98,
20
],
[
44,
46
],
[
49,
37
],
[
413,
323
],
[
429,
322
],
[
91,
300
],
[
57,
294
],
[
75,
71
],
[
406,
339
],
[
98,
87
],
[
349,
101
],
[
40,
39
],
[
41,
43
],
[
36,
67
],
[
44,
39
],
[
73,
92
],
[
367,
291
],
[
36,
57
],
[
345,
101
],
[
427,
324
],
[
71,
55
],
[
411,
333
],
[
22,
118
],
[
61,
563
],
[
63,
38
],
[
75,
96
],
[
36,
49
],
[
41,
47
],
[
38,
62
],
[
73,
68
],
[
98,
91
],
[
347,
293
],
[
61,
40
],
[
40,
290
],
[
41,
42
],
[
57,
274
],
[
98,
356
],
[
87,
85
],
[
22,
119
],
[
22,
121
],
[
41,
65
],
[
53,
312
],
[
95,
97
],
[
36,
46
],
[
39,
43
],
[
56,
38
],
[
85,
91
],
[
384,
424
],
[
88,
70
],
[
61,
329
],
[
67,
57
],
[
74,
275
],
[
22,
120
],
[
41,
59
],
[
84,
87
],
[
63,
61
],
[
49,
56
],
[
47,
295
],
[
54,
65
],
[
75,
73
],
[
93,
20
],
[
49,
266
],
[
84,
89
],
[
72,
99
],
[
67,
60
],
[
94,
366
],
[
22,
124
],
[
98,
89
],
[
57,
355
],
[
66,
53
],
[
50,
57
],
[
60,
38
],
[
94,
91
],
[
58,
42
],
[
22,
127
],
[
38,
37
],
[
68,
85
],
[
22,
129
],
[
314,
475
],
[
96,
359
],
[
49,
47
],
[
40,
60
],
[
48,
66
],
[
346,
88
],
[
46,
114
],
[
51,
46
],
[
56,
65
],
[
58,
43
],
[
63,
67
],
[
74,
85
],
[
36,
53
],
[
50,
65
],
[
37,
67
],
[
53,
38
],
[
12,
290
],
[
40,
62
],
[
30,
686
],
[
45,
61
],
[
49,
51
],
[
54,
47
],
[
42,
42
],
[
87,
271
],
[
22,
130
],
[
43,
51
],
[
61,
41
],
[
74,
285
],
[
39,
47
],
[
61,
272
],
[
67,
67
],
[
38,
412
],
[
97,
93
],
[
49,
63
],
[
89,
93
],
[
43,
365
],
[
64,
38
],
[
74,
351
],
[
756,
261
],
[
328,
56
],
[
49,
59
],
[
342,
26
],
[
454,
70
],
[
98,
94
],
[
98,
264
],
[
49,
61
],
[
68,
26
],
[
74,
89
],
[
50,
66
],
[
751,
263
],
[
22,
125
],
[
57,
47
],
[
38,
266
],
[
332,
116
],
[
28,
707
],
[
96,
86
],
[
96,
73
],
[
276,
735
],
[
343,
101
],
[
8,
705
],
[
91,
93
],
[
279,
296
],
[
22,
94
],
[
50,
41
],
[
570,
112
],
[
43,
273
],
[
55,
114
],
[
278,
632
],
[
44,
41
],
[
308,
265
],
[
91,
97
],
[
432,
62
],
[
61,
66
],
[
830,
277
],
[
39,
65
],
[
85,
89
],
[
22,
126
],
[
51,
44
],
[
40,
53
],
[
299,
261
],
[
533,
329
],
[
95,
91
],
[
350,
824
],
[
98,
262
],
[
41,
56
],
[
281,
659
],
[
22,
131
],
[
55,
329
],
[
431,
96
],
[
298,
263
],
[
42,
299
],
[
85,
262
],
[
311,
260
],
[
43,
680
],
[
51,
43
],
[
280,
332
],
[
281,
57
],
[
44,
651
],
[
313,
266
],
[
84,
356
],
[
74,
464
],
[
74,
264
],
[
315,
264
],
[
374,
318
],
[
51,
337
],
[
89,
72
],
[
862,
279
],
[
54,
63
],
[
509,
523
],
[
36,
60
],
[
39,
360
],
[
84,
93
],
[
97,
88
],
[
888,
582
],
[
50,
60
],
[
98,
96
],
[
68,
71
],
[
695,
114
],
[
45,
47
],
[
54,
266
],
[
59,
43
],
[
373,
477
],
[
51,
444
],
[
518,
526
],
[
663,
26
],
[
36,
418
],
[
40,
299
],
[
87,
261
],
[
879,
644
],
[
48,
266
],
[
91,
75
],
[
48,
362
],
[
94,
87
],
[
338,
112
],
[
46,
38
],
[
89,
85
],
[
40,
66
],
[
74,
315
],
[
364,
320
],
[
59,
266
],
[
91,
99
],
[
433,
319
],
[
46,
678
],
[
297,
333
],
[
48,
39
],
[
85,
95
],
[
432,
316
]
],
"training_metadata": {
"dataset_sha256": "a68deecf98a7b4911a6a1594e32d66dc8c4a1614f13eaf76bbf40a0845ffd6d3",
"train_indices_sha256": "f1f4d74fed4345d743a0f6de93e2de24ebfd16133016ca44400fd8a898ed4504",
"example_unit": "line",
"window_bytes": null,
"role": "source"
},
"input_mode": "cipher_bits",
"pretokenizer": "whole"
}