nicholasbien commited on
Commit
26ea417
·
verified ·
1 Parent(s): 0a0ef8e

v4: v4/tokenizer.json

Browse files
Files changed (1) hide show
  1. v4/tokenizer.json +511 -0
v4/tokenizer.json ADDED
@@ -0,0 +1,511 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "pitch_range": [
4
+ 0,
5
+ 127
6
+ ],
7
+ "beat_res": {
8
+ "0_4": 8,
9
+ "4_12": 4
10
+ },
11
+ "num_velocities": 32,
12
+ "remove_duplicated_notes": false,
13
+ "encode_ids_split": "bar",
14
+ "special_tokens": [
15
+ "PAD_None",
16
+ "BOS_None",
17
+ "EOS_None",
18
+ "SEP_None",
19
+ "MASK_None",
20
+ "Task_accomp",
21
+ "Task_infill",
22
+ "Inst_Piano",
23
+ "Inst_ChromPerc",
24
+ "Inst_Organ",
25
+ "Inst_Guitar",
26
+ "Inst_Bass",
27
+ "Inst_Strings",
28
+ "Inst_Ensemble",
29
+ "Inst_Brass",
30
+ "Inst_Reed",
31
+ "Inst_Pipe",
32
+ "Inst_SynthLead",
33
+ "Inst_SynthPad",
34
+ "Inst_SynthFX",
35
+ "Inst_Ethnic",
36
+ "Inst_Percussive",
37
+ "Inst_SoundFX",
38
+ "Inst_Drums",
39
+ "Density_0",
40
+ "Density_1",
41
+ "Density_2",
42
+ "Density_3",
43
+ "Poly_0",
44
+ "Poly_1",
45
+ "Poly_2",
46
+ "Range_0",
47
+ "Range_1",
48
+ "Range_2",
49
+ "Source_lakh",
50
+ "Source_lamd",
51
+ "Source_aria",
52
+ "Source_gigamidi",
53
+ "Source_maestro",
54
+ "Source_pop909",
55
+ "Source_giantmidi",
56
+ "Source_user",
57
+ "Genre_rock",
58
+ "Genre_pop",
59
+ "Genre_jazz",
60
+ "Genre_classical",
61
+ "Genre_electronic",
62
+ "Genre_hiphop",
63
+ "Genre_rnb",
64
+ "Genre_country",
65
+ "Genre_folk",
66
+ "Genre_latin",
67
+ "Genre_blues",
68
+ "Genre_metal",
69
+ "Genre_reggae",
70
+ "Genre_soul",
71
+ "Genre_world",
72
+ "Genre_other",
73
+ "Quality_0",
74
+ "Quality_1",
75
+ "Quality_2",
76
+ "Quality_3"
77
+ ],
78
+ "use_velocities": true,
79
+ "use_note_duration_programs": [
80
+ 0,
81
+ 1,
82
+ 2,
83
+ 3,
84
+ 4,
85
+ 5,
86
+ 6,
87
+ 7,
88
+ 8,
89
+ 9,
90
+ 10,
91
+ 11,
92
+ 12,
93
+ 13,
94
+ 14,
95
+ 15,
96
+ 16,
97
+ 17,
98
+ 18,
99
+ 19,
100
+ 20,
101
+ 21,
102
+ 22,
103
+ 23,
104
+ 24,
105
+ 25,
106
+ 26,
107
+ 27,
108
+ 28,
109
+ 29,
110
+ 30,
111
+ 31,
112
+ 32,
113
+ 33,
114
+ 34,
115
+ 35,
116
+ 36,
117
+ 37,
118
+ 38,
119
+ 39,
120
+ 40,
121
+ 41,
122
+ 42,
123
+ 43,
124
+ 44,
125
+ 45,
126
+ 46,
127
+ 47,
128
+ 48,
129
+ 49,
130
+ 50,
131
+ 51,
132
+ 52,
133
+ 53,
134
+ 54,
135
+ 55,
136
+ 56,
137
+ 57,
138
+ 58,
139
+ 59,
140
+ 60,
141
+ 61,
142
+ 62,
143
+ 63,
144
+ 64,
145
+ 65,
146
+ 66,
147
+ 67,
148
+ 68,
149
+ 69,
150
+ 70,
151
+ 71,
152
+ 72,
153
+ 73,
154
+ 74,
155
+ 75,
156
+ 76,
157
+ 77,
158
+ 78,
159
+ 79,
160
+ 80,
161
+ 81,
162
+ 82,
163
+ 83,
164
+ 84,
165
+ 85,
166
+ 86,
167
+ 87,
168
+ 88,
169
+ 89,
170
+ 90,
171
+ 91,
172
+ 92,
173
+ 93,
174
+ 94,
175
+ 95,
176
+ 96,
177
+ 97,
178
+ 98,
179
+ 99,
180
+ 100,
181
+ 101,
182
+ 102,
183
+ 103,
184
+ 104,
185
+ 105,
186
+ 106,
187
+ 107,
188
+ 108,
189
+ 109,
190
+ 110,
191
+ 111,
192
+ 112,
193
+ 113,
194
+ 114,
195
+ 115,
196
+ 116,
197
+ 117,
198
+ 118,
199
+ 119,
200
+ 120,
201
+ 121,
202
+ 122,
203
+ 123,
204
+ 124,
205
+ 125,
206
+ 126,
207
+ 127,
208
+ -1
209
+ ],
210
+ "use_chords": false,
211
+ "use_rests": false,
212
+ "use_tempos": false,
213
+ "use_time_signatures": true,
214
+ "use_sustain_pedals": false,
215
+ "use_pitch_bends": false,
216
+ "use_programs": true,
217
+ "use_pitch_intervals": false,
218
+ "use_pitchdrum_tokens": true,
219
+ "default_note_duration": 0.5,
220
+ "programs": [
221
+ 0,
222
+ 1,
223
+ 2,
224
+ 3,
225
+ 4,
226
+ 5,
227
+ 6,
228
+ 7,
229
+ 8,
230
+ 9,
231
+ 10,
232
+ 11,
233
+ 12,
234
+ 13,
235
+ 14,
236
+ 15,
237
+ 16,
238
+ 17,
239
+ 18,
240
+ 19,
241
+ 20,
242
+ 21,
243
+ 22,
244
+ 23,
245
+ 24,
246
+ 25,
247
+ 26,
248
+ 27,
249
+ 28,
250
+ 29,
251
+ 30,
252
+ 31,
253
+ 32,
254
+ 33,
255
+ 34,
256
+ 35,
257
+ 36,
258
+ 37,
259
+ 38,
260
+ 39,
261
+ 40,
262
+ 41,
263
+ 42,
264
+ 43,
265
+ 44,
266
+ 45,
267
+ 46,
268
+ 47,
269
+ 48,
270
+ 49,
271
+ 50,
272
+ 51,
273
+ 52,
274
+ 53,
275
+ 54,
276
+ 55,
277
+ 56,
278
+ 57,
279
+ 58,
280
+ 59,
281
+ 60,
282
+ 61,
283
+ 62,
284
+ 63,
285
+ 64,
286
+ 65,
287
+ 66,
288
+ 67,
289
+ 68,
290
+ 69,
291
+ 70,
292
+ 71,
293
+ 72,
294
+ 73,
295
+ 74,
296
+ 75,
297
+ 76,
298
+ 77,
299
+ 78,
300
+ 79,
301
+ 80,
302
+ 81,
303
+ 82,
304
+ 83,
305
+ 84,
306
+ 85,
307
+ 86,
308
+ 87,
309
+ 88,
310
+ 89,
311
+ 90,
312
+ 91,
313
+ 92,
314
+ 93,
315
+ 94,
316
+ 95,
317
+ 96,
318
+ 97,
319
+ 98,
320
+ 99,
321
+ 100,
322
+ 101,
323
+ 102,
324
+ 103,
325
+ 104,
326
+ 105,
327
+ 106,
328
+ 107,
329
+ 108,
330
+ 109,
331
+ 110,
332
+ 111,
333
+ 112,
334
+ 113,
335
+ 114,
336
+ 115,
337
+ 116,
338
+ 117,
339
+ 118,
340
+ 119,
341
+ 120,
342
+ 121,
343
+ 122,
344
+ 123,
345
+ 124,
346
+ 125,
347
+ 126,
348
+ 127,
349
+ -1
350
+ ],
351
+ "one_token_stream_for_programs": true,
352
+ "program_changes": true,
353
+ "beat_res_rest": {
354
+ "0_1": 8,
355
+ "1_2": 4,
356
+ "2_12": 2
357
+ },
358
+ "chord_maps": {
359
+ "min": [
360
+ 0,
361
+ 3,
362
+ 7
363
+ ],
364
+ "maj": [
365
+ 0,
366
+ 4,
367
+ 7
368
+ ],
369
+ "dim": [
370
+ 0,
371
+ 3,
372
+ 6
373
+ ],
374
+ "aug": [
375
+ 0,
376
+ 4,
377
+ 8
378
+ ],
379
+ "sus2": [
380
+ 0,
381
+ 2,
382
+ 7
383
+ ],
384
+ "sus4": [
385
+ 0,
386
+ 5,
387
+ 7
388
+ ],
389
+ "7dom": [
390
+ 0,
391
+ 4,
392
+ 7,
393
+ 10
394
+ ],
395
+ "7min": [
396
+ 0,
397
+ 3,
398
+ 7,
399
+ 10
400
+ ],
401
+ "7maj": [
402
+ 0,
403
+ 4,
404
+ 7,
405
+ 11
406
+ ],
407
+ "7halfdim": [
408
+ 0,
409
+ 3,
410
+ 6,
411
+ 10
412
+ ],
413
+ "7dim": [
414
+ 0,
415
+ 3,
416
+ 6,
417
+ 9
418
+ ],
419
+ "7aug": [
420
+ 0,
421
+ 4,
422
+ 8,
423
+ 11
424
+ ],
425
+ "9maj": [
426
+ 0,
427
+ 4,
428
+ 7,
429
+ 10,
430
+ 14
431
+ ],
432
+ "9min": [
433
+ 0,
434
+ 4,
435
+ 7,
436
+ 10,
437
+ 13
438
+ ]
439
+ },
440
+ "chord_tokens_with_root_note": false,
441
+ "chord_unknown": null,
442
+ "num_tempos": 32,
443
+ "tempo_range": [
444
+ 40,
445
+ 250
446
+ ],
447
+ "log_tempos": false,
448
+ "delete_equal_successive_tempo_changes": false,
449
+ "time_signature_range": {
450
+ "2": [
451
+ 2,
452
+ 3,
453
+ 4
454
+ ],
455
+ "4": [
456
+ 1,
457
+ 2,
458
+ 3,
459
+ 4,
460
+ 5,
461
+ 6,
462
+ 7
463
+ ],
464
+ "8": [
465
+ 3,
466
+ 5,
467
+ 6,
468
+ 7,
469
+ 9,
470
+ 12
471
+ ]
472
+ },
473
+ "delete_equal_successive_time_sig_changes": false,
474
+ "sustain_pedal_duration": false,
475
+ "pitch_bend_range": [
476
+ -8192,
477
+ 8191,
478
+ 32
479
+ ],
480
+ "max_pitch_interval": 16,
481
+ "pitch_intervals_max_time_dist": 1,
482
+ "drums_pitch_range": [
483
+ 27,
484
+ 88
485
+ ],
486
+ "ac_polyphony_track": false,
487
+ "ac_polyphony_bar": false,
488
+ "ac_polyphony_min": 1,
489
+ "ac_polyphony_max": 6,
490
+ "ac_pitch_class_bar": false,
491
+ "ac_note_density_track": false,
492
+ "ac_note_density_track_min": 0,
493
+ "ac_note_density_track_max": 18,
494
+ "ac_note_density_bar": false,
495
+ "ac_note_density_bar_max": 18,
496
+ "ac_note_duration_bar": false,
497
+ "ac_note_duration_track": false,
498
+ "ac_repetition_track": false,
499
+ "ac_repetition_track_num_bins": 10,
500
+ "ac_repetition_track_num_consec_bars": 4,
501
+ "additional_params": {
502
+ "max_bar_embedding": null,
503
+ "use_bar_end_tokens": false,
504
+ "add_trailing_bars": false
505
+ }
506
+ },
507
+ "tokenization": "REMI",
508
+ "miditok_version": "3.0.6.post1",
509
+ "symusic_version": "0.6.0",
510
+ "hf_tokenizers_version": "0.19.1"
511
+ }