Sjdan commited on
Commit
d168e9d
·
1 Parent(s): 4d9c529

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"h": 0, "f": 1, "w": 2, "s": 3, "a": 4, "t": 5, "k": 6, "d": 7, "u": 8, "n": 9, "m": 10, "g": 12, "c": 13, "b": 14, "o": 15, "r": 16, "i": 17, "p": 18, "y": 19, "e": 20, "l": 21, "|": 11, "[UNK]": 22, "[PAD]": 23}
 
1
+ {"t": 0, "o": 1, "i": 2, "c": 3, "s": 4, "n": 5, "y": 6, "m": 7, "g": 8, "f": 9, "d": 10, "h": 11, "w": 12, "p": 13, "r": 14, "u": 15, "b": 16, "a": 17, "l": 19, "e": 20, "k": 21, "|": 18, "[UNK]": 22, "[PAD]": 23}