-
add: 4k simplestories
-
add: mdm mask spec token
-
remove: mdm mask
-
add: mdm mask in special tokens
-
add: simplestories 8k tokenizer config
-
add: 8k vocab
-
remove: pad token
-
add: pad token
-
remove: pad token
-
add: pad token
-
refactor: remove pad token
-
add: pad special token
-
exp: remove bos token
-
clean: extra comma in json
-
fix: remove eos from vocab
-
add: tokenizer config
-
initial commit