add tokenizer

Browse files

Files changed (1) hide show

vocab.json +1 -1

vocab.json CHANGED Viewed

@@ -1 +1 @@

- {"ﺃ": 0, "ذ": 2, "ض": 3, "د": 4, "ڨ": 5, "س": 6, "ق": 7, "ظ": 8, "خ": 9, "ة": 10, "ی": 11, "ف": 12, "ي": 13, "ب": 14, "ل": 15, "ن": 16, "ج": 17, "ط": 18, "ص": 19, "ك": 20, "ه": 21, "غ": 22, "ء": 23, "ھ": 24, "ث": 25, "ى": 26, "ا": 27, "ش": 28, "ع": ~~29, "ر":~~ 30, "ک": 31, "ﻻ": 32, "ئ": 33, "م": 34, "و": 35, "ز": 36, "ؤ": 37, "ح": 38, "ت": 39, "|": 1, "[UNK]": 40, "[PAD]": 41}

+ {"ف": 0, "ذ": 1, "ر": 2, "ڨ": 3, "ط": 4, "ﺃ": 5, "ھ": 6, "ض": 7, "ح": 8, "ص": 9, "ك": 10, "ه": 11, "خ": 12, "س": 13, "ا": 14, "ء": 15, "ى": 16, "ک": 17, "ز": 18, "م": 19, "ن": 20, "و": 21, "د": 22, "ع": 23, "غ": 24, "ئ": 25, "ی": 26, "ب": 27, "ق": 28, "ت": 30, "ظ": 31, "ث": 32, "ش": 33, "ج": 34, "ؤ": 35, "ة": 36, "ل": 37, "ي": 38, "ﻻ": 39, "|": 29, "[UNK]": 40, "[PAD]": 41}