behroz commited on
Commit
65cb4a2
·
1 Parent(s): a0f64c4

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"ح": 0, "آ": 1, "ف": 2, "م": 3, "ھ": 4, "ہ": 5, "گ": 6, "ق": 7, "پ": 8, "د": 9, "ت": 11, "غ": 12, "ں": 13, "و": 14, "ل": 15, "ے": 16, "ک": 17, "ر": 18, "س": 19, "ش": 20, "ٹ": 21, "ی": 22, "ا": 23, "ڈ": 24, "ن": 25, "|": 10, "[UNK]": 26, "[PAD]": 27}
 
1
+ {"ہ": 0, "د": 1, "ی": 2, "ف": 3, "آ": 4, "غ": 5, "ا": 6, "ے": 7, "ش": 8, "ھ": 9, "م": 10, "ٹ": 11, "ت": 12, "و": 13, "ل": 14, "گ": 15, "ک": 16, "ق": 17, "س": 18, "ر": 20, "ں": 21, "ح": 22, "پ": 23, "ن": 24, "ڈ": 25, "|": 19, "[UNK]": 26, "[PAD]": 27}