Transformers
NIVED47 commited on
Commit
ae4e5e4
·
verified ·
1 Parent(s): fec209c

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +69 -69
vocab.json CHANGED
@@ -1,74 +1,74 @@
1
  {
2
- "\"": 43,
3
- "'": 19,
4
- ",": 16,
5
- "-": 31,
6
- "?": 60,
7
  "[PAD]": 71,
8
  "[UNK]": 70,
9
- "|": 45,
10
- " ": 30,
11
  "ँ": 67,
12
- "ं": 27,
13
- "ः": 26,
14
- "अ": 36,
15
- "आ": 8,
16
- "इ": 5,
17
- "ई": 50,
18
- "उ": 11,
19
- "ऊ": 34,
20
- "ऋ": 59,
21
- "ए": 0,
22
- "ऐ": 47,
23
- "ओ": 54,
24
- "औ": 48,
25
- "क": 12,
26
- "ख": 37,
27
- "ग": 42,
28
- "घ": 38,
29
- "ङ": 55,
30
- "च": 7,
31
- "छ": 21,
32
- "ज": 39,
33
- "झ": 18,
34
- "ञ": 10,
35
- "ट": 57,
36
- "ठ": 17,
37
- "ड": 22,
38
- "ढ": 69,
39
- "ण": 32,
40
- "त": 4,
41
- "थ": 6,
42
- "द": 35,
43
- "ध": 66,
44
- "न": 53,
45
- "प": 46,
46
- "फ": 58,
47
- "ब": 14,
48
- "भ": 1,
49
- "म": 63,
50
- "य": 56,
51
- "र": 24,
52
- "ल": 29,
53
- "ळ": 28,
54
- "व": 51,
55
- "श": 49,
56
- "ष": 13,
57
- "स": 44,
58
- "ह": 33,
59
- "ऽ": 3,
60
- "ा": 62,
61
- "ि": 15,
62
- "ी": 9,
63
- "ु": 61,
64
- "ू": 25,
65
- "ृ": 23,
66
- "ॄ": 2,
67
- "े": 65,
68
- "ै": 41,
69
- "ो": 20,
70
- "ौ": 64,
71
- "्": 52,
72
- "ॠ": 68,
73
- "।": 40
74
  }
 
1
  {
2
+ "\"": 10,
3
+ "'": 32,
4
+ ",": 65,
5
+ "-": 53,
6
+ "?": 45,
7
  "[PAD]": 71,
8
  "[UNK]": 70,
9
+ "|": 6,
10
+ " ": 62,
11
  "ँ": 67,
12
+ "ं": 66,
13
+ "ः": 51,
14
+ "अ": 35,
15
+ "आ": 34,
16
+ "इ": 44,
17
+ "ई": 16,
18
+ "उ": 55,
19
+ "ऊ": 30,
20
+ "ऋ": 1,
21
+ "ए": 23,
22
+ "ऐ": 33,
23
+ "ओ": 9,
24
+ "औ": 28,
25
+ "क": 39,
26
+ "ख": 56,
27
+ "ग": 5,
28
+ "घ": 59,
29
+ "ङ": 50,
30
+ "च": 19,
31
+ "छ": 13,
32
+ "ज": 69,
33
+ "झ": 0,
34
+ "ञ": 48,
35
+ "ट": 41,
36
+ "ठ": 60,
37
+ "ड": 42,
38
+ "ढ": 8,
39
+ "ण": 26,
40
+ "त": 29,
41
+ "थ": 14,
42
+ "द": 64,
43
+ "ध": 3,
44
+ "न": 36,
45
+ "प": 37,
46
+ "फ": 7,
47
+ "ब": 12,
48
+ "भ": 22,
49
+ "म": 38,
50
+ "य": 63,
51
+ "र": 43,
52
+ "ल": 49,
53
+ "ळ": 52,
54
+ "व": 46,
55
+ "श": 15,
56
+ "ष": 54,
57
+ "स": 40,
58
+ "ह": 17,
59
+ "ऽ": 68,
60
+ "ा": 31,
61
+ "ि": 47,
62
+ "ी": 27,
63
+ "ु": 58,
64
+ "ू": 18,
65
+ "ृ": 61,
66
+ "ॄ": 24,
67
+ "े": 4,
68
+ "ै": 20,
69
+ "ो": 11,
70
+ "ौ": 21,
71
+ "्": 25,
72
+ "ॠ": 57,
73
+ "।": 2
74
  }