Tarush-AI commited on
Commit
8e4b36a
·
verified ·
1 Parent(s): 9d9a4f1

Upload tokenizer.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer.py +2 -2
tokenizer.py CHANGED
@@ -23,10 +23,10 @@ class Tokenizer:
23
  self.sp.load(self.path)
24
 
25
  def encode(self, text):
26
- return self.sp.encode(text)
27
 
28
  def decode(self, ids):
29
- return self.sp.decode(ids)
30
 
31
 
32
  def test(self, file):
 
23
  self.sp.load(self.path)
24
 
25
  def encode(self, text):
26
+ return self.sp.EncodeAsIds(text)
27
 
28
  def decode(self, ids):
29
+ return self.sp.DecodeIds(ids)
30
 
31
 
32
  def test(self, file):