tchbcb
/

lalmodel-distributed

tchbcb commited on Aug 13

Commit

455976a

verified ·

1 Parent(s): 7b46abe

Upload prepare_wiki_data.py with huggingface_hub

Files changed (1) hide show

prepare_wiki_data.py CHANGED Viewed

@@ -121,7 +121,7 @@ def tokenize_with_bpe(texts, tokenizer_model):
         subprocess.run([sys.executable, "-m", "pip", "install", "-q", "sentencepiece"], check=True)
         import sentencepiece as spm
-    sp = spm.SentenceProcessor()
     sp.Load(tokenizer_model)
     all_samples = []

         subprocess.run([sys.executable, "-m", "pip", "install", "-q", "sentencepiece"], check=True)
         import sentencepiece as spm
+    sp = spm.SentencePieceProcessor()
     sp.Load(tokenizer_model)
     all_samples = []