File size: 1,508 Bytes
ca0bb3f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | import sys
import random
import torch
from torch.utils.data import IterableDataset
import sentencepiece as spm
class StreamDataset(IterableDataset):
def __init__(self, text, sp_model_path, chunk_size, stride=None):
self.sp = spm.SentencePieceProcessor()
self.sp.Load(sp_model_path)
self.chunk_size = chunk_size
self.stride = stride or chunk_size // 2
ids = self.sp.Encode(text)
self.ids = torch.tensor(ids, dtype=torch.long)
def __iter__(self):
data = self.ids
n = len(data) - self.chunk_size - 1
indices = list(range(0, n, self.stride))
random.shuffle(indices)
for i in indices:
x = data[i : i + self.chunk_size]
y = data[i + 1 : i + self.chunk_size + 1]
yield x, y
def vocab_size(self):
return self.sp.GetPieceSize()
def decode(self, ids):
return self.sp.Decode(ids)
def load_text(path_or_stdin):
if path_or_stdin == "-":
return sys.stdin.read()
with open(path_or_stdin, "r", encoding="utf-8", errors="ignore") as f:
return f.read()
def build_vocab(text_path, sp_model_prefix, vocab_size, input_sentence_size=2000000):
spm.SentencePieceTrainer.Train(
input=text_path,
model_prefix=sp_model_prefix,
vocab_size=vocab_size,
model_type="bpe",
character_coverage=1.0,
input_sentence_size=input_sentence_size,
shuffle_input_sentence=True,
pad_id=3,
)
|