import sys import random import torch from torch.utils.data import IterableDataset import sentencepiece as spm class StreamDataset(IterableDataset): def __init__(self, text, sp_model_path, chunk_size, stride=None): self.sp = spm.SentencePieceProcessor() self.sp.Load(sp_model_path) self.chunk_size = chunk_size self.stride = stride or chunk_size // 2 ids = self.sp.Encode(text) self.ids = torch.tensor(ids, dtype=torch.long) def __iter__(self): data = self.ids n = len(data) - self.chunk_size - 1 indices = list(range(0, n, self.stride)) random.shuffle(indices) for i in indices: x = data[i : i + self.chunk_size] y = data[i + 1 : i + self.chunk_size + 1] yield x, y def vocab_size(self): return self.sp.GetPieceSize() def decode(self, ids): return self.sp.Decode(ids) def load_text(path_or_stdin): if path_or_stdin == "-": return sys.stdin.read() with open(path_or_stdin, "r", encoding="utf-8", errors="ignore") as f: return f.read() def build_vocab(text_path, sp_model_prefix, vocab_size, input_sentence_size=2000000): spm.SentencePieceTrainer.Train( input=text_path, model_prefix=sp_model_prefix, vocab_size=vocab_size, model_type="bpe", character_coverage=1.0, input_sentence_size=input_sentence_size, shuffle_input_sentence=True, pad_id=3, )