Download data.py from Adam25613/AMAIA: direct link, hf CLI and curl.
- Browser
- Download file 1.51 kB
-
https://huggingface.co/spaces/Adam25613/AMAIA/resolve/main/data.py
- Command line
-
hf download hf://spaces/Adam25613/AMAIA/data.py
-
curl -L -o data.py https://huggingface.co/spaces/Adam25613/AMAIA/resolve/main/data.py
1.51 kB
| import sys | |
| import random | |
| import torch | |
| from torch.utils.data import IterableDataset | |
| import sentencepiece as spm | |
| class StreamDataset(IterableDataset): | |
| def __init__(self, text, sp_model_path, chunk_size, stride=None): | |
| self.sp = spm.SentencePieceProcessor() | |
| self.sp.Load(sp_model_path) | |
| self.chunk_size = chunk_size | |
| self.stride = stride or chunk_size // 2 | |
| ids = self.sp.Encode(text) | |
| self.ids = torch.tensor(ids, dtype=torch.long) | |
| def __iter__(self): | |
| data = self.ids | |
| n = len(data) - self.chunk_size - 1 | |
| indices = list(range(0, n, self.stride)) | |
| random.shuffle(indices) | |
| for i in indices: | |
| x = data[i : i + self.chunk_size] | |
| y = data[i + 1 : i + self.chunk_size + 1] | |
| yield x, y | |
| def vocab_size(self): | |
| return self.sp.GetPieceSize() | |
| def decode(self, ids): | |
| return self.sp.Decode(ids) | |
| def load_text(path_or_stdin): | |
| if path_or_stdin == "-": | |
| return sys.stdin.read() | |
| with open(path_or_stdin, "r", encoding="utf-8", errors="ignore") as f: | |
| return f.read() | |
| def build_vocab(text_path, sp_model_prefix, vocab_size, input_sentence_size=2000000): | |
| spm.SentencePieceTrainer.Train( | |
| input=text_path, | |
| model_prefix=sp_model_prefix, | |
| vocab_size=vocab_size, | |
| model_type="bpe", | |
| character_coverage=1.0, | |
| input_sentence_size=input_sentence_size, | |
| shuffle_input_sentence=True, | |
| pad_id=3, | |
| ) | |