File size: 1,508 Bytes
ca0bb3f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
import sys
import random
import torch
from torch.utils.data import IterableDataset
import sentencepiece as spm


class StreamDataset(IterableDataset):
    def __init__(self, text, sp_model_path, chunk_size, stride=None):
        self.sp = spm.SentencePieceProcessor()
        self.sp.Load(sp_model_path)
        self.chunk_size = chunk_size
        self.stride = stride or chunk_size // 2
        ids = self.sp.Encode(text)
        self.ids = torch.tensor(ids, dtype=torch.long)

    def __iter__(self):
        data = self.ids
        n = len(data) - self.chunk_size - 1
        indices = list(range(0, n, self.stride))
        random.shuffle(indices)
        for i in indices:
            x = data[i : i + self.chunk_size]
            y = data[i + 1 : i + self.chunk_size + 1]
            yield x, y

    def vocab_size(self):
        return self.sp.GetPieceSize()

    def decode(self, ids):
        return self.sp.Decode(ids)


def load_text(path_or_stdin):
    if path_or_stdin == "-":
        return sys.stdin.read()
    with open(path_or_stdin, "r", encoding="utf-8", errors="ignore") as f:
        return f.read()


def build_vocab(text_path, sp_model_prefix, vocab_size, input_sentence_size=2000000):
    spm.SentencePieceTrainer.Train(
        input=text_path,
        model_prefix=sp_model_prefix,
        vocab_size=vocab_size,
        model_type="bpe",
        character_coverage=1.0,
        input_sentence_size=input_sentence_size,
        shuffle_input_sentence=True,
        pad_id=3,
    )