Text Generation
English
transformer
abhinavv3 commited on
Commit
2199e22
·
1 Parent(s): f6d6286

Rewrote the entire Dataloader

Browse files
Files changed (1) hide show
  1. model_core/dataloader.py +149 -31
model_core/dataloader.py CHANGED
@@ -1,53 +1,171 @@
1
- import os
2
- import numpy as np
3
- import torch
4
-
5
- #Data loader
6
- class DataLoader_1:
7
- def __init__(self, B, T, process_rank, num_processes, split, master_process):
8
  self.B = B
9
  self.T = T
10
- self.process_rank = process_rank
11
  self.num_processes = num_processes
12
- assert split in {'train', 'val'}
13
-
14
-
15
- data_root = "data/edu_fineweb10B"
16
  shards = os.listdir(data_root)
17
  shards = [s for s in shards if split in s]
18
  shards = sorted(shards)
19
- shards = [os.path.join(data_root, s) for s in shards]
20
  self.shards = shards
21
- assert len(shards)> 0, f"no shards found for split {split}"
22
  if master_process:
23
- print(f"found {len(shards)} shards for split {split}")
24
- self.reset()
 
 
 
25
 
26
  def load_tokens(self, filename):
27
  npt = np.load(filename)
28
  npt = npt.astype(np.int32)
29
  ptt = torch.tensor(npt, dtype=torch.long)
30
  return ptt
31
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
32
 
33
  def reset(self):
34
- #state, init at shard 0
35
  self.current_shard = 0
36
- self.tokens = self.load_tokens(self.shards[self.current_shard])
37
- self.current_position = self.B * self.T * self.process_rank
 
 
 
 
38
 
39
- def next_batch(self):
40
- B, T = self.B, self.T
41
- buf = self.tokens[self.current_position:self.current_position + B*T+1]
42
- x = (buf[:-1]).view(B,T)
43
- y = (buf[1:]).view(B,T)
44
 
45
- self.current_position += B * T * self.num_processes
46
 
47
- if self.current_position + (B * T * self.num_processes + 1) > len(self.tokens):
48
- self.current_shard = (self.current_shard + 1) % len(self.shards)
49
- self.tokens = self.load_tokens(self.shards[self.current_shard])
50
- self.current_position = B * T * self.process_rank
51
- return x, y
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
52
 
53
 
 
1
+ class Dataloader_memT:
2
+ def __init__(self, B, T, num_processes, split, master_process):
 
 
 
 
 
3
  self.B = B
4
  self.T = T
 
5
  self.num_processes = num_processes
6
+ assert split in {'trai', 'val'}
7
+
8
+ data_root = 'edu_fineweb10B'
 
9
  shards = os.listdir(data_root)
10
  shards = [s for s in shards if split in s]
11
  shards = sorted(shards)
12
+ shards = [os.path.join(data_root,s) for s in shards]
13
  self.shards = shards
14
+ assert len(shards) > 0, f"no shard found for the split {split}"
15
  if master_process:
16
+ print(f"found {len(shards)} shards for split {split}")
17
+ self.current_shard = 0
18
+ self.current_position = 0
19
+ self.tokens = 0
20
+ self._load_and_prepare_tokens()
21
 
22
  def load_tokens(self, filename):
23
  npt = np.load(filename)
24
  npt = npt.astype(np.int32)
25
  ptt = torch.tensor(npt, dtype=torch.long)
26
  return ptt
27
+
28
+ def _load_and_prepare_tokens(self):
29
+ all_tokens_in_shard = load_tokens(self.shards[self.current_shard])
30
+ num_tokens_in_shard = all_tokens_in_shard.nelement()
31
+ tokens_per_process = num_tokens_in_shard // self.num_processes
32
+ start = self.process_rank * num_tokens_in_shard
33
+ end = start + tokens_per_process
34
+
35
+ process_tokens = all_tokens_in_shard[start:end]
36
+
37
+ num_batches_per_process = process_tokens.nelement() // (self.B * self.T)
38
+
39
+ if num_batches_per_process == 0:
40
+ self.tokens = torch.empty((self.B , 0), dtype=torch.long)
41
+
42
+ else:
43
+ num_tokens_to_use = num_batches_per_process * self.B * self.T
44
+ process_tokens = process_tokens[:num_tokens_to_use]
45
+ self.tokens = self.process_tokens.view(self.B, -1)#(B,num_batches_per_process*T)
46
+
47
+ self.current_position = 0
48
+
49
+ def next_batch():
50
+ B, T = self.B, self.T
51
+
52
+ if self.current_position + T + 1 > self.tokens.size(1):
53
+ print("called next shard")
54
+ self.current_shard = (self.current_shard + 1) % len(self.shards)
55
+ self._load_and_prepare_tokens()
56
+
57
+ if self.tokens.size(1) == 0:
58
+ print(f"WARNING: shard {self.current_shard} is too small for process {self.process_rank}, skipping." )
59
+ return self.next_batch()
60
+
61
+ current_shard_num = self.current_shard
62
+
63
+ x = self.tokens[:, self.current_position : self.current_position + T]
64
+ y = self.tokens[:,self.current_position + 1 : self.current_position + 1 + T]
65
+
66
+ self.current_position += T
67
+ return x, y, current_shard_num
68
 
69
  def reset(self):
 
70
  self.current_shard = 0
71
+ self.current_position = 0
72
+ self._load_and_prepare_tokens()
73
+
74
+
75
+
76
+
77
 
 
 
 
 
 
78
 
 
79
 
80
+
81
+
82
+
83
+
84
+
85
+
86
+
87
+
88
+
89
+
90
+
91
+
92
+
93
+
94
+
95
+
96
+
97
+
98
+
99
+
100
+
101
+
102
+
103
+
104
+
105
+
106
+
107
+
108
+
109
+
110
+
111
+
112
+
113
+
114
+
115
+
116
+
117
+
118
+
119
+ # import os
120
+ # import numpy as np
121
+ # import torch
122
+
123
+ # #Data loader
124
+ # class DataLoader_1:
125
+ # def __init__(self, B, T, process_rank, num_processes, split, master_process):
126
+ # self.B = B
127
+ # self.T = T
128
+ # self.process_rank = process_rank
129
+ # self.num_processes = num_processes
130
+ # assert split in {'train', 'val'}
131
+
132
+
133
+ # data_root = "data/edu_fineweb10B"
134
+ # shards = os.listdir(data_root)
135
+ # shards = [s for s in shards if split in s]
136
+ # shards = sorted(shards)
137
+ # shards = [os.path.join(data_root, s) for s in shards]
138
+ # self.shards = shards
139
+ # assert len(shards)> 0, f"no shards found for split {split}"
140
+ # if master_process:
141
+ # print(f"found {len(shards)} shards for split {split}")
142
+ # self.reset()
143
+
144
+ # def load_tokens(self, filename):
145
+ # npt = np.load(filename)
146
+ # npt = npt.astype(np.int32)
147
+ # ptt = torch.tensor(npt, dtype=torch.long)
148
+ # return ptt
149
+
150
+
151
+ # def reset(self):
152
+ # #state, init at shard 0
153
+ # self.current_shard = 0
154
+ # self.tokens = self.load_tokens(self.shards[self.current_shard])
155
+ # self.current_position = self.B * self.T * self.process_rank
156
+
157
+ # def next_batch(self):
158
+ # B, T = self.B, self.T
159
+ # buf = self.tokens[self.current_position:self.current_position + B*T+1]
160
+ # x = (buf[:-1]).view(B,T)
161
+ # y = (buf[1:]).view(B,T)
162
+
163
+ # self.current_position += B * T * self.num_processes
164
+
165
+ # if self.current_position + (B * T * self.num_processes + 1) > len(self.tokens):
166
+ # self.current_shard = (self.current_shard + 1) % len(self.shards)
167
+ # self.tokens = self.load_tokens(self.shards[self.current_shard])
168
+ # self.current_position = B * T * self.process_rank
169
+ # return x, y
170
 
171