Rewrote the entire Dataloader
Browse files- model_core/dataloader.py +149 -31
model_core/dataloader.py
CHANGED
|
@@ -1,53 +1,171 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
import torch
|
| 4 |
-
|
| 5 |
-
#Data loader
|
| 6 |
-
class DataLoader_1:
|
| 7 |
-
def __init__(self, B, T, process_rank, num_processes, split, master_process):
|
| 8 |
self.B = B
|
| 9 |
self.T = T
|
| 10 |
-
self.process_rank = process_rank
|
| 11 |
self.num_processes = num_processes
|
| 12 |
-
assert split in {'
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
data_root = "data/edu_fineweb10B"
|
| 16 |
shards = os.listdir(data_root)
|
| 17 |
shards = [s for s in shards if split in s]
|
| 18 |
shards = sorted(shards)
|
| 19 |
-
shards = [os.path.join(data_root,
|
| 20 |
self.shards = shards
|
| 21 |
-
assert len(shards)> 0, f"no
|
| 22 |
if master_process:
|
| 23 |
-
print(f"found {len(shards)} shards for split {split}")
|
| 24 |
-
self.
|
|
|
|
|
|
|
|
|
|
| 25 |
|
| 26 |
def load_tokens(self, filename):
|
| 27 |
npt = np.load(filename)
|
| 28 |
npt = npt.astype(np.int32)
|
| 29 |
ptt = torch.tensor(npt, dtype=torch.long)
|
| 30 |
return ptt
|
| 31 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
|
| 33 |
def reset(self):
|
| 34 |
-
#state, init at shard 0
|
| 35 |
self.current_shard = 0
|
| 36 |
-
self.
|
| 37 |
-
self.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
|
| 39 |
-
def next_batch(self):
|
| 40 |
-
B, T = self.B, self.T
|
| 41 |
-
buf = self.tokens[self.current_position:self.current_position + B*T+1]
|
| 42 |
-
x = (buf[:-1]).view(B,T)
|
| 43 |
-
y = (buf[1:]).view(B,T)
|
| 44 |
|
| 45 |
-
self.current_position += B * T * self.num_processes
|
| 46 |
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
|
| 53 |
|
|
|
|
| 1 |
+
class Dataloader_memT:
|
| 2 |
+
def __init__(self, B, T, num_processes, split, master_process):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
self.B = B
|
| 4 |
self.T = T
|
|
|
|
| 5 |
self.num_processes = num_processes
|
| 6 |
+
assert split in {'trai', 'val'}
|
| 7 |
+
|
| 8 |
+
data_root = 'edu_fineweb10B'
|
|
|
|
| 9 |
shards = os.listdir(data_root)
|
| 10 |
shards = [s for s in shards if split in s]
|
| 11 |
shards = sorted(shards)
|
| 12 |
+
shards = [os.path.join(data_root,s) for s in shards]
|
| 13 |
self.shards = shards
|
| 14 |
+
assert len(shards) > 0, f"no shard found for the split {split}"
|
| 15 |
if master_process:
|
| 16 |
+
print(f"found {len(shards)} shards for split {split}")
|
| 17 |
+
self.current_shard = 0
|
| 18 |
+
self.current_position = 0
|
| 19 |
+
self.tokens = 0
|
| 20 |
+
self._load_and_prepare_tokens()
|
| 21 |
|
| 22 |
def load_tokens(self, filename):
|
| 23 |
npt = np.load(filename)
|
| 24 |
npt = npt.astype(np.int32)
|
| 25 |
ptt = torch.tensor(npt, dtype=torch.long)
|
| 26 |
return ptt
|
| 27 |
+
|
| 28 |
+
def _load_and_prepare_tokens(self):
|
| 29 |
+
all_tokens_in_shard = load_tokens(self.shards[self.current_shard])
|
| 30 |
+
num_tokens_in_shard = all_tokens_in_shard.nelement()
|
| 31 |
+
tokens_per_process = num_tokens_in_shard // self.num_processes
|
| 32 |
+
start = self.process_rank * num_tokens_in_shard
|
| 33 |
+
end = start + tokens_per_process
|
| 34 |
+
|
| 35 |
+
process_tokens = all_tokens_in_shard[start:end]
|
| 36 |
+
|
| 37 |
+
num_batches_per_process = process_tokens.nelement() // (self.B * self.T)
|
| 38 |
+
|
| 39 |
+
if num_batches_per_process == 0:
|
| 40 |
+
self.tokens = torch.empty((self.B , 0), dtype=torch.long)
|
| 41 |
+
|
| 42 |
+
else:
|
| 43 |
+
num_tokens_to_use = num_batches_per_process * self.B * self.T
|
| 44 |
+
process_tokens = process_tokens[:num_tokens_to_use]
|
| 45 |
+
self.tokens = self.process_tokens.view(self.B, -1)#(B,num_batches_per_process*T)
|
| 46 |
+
|
| 47 |
+
self.current_position = 0
|
| 48 |
+
|
| 49 |
+
def next_batch():
|
| 50 |
+
B, T = self.B, self.T
|
| 51 |
+
|
| 52 |
+
if self.current_position + T + 1 > self.tokens.size(1):
|
| 53 |
+
print("called next shard")
|
| 54 |
+
self.current_shard = (self.current_shard + 1) % len(self.shards)
|
| 55 |
+
self._load_and_prepare_tokens()
|
| 56 |
+
|
| 57 |
+
if self.tokens.size(1) == 0:
|
| 58 |
+
print(f"WARNING: shard {self.current_shard} is too small for process {self.process_rank}, skipping." )
|
| 59 |
+
return self.next_batch()
|
| 60 |
+
|
| 61 |
+
current_shard_num = self.current_shard
|
| 62 |
+
|
| 63 |
+
x = self.tokens[:, self.current_position : self.current_position + T]
|
| 64 |
+
y = self.tokens[:,self.current_position + 1 : self.current_position + 1 + T]
|
| 65 |
+
|
| 66 |
+
self.current_position += T
|
| 67 |
+
return x, y, current_shard_num
|
| 68 |
|
| 69 |
def reset(self):
|
|
|
|
| 70 |
self.current_shard = 0
|
| 71 |
+
self.current_position = 0
|
| 72 |
+
self._load_and_prepare_tokens()
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
|
| 77 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
|
|
|
|
| 79 |
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
# import os
|
| 120 |
+
# import numpy as np
|
| 121 |
+
# import torch
|
| 122 |
+
|
| 123 |
+
# #Data loader
|
| 124 |
+
# class DataLoader_1:
|
| 125 |
+
# def __init__(self, B, T, process_rank, num_processes, split, master_process):
|
| 126 |
+
# self.B = B
|
| 127 |
+
# self.T = T
|
| 128 |
+
# self.process_rank = process_rank
|
| 129 |
+
# self.num_processes = num_processes
|
| 130 |
+
# assert split in {'train', 'val'}
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
# data_root = "data/edu_fineweb10B"
|
| 134 |
+
# shards = os.listdir(data_root)
|
| 135 |
+
# shards = [s for s in shards if split in s]
|
| 136 |
+
# shards = sorted(shards)
|
| 137 |
+
# shards = [os.path.join(data_root, s) for s in shards]
|
| 138 |
+
# self.shards = shards
|
| 139 |
+
# assert len(shards)> 0, f"no shards found for split {split}"
|
| 140 |
+
# if master_process:
|
| 141 |
+
# print(f"found {len(shards)} shards for split {split}")
|
| 142 |
+
# self.reset()
|
| 143 |
+
|
| 144 |
+
# def load_tokens(self, filename):
|
| 145 |
+
# npt = np.load(filename)
|
| 146 |
+
# npt = npt.astype(np.int32)
|
| 147 |
+
# ptt = torch.tensor(npt, dtype=torch.long)
|
| 148 |
+
# return ptt
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
# def reset(self):
|
| 152 |
+
# #state, init at shard 0
|
| 153 |
+
# self.current_shard = 0
|
| 154 |
+
# self.tokens = self.load_tokens(self.shards[self.current_shard])
|
| 155 |
+
# self.current_position = self.B * self.T * self.process_rank
|
| 156 |
+
|
| 157 |
+
# def next_batch(self):
|
| 158 |
+
# B, T = self.B, self.T
|
| 159 |
+
# buf = self.tokens[self.current_position:self.current_position + B*T+1]
|
| 160 |
+
# x = (buf[:-1]).view(B,T)
|
| 161 |
+
# y = (buf[1:]).view(B,T)
|
| 162 |
+
|
| 163 |
+
# self.current_position += B * T * self.num_processes
|
| 164 |
+
|
| 165 |
+
# if self.current_position + (B * T * self.num_processes + 1) > len(self.tokens):
|
| 166 |
+
# self.current_shard = (self.current_shard + 1) % len(self.shards)
|
| 167 |
+
# self.tokens = self.load_tokens(self.shards[self.current_shard])
|
| 168 |
+
# self.current_position = B * T * self.process_rank
|
| 169 |
+
# return x, y
|
| 170 |
|
| 171 |
|