Delete main/src/sequence_rebatch.py with huggingface_hub
Browse files- main/src/sequence_rebatch.py +0 -163
main/src/sequence_rebatch.py
DELETED
|
@@ -1,163 +0,0 @@
|
|
| 1 |
-
from asyncio.log import logger
|
| 2 |
-
import time
|
| 3 |
-
from utils import len_batch
|
| 4 |
-
from sequence import Sequence, SequenceIterator, default_collate_list_of_dicts
|
| 5 |
-
import numpy as np
|
| 6 |
-
import torch
|
| 7 |
-
import collections
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
def split_in_2_batches(batch: collections.MutableMapping, first_batch_size: int):
|
| 11 |
-
"""
|
| 12 |
-
Split a single batch into 2 batches. The first batch will have a fixed size.
|
| 13 |
-
|
| 14 |
-
If there is not enough sample to split the batch, return (batch, None)
|
| 15 |
-
|
| 16 |
-
Args:
|
| 17 |
-
batch: the batch to split
|
| 18 |
-
first_batch_size: the batch size of the first batch. The remaining samples will be in the second batch
|
| 19 |
-
|
| 20 |
-
Returns:
|
| 21 |
-
a tuple (first batch, second batch)
|
| 22 |
-
"""
|
| 23 |
-
batch_size = len_batch(batch)
|
| 24 |
-
if batch_size <= first_batch_size:
|
| 25 |
-
return batch, None
|
| 26 |
-
|
| 27 |
-
# actually, split!
|
| 28 |
-
batch_1 = type(batch)()
|
| 29 |
-
batch_2 = type(batch)()
|
| 30 |
-
|
| 31 |
-
for name, value in batch.items():
|
| 32 |
-
if isinstance(value, (np.ndarray, torch.Tensor, list)):
|
| 33 |
-
# split the array/list
|
| 34 |
-
batch_1[name] = value[:first_batch_size]
|
| 35 |
-
batch_2[name] = value[first_batch_size:]
|
| 36 |
-
else:
|
| 37 |
-
# for other types, simply duplicate
|
| 38 |
-
batch_1[name] = value
|
| 39 |
-
batch_2[name] = value
|
| 40 |
-
|
| 41 |
-
return batch_1, batch_2
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
class RebatchStatistics:
|
| 45 |
-
def __init__(self):
|
| 46 |
-
self.reset()
|
| 47 |
-
|
| 48 |
-
def reset(self):
|
| 49 |
-
self.nb_batches_input = 0
|
| 50 |
-
self.nb_batches_output = 0
|
| 51 |
-
self.waiting_time_batches = 0
|
| 52 |
-
self.waiting_time_rebatch = 0
|
| 53 |
-
self.waiting_time_split = 0
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
class SequenceReBatch(Sequence, SequenceIterator):
|
| 57 |
-
"""
|
| 58 |
-
This sequence will normalize the batch size of an underlying sequence
|
| 59 |
-
|
| 60 |
-
If the underlying sequence batch is too large, it will be split in multiple batches. Conversely,
|
| 61 |
-
if the size of the batch is too small, it several batches will be merged until we reach the expected batch size.
|
| 62 |
-
"""
|
| 63 |
-
def __init__(self, source_split, batch_size, discard_batch_not_full=False, collate_fn=default_collate_list_of_dicts):
|
| 64 |
-
"""
|
| 65 |
-
Normalize a sequence to identical batch size given an input sequence with varying batch size
|
| 66 |
-
|
| 67 |
-
Args:
|
| 68 |
-
source_split: the underlying sequence to normalize
|
| 69 |
-
batch_size: the size of the batches created by this sequence
|
| 70 |
-
discard_batch_not_full: if True, the last batch will be discarded if not full
|
| 71 |
-
collate_fn: function to merge multiple batches
|
| 72 |
-
"""
|
| 73 |
-
super().__init__(source_split)
|
| 74 |
-
|
| 75 |
-
assert batch_size > 0
|
| 76 |
-
assert isinstance(source_split, Sequence), '`source_split` must be a `Sequence`'
|
| 77 |
-
self.source_split = source_split
|
| 78 |
-
self.batch_size = batch_size
|
| 79 |
-
self.discard_batch_not_full = discard_batch_not_full
|
| 80 |
-
self.iter_source = None
|
| 81 |
-
self.iter_overflow = None
|
| 82 |
-
self.collate_fn = collate_fn
|
| 83 |
-
self.statistics = RebatchStatistics()
|
| 84 |
-
|
| 85 |
-
def subsample(self, nb_samples):
|
| 86 |
-
subsampled_source = self.source_split.subsample(nb_samples)
|
| 87 |
-
return SequenceReBatch(subsampled_source, batch_size=self.batch_size, discard_batch_not_full=self.discard_batch_not_full, collate_fn=self.collate_fn)
|
| 88 |
-
|
| 89 |
-
def subsample_uids(self, uids, uids_name, new_sampler=None):
|
| 90 |
-
subsampled_source = self.source_split.subsample_uids(uids, uids_name, new_sampler)
|
| 91 |
-
return SequenceReBatch(subsampled_source, batch_size=self.batch_size, discard_batch_not_full=self.discard_batch_not_full, collate_fn=self.collate_fn)
|
| 92 |
-
|
| 93 |
-
def __next__(self):
|
| 94 |
-
batches = []
|
| 95 |
-
total_nb_samples = 0
|
| 96 |
-
try:
|
| 97 |
-
while True:
|
| 98 |
-
if self.iter_overflow is not None:
|
| 99 |
-
# handles the samples that had previously overflown
|
| 100 |
-
batch = self.iter_overflow
|
| 101 |
-
self.iter_overflow = None
|
| 102 |
-
else:
|
| 103 |
-
# if not, get the next batch
|
| 104 |
-
time_wait_start = time.perf_counter()
|
| 105 |
-
batch = self.iter_source.__next__()
|
| 106 |
-
time_wait_end = time.perf_counter()
|
| 107 |
-
self.statistics.waiting_time_batches += time_wait_end - time_wait_start
|
| 108 |
-
self.statistics.nb_batches_input += 1
|
| 109 |
-
|
| 110 |
-
if batch is None or len(batch) == 0:
|
| 111 |
-
# for some reason, the batch is empty
|
| 112 |
-
# get a new one!
|
| 113 |
-
continue
|
| 114 |
-
|
| 115 |
-
nb_samples = len_batch(batch)
|
| 116 |
-
if total_nb_samples + nb_samples == self.batch_size:
|
| 117 |
-
# here we are good!
|
| 118 |
-
batches.append(batch)
|
| 119 |
-
break
|
| 120 |
-
|
| 121 |
-
if total_nb_samples + nb_samples > self.batch_size:
|
| 122 |
-
# too many samples, split the batch and keep the extra samples in the overflow
|
| 123 |
-
first_batch_size = self.batch_size - total_nb_samples
|
| 124 |
-
|
| 125 |
-
time_start = time.perf_counter()
|
| 126 |
-
first_batch, overflow_batch = split_in_2_batches(batch, first_batch_size)
|
| 127 |
-
time_end = time.perf_counter()
|
| 128 |
-
self.statistics.waiting_time_split += time_end - time_start
|
| 129 |
-
|
| 130 |
-
batches.append(first_batch)
|
| 131 |
-
self.iter_overflow = overflow_batch
|
| 132 |
-
break
|
| 133 |
-
|
| 134 |
-
# else keep accumulating until we have enough samples
|
| 135 |
-
total_nb_samples += nb_samples
|
| 136 |
-
batches.append(batch)
|
| 137 |
-
|
| 138 |
-
except StopIteration:
|
| 139 |
-
if len(batches) == 0 or (len(batches) != self.batch_size and self.discard_batch_not_full):
|
| 140 |
-
# end the sequence
|
| 141 |
-
overhead_time = self.statistics.waiting_time_batches + self.statistics.waiting_time_rebatch + self.statistics.waiting_time_split
|
| 142 |
-
logger.info(f'SequenceReBatch={self}, overhead={overhead_time}, waiting_time_batches={self.statistics.waiting_time_batches}, waiting_time_rebatch={self.statistics.waiting_time_rebatch}, waiting_time_split={self.statistics.waiting_time_split}')
|
| 143 |
-
raise StopIteration()
|
| 144 |
-
|
| 145 |
-
if self.collate_fn is not None:
|
| 146 |
-
# finally make a batch
|
| 147 |
-
time_start = time.perf_counter()
|
| 148 |
-
batches = self.collate_fn(batches)
|
| 149 |
-
time_end = time.perf_counter()
|
| 150 |
-
self.statistics.waiting_time_rebatch += time_end - time_start
|
| 151 |
-
|
| 152 |
-
self.statistics.nb_batches_output += 1
|
| 153 |
-
return batches
|
| 154 |
-
|
| 155 |
-
def __iter__(self):
|
| 156 |
-
self.statistics.reset()
|
| 157 |
-
self.iter_source = self.source_split.__iter__()
|
| 158 |
-
return self
|
| 159 |
-
|
| 160 |
-
def close(self):
|
| 161 |
-
if self.source_split is not None:
|
| 162 |
-
self.source_split.close()
|
| 163 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|