fyrizi commited on
Commit
39d8a65
·
verified ·
1 Parent(s): ce4713c

Delete main/src/sequence_rebatch.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. main/src/sequence_rebatch.py +0 -163
main/src/sequence_rebatch.py DELETED
@@ -1,163 +0,0 @@
1
- from asyncio.log import logger
2
- import time
3
- from utils import len_batch
4
- from sequence import Sequence, SequenceIterator, default_collate_list_of_dicts
5
- import numpy as np
6
- import torch
7
- import collections
8
-
9
-
10
- def split_in_2_batches(batch: collections.MutableMapping, first_batch_size: int):
11
- """
12
- Split a single batch into 2 batches. The first batch will have a fixed size.
13
-
14
- If there is not enough sample to split the batch, return (batch, None)
15
-
16
- Args:
17
- batch: the batch to split
18
- first_batch_size: the batch size of the first batch. The remaining samples will be in the second batch
19
-
20
- Returns:
21
- a tuple (first batch, second batch)
22
- """
23
- batch_size = len_batch(batch)
24
- if batch_size <= first_batch_size:
25
- return batch, None
26
-
27
- # actually, split!
28
- batch_1 = type(batch)()
29
- batch_2 = type(batch)()
30
-
31
- for name, value in batch.items():
32
- if isinstance(value, (np.ndarray, torch.Tensor, list)):
33
- # split the array/list
34
- batch_1[name] = value[:first_batch_size]
35
- batch_2[name] = value[first_batch_size:]
36
- else:
37
- # for other types, simply duplicate
38
- batch_1[name] = value
39
- batch_2[name] = value
40
-
41
- return batch_1, batch_2
42
-
43
-
44
- class RebatchStatistics:
45
- def __init__(self):
46
- self.reset()
47
-
48
- def reset(self):
49
- self.nb_batches_input = 0
50
- self.nb_batches_output = 0
51
- self.waiting_time_batches = 0
52
- self.waiting_time_rebatch = 0
53
- self.waiting_time_split = 0
54
-
55
-
56
- class SequenceReBatch(Sequence, SequenceIterator):
57
- """
58
- This sequence will normalize the batch size of an underlying sequence
59
-
60
- If the underlying sequence batch is too large, it will be split in multiple batches. Conversely,
61
- if the size of the batch is too small, it several batches will be merged until we reach the expected batch size.
62
- """
63
- def __init__(self, source_split, batch_size, discard_batch_not_full=False, collate_fn=default_collate_list_of_dicts):
64
- """
65
- Normalize a sequence to identical batch size given an input sequence with varying batch size
66
-
67
- Args:
68
- source_split: the underlying sequence to normalize
69
- batch_size: the size of the batches created by this sequence
70
- discard_batch_not_full: if True, the last batch will be discarded if not full
71
- collate_fn: function to merge multiple batches
72
- """
73
- super().__init__(source_split)
74
-
75
- assert batch_size > 0
76
- assert isinstance(source_split, Sequence), '`source_split` must be a `Sequence`'
77
- self.source_split = source_split
78
- self.batch_size = batch_size
79
- self.discard_batch_not_full = discard_batch_not_full
80
- self.iter_source = None
81
- self.iter_overflow = None
82
- self.collate_fn = collate_fn
83
- self.statistics = RebatchStatistics()
84
-
85
- def subsample(self, nb_samples):
86
- subsampled_source = self.source_split.subsample(nb_samples)
87
- return SequenceReBatch(subsampled_source, batch_size=self.batch_size, discard_batch_not_full=self.discard_batch_not_full, collate_fn=self.collate_fn)
88
-
89
- def subsample_uids(self, uids, uids_name, new_sampler=None):
90
- subsampled_source = self.source_split.subsample_uids(uids, uids_name, new_sampler)
91
- return SequenceReBatch(subsampled_source, batch_size=self.batch_size, discard_batch_not_full=self.discard_batch_not_full, collate_fn=self.collate_fn)
92
-
93
- def __next__(self):
94
- batches = []
95
- total_nb_samples = 0
96
- try:
97
- while True:
98
- if self.iter_overflow is not None:
99
- # handles the samples that had previously overflown
100
- batch = self.iter_overflow
101
- self.iter_overflow = None
102
- else:
103
- # if not, get the next batch
104
- time_wait_start = time.perf_counter()
105
- batch = self.iter_source.__next__()
106
- time_wait_end = time.perf_counter()
107
- self.statistics.waiting_time_batches += time_wait_end - time_wait_start
108
- self.statistics.nb_batches_input += 1
109
-
110
- if batch is None or len(batch) == 0:
111
- # for some reason, the batch is empty
112
- # get a new one!
113
- continue
114
-
115
- nb_samples = len_batch(batch)
116
- if total_nb_samples + nb_samples == self.batch_size:
117
- # here we are good!
118
- batches.append(batch)
119
- break
120
-
121
- if total_nb_samples + nb_samples > self.batch_size:
122
- # too many samples, split the batch and keep the extra samples in the overflow
123
- first_batch_size = self.batch_size - total_nb_samples
124
-
125
- time_start = time.perf_counter()
126
- first_batch, overflow_batch = split_in_2_batches(batch, first_batch_size)
127
- time_end = time.perf_counter()
128
- self.statistics.waiting_time_split += time_end - time_start
129
-
130
- batches.append(first_batch)
131
- self.iter_overflow = overflow_batch
132
- break
133
-
134
- # else keep accumulating until we have enough samples
135
- total_nb_samples += nb_samples
136
- batches.append(batch)
137
-
138
- except StopIteration:
139
- if len(batches) == 0 or (len(batches) != self.batch_size and self.discard_batch_not_full):
140
- # end the sequence
141
- overhead_time = self.statistics.waiting_time_batches + self.statistics.waiting_time_rebatch + self.statistics.waiting_time_split
142
- logger.info(f'SequenceReBatch={self}, overhead={overhead_time}, waiting_time_batches={self.statistics.waiting_time_batches}, waiting_time_rebatch={self.statistics.waiting_time_rebatch}, waiting_time_split={self.statistics.waiting_time_split}')
143
- raise StopIteration()
144
-
145
- if self.collate_fn is not None:
146
- # finally make a batch
147
- time_start = time.perf_counter()
148
- batches = self.collate_fn(batches)
149
- time_end = time.perf_counter()
150
- self.statistics.waiting_time_rebatch += time_end - time_start
151
-
152
- self.statistics.nb_batches_output += 1
153
- return batches
154
-
155
- def __iter__(self):
156
- self.statistics.reset()
157
- self.iter_source = self.source_split.__iter__()
158
- return self
159
-
160
- def close(self):
161
- if self.source_split is not None:
162
- self.source_split.close()
163
-