fyrizi commited on
Commit
f0830c5
·
verified ·
1 Parent(s): c4e898c

Delete main/src/sequence_async_reservoir.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. main/src/sequence_async_reservoir.py +0 -278
main/src/sequence_async_reservoir.py DELETED
@@ -1,278 +0,0 @@
1
- from job_executor2 import JobExecutor2
2
- import sequence
3
-
4
- import collections
5
- from queue import Empty
6
- import copy
7
- import time
8
-
9
-
10
- class Performance:
11
- def __init__(self):
12
- self.nb_times = 0
13
- self.total_time_sec = 0.0
14
-
15
- def add(self, time_elapsed):
16
- self.nb_times += 1
17
- self.total_time_sec += time_elapsed
18
-
19
- def get_average_time(self):
20
- if self.nb_times == 0:
21
- return 0
22
-
23
- return self.total_time_sec / self.nb_times
24
-
25
-
26
- class SequenceAsyncReservoir(sequence.Sequence):
27
- """
28
- This sequence will asynchronously process data and keep a reserve of loaded samples
29
-
30
- The idea is to have long loading processes work in the background while still using as efficiently as possible
31
- the data that is currently loaded. The data is slowly being replaced by freshly loaded data over time.
32
-
33
- Jobs are started and results retrieved at the beginning of each epoch
34
-
35
- This sequence can be interrupted (e.g., after a certain number of batches have been returned). When the sequence
36
- is restarted, the reservoir will not be emptied.
37
- """
38
- def __init__(
39
- self,
40
- source_split,
41
- max_reservoir_samples,
42
- function_to_run,
43
- *,
44
- min_reservoir_samples=1,
45
- nb_workers=1,
46
- max_jobs_at_once=None,
47
- reservoir_sampler=None,
48
- collate_fn=sequence.remove_nested_list,
49
- maximum_number_of_samples_per_epoch=None,
50
- max_reservoir_replacement_size=None):
51
- """
52
- Args:
53
- source_split: the source split to iterate
54
- max_reservoir_samples: the maximum number of samples of the reservoir
55
- function_to_run: the function to run asynchronously
56
- min_reservoir_samples: the minimum of samples of the reservoir needed before an output sequence
57
- can be created
58
- nb_workers: the number of workers that will process `function_to_run` to fill the reservoir. Must be >= 1
59
- max_jobs_at_once: the maximum number of jobs that can be started and stored by epoch by the workers.
60
- If 0, no limit. If None: set to the number of workers
61
- reservoir_sampler: a sampler that will be used to sample the reservoir or None for sequential sampling
62
- of the reservoir
63
- collate_fn: a function to post-process the samples into a single batch, or None if not to be collated
64
- maximum_number_of_samples_per_epoch: the maximum number of samples that will be generated per epoch.
65
- If we reach this maximum, the sequence will be interrupted
66
- max_reservoir_replacement_size: Specify the maximum number of samples replaced in the reservoir by epoch.
67
- If `None`, we will use the whole result queue. This can be useful to control explicitly how the
68
- reservoir is updated and depend less on the speed of hardware. Note that to have an effect,
69
- `max_jobs_at_once` should be greater than `max_reservoir_replacement_size`.
70
- """
71
- super().__init__(source_split)
72
- self.max_reservoir_samples = max_reservoir_samples
73
- self.min_reservoir_samples = min_reservoir_samples
74
- self.function_to_run = function_to_run
75
- self.nb_workers = nb_workers
76
- self.collate_fn = collate_fn
77
- self.max_jobs_at_once = max_jobs_at_once
78
- if max_jobs_at_once is None:
79
- self.max_jobs_at_once = nb_workers
80
- self.reservoir_sampler = reservoir_sampler
81
- assert min_reservoir_samples <= max_reservoir_samples
82
- assert min_reservoir_samples >= 1
83
-
84
- # this is where we store
85
- self.reservoir = collections.deque(maxlen=max_reservoir_samples)
86
- self.iter_source = None
87
- self.iter_reservoir = None
88
-
89
- self.perf_receiving = Performance()
90
- self.perf_sending = Performance()
91
-
92
- self.job_executer = None
93
- assert nb_workers != 0, 'must have background workers'
94
-
95
- if max_jobs_at_once is None:
96
- # default: each worker can push at least one item
97
- # before blocking
98
- max_jobs_at_once = nb_workers
99
-
100
- self.job_executer = JobExecutor2(
101
- nb_workers=nb_workers,
102
- function_to_run=self.function_to_run,
103
- max_queue_size_pin_thread_per_worker=max_jobs_at_once,
104
- max_queue_size_per_worker=max_jobs_at_once // nb_workers)
105
-
106
- self.maximum_number_of_samples_per_epoch = maximum_number_of_samples_per_epoch
107
- self.number_samples_generated = 0
108
- self.max_reservoir_replacement_size = max_reservoir_replacement_size
109
-
110
- def subsample(self, nb_samples):
111
- subsampled_source = self.source_split.subsample(nb_samples)
112
- return SequenceAsyncReservoir(subsampled_source, self.max_reservoir_samples, self.function_to_run,
113
- min_reservoir_samples=self.min_reservoir_samples, nb_workers=1,
114
- max_jobs_at_once=self.max_jobs_at_once,
115
- reservoir_sampler=copy.deepcopy(self.reservoir_sampler),
116
- collate_fn=self.collate_fn, maximum_number_of_samples_per_epoch=None,
117
- max_reservoir_replacement_size=self.max_reservoir_replacement_size)
118
-
119
- def reservoir_size(self):
120
- """
121
- Returns:
122
- The current number of samples in the reservoir
123
- """
124
- return len(self.reservoir)
125
-
126
- def subsample_uids(self, uids, uids_name, new_sampler=None):
127
- if new_sampler is None:
128
- sampler_to_use = copy.deepcopy(self.reservoir_sampler)
129
- else:
130
- sampler_to_use = copy.deepcopy(new_sampler)
131
-
132
- subsampled_source = self.source_split.subsample_uids(uids, uids_name, new_sampler)
133
-
134
- return SequenceAsyncReservoir(subsampled_source, self.max_reservoir_samples, self.function_to_run,
135
- min_reservoir_samples=self.min_reservoir_samples, nb_workers=1,
136
- max_jobs_at_once=self.max_jobs_at_once, reservoir_sampler=sampler_to_use,
137
- collate_fn=self.collate_fn, maximum_number_of_samples_per_epoch=None,
138
- max_reservoir_replacement_size=self.max_reservoir_replacement_size)
139
-
140
- def initializer(self):
141
- if self.iter_source is None:
142
- self.iter_source = self.source_split.__iter__()
143
-
144
- self._retrieve_results_and_fill_queue()
145
-
146
- if len(self.reservoir) < self.min_reservoir_samples:
147
- # before we can iterate on sample, we must have enough samples in the reservoir!
148
- self._wait_for_job_completion()
149
-
150
- def fill_queue(self):
151
- """
152
- Fill the input queue of jobs to be completed
153
- """
154
- nb_queued = 0
155
- try:
156
- while not self.job_executer.is_full():
157
- i = self.iter_source.next_item(blocking=False)
158
- nb_queued += 1
159
-
160
- time_blocked_start = time.perf_counter()
161
- self.job_executer.put(i)
162
- time_blocked_end = time.perf_counter()
163
- self.perf_sending.add(time_blocked_end - time_blocked_start)
164
- #print('RESERVOIR_QUEUE_FILL job_executer.put')
165
- except StopIteration:
166
- # we are done! Reset the input iterator
167
- self.iter_source = self.source_split.__iter__()
168
-
169
- def _retrieve_results_and_fill_queue(self):
170
- """
171
- Retrieve results from the output queue
172
- """
173
- # first make sure we can't be `starved`, so fill the input queue
174
- self.fill_queue()
175
-
176
- nb_samples_replaced = 0
177
-
178
- # retrieve the results from the output queue and fill the reservoir
179
- while not self.job_executer.pin_memory_queue.empty():
180
- try:
181
- time_blocked_start = time.perf_counter()
182
- metadata, items = self.job_executer.pin_memory_queue.get()
183
- if items is None:
184
- # the job failed!
185
- continue
186
- time_blocked_end = time.perf_counter()
187
- self.perf_receiving.add(time_blocked_end - time_blocked_start)
188
- self.reservoir.append(items)
189
- nb_samples_replaced += 1
190
-
191
- if self.max_reservoir_replacement_size is not None and nb_samples_replaced >= self.max_reservoir_replacement_size:
192
- # we have reached the maximum replacement, so stop here
193
- break
194
- except Empty:
195
- break
196
-
197
- def _wait_for_job_completion(self):
198
- """
199
- Block the processing until we have enough result in the reservoir
200
- """
201
- while len(self.reservoir) < self.min_reservoir_samples:
202
- self._retrieve_results_and_fill_queue()
203
-
204
- def __iter__(self):
205
- self.initializer()
206
- iterator = SequenceAsyncReservoirIterator(self, copy.deepcopy(self.reservoir_sampler))
207
-
208
- # keep track of average performance of receiving and sending objects to the workers
209
- self.perf_receiving = Performance()
210
- self.perf_sending = Performance()
211
- return iterator
212
-
213
- def close(self):
214
- """
215
- Finish and join the existing pool processes
216
- """
217
- if self.job_executer is not None:
218
- self.job_executer.close()
219
-
220
- if self.source_split is not None:
221
- self.source_split.close()
222
-
223
-
224
- class SequenceAsyncReservoirIterator(sequence.SequenceIterator):
225
- """
226
- Iterate through the SequenceAsyncReservoir sequence
227
- """
228
- def __init__(self, base_sequence, reservoir_sampler):
229
- super().__init__()
230
- self.base_sequence = base_sequence
231
- self.reservoir_sampler = reservoir_sampler
232
-
233
- self._reset_iter_reservoir()
234
- self.number_samples_generated = 0
235
-
236
- def _reset_iter_reservoir(self):
237
- # make sure the reservoir is not changed during iteration, so take a copy
238
- # when the iterator is created
239
- self.reservoir_copy = list(self.base_sequence.reservoir)
240
-
241
- # print(f'LAST_receiving=', self.base_sequence.perf_receiving.get_average_time())
242
- # Note: in case the queue receive stalls the sequence, it is worth trying a different
243
- # sharing strategy: torch.multiprocessing.set_sharing_strategy('file_system')
244
-
245
- # create the reservoir sampler if necessary
246
- if self.reservoir_sampler is not None:
247
- # If no sampler, we can make it faster
248
- self.reservoir_sampler.initializer(self.reservoir_copy)
249
- self.iter_reservoir = iter(self.reservoir_sampler)
250
- else:
251
- self.iter_reservoir = self.reservoir_copy.__iter__()
252
-
253
- def __next__(self):
254
- if self.base_sequence.maximum_number_of_samples_per_epoch is not None and \
255
- self.number_samples_generated >= self.base_sequence.maximum_number_of_samples_per_epoch:
256
- # we have reached the maximum number of samples, stop the sequence
257
- raise StopIteration()
258
-
259
- if self.reservoir_sampler is not None:
260
- # items are actually a list of indices!
261
- indices = self.iter_reservoir.__next__()
262
- if isinstance(indices, collections.Iterable):
263
- items = [self.reservoir_copy[index] for index in indices]
264
- self.number_samples_generated += len(items)
265
- else:
266
- self.number_samples_generated += 1
267
- items = [self.reservoir_copy[indices]]
268
- else:
269
- self.number_samples_generated += 1
270
- items = [self.iter_reservoir.__next__()]
271
-
272
- if self.base_sequence.collate_fn is not None:
273
- return self.base_sequence.collate_fn(items)
274
-
275
- return items
276
-
277
- def close(self):
278
- self.base_sequence.close()