fyrizi commited on
Commit
c4e898c
·
verified ·
1 Parent(s): fab1eb4

Delete main/src/sequence_array.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. main/src/sequence_array.py +0 -217
main/src/sequence_array.py DELETED
@@ -1,217 +0,0 @@
1
- import sequence
2
- import sampler as sampler_src
3
- import numpy as np
4
- import collections
5
- import copy
6
- import torch
7
-
8
- # this the name used for the sample UID
9
- sample_uid_name = 'sample_uid'
10
-
11
-
12
- def get_batch_n(split, nb_samples, indices, transforms, use_advanced_indexing):
13
- """
14
- Collect the split indices given and apply a series of transformations
15
-
16
- Args:
17
- nb_samples: the total number of samples of split
18
- split: a mapping of `np.ndarray` or `torch.Tensor`
19
- indices: a list of indices as numpy array
20
- transforms: a transformation or list of transformations or None
21
- use_advanced_indexing: if True, use the advanced indexing mechanism else
22
- use a simple list (original data is referenced)
23
- advanced indexing is typically faster for small objects, however for large objects (e.g., 3D data)
24
- the advanced indexing makes a copy of the data making it very slow.
25
-
26
- Returns:
27
- a split with the indices provided
28
- """
29
- data = {}
30
- for split_name, split_data in split.items():
31
- if isinstance(split_data, (torch.Tensor, np.ndarray)) and len(split_data) == nb_samples:
32
- # here we prefer [split_data[i] for i in indices] over split_data[indices]
33
- # this is because split_data[indices] will make a deep copy of the data which may be time consuming
34
- # for large data
35
- if use_advanced_indexing:
36
- split_data = split_data[indices]
37
- else:
38
- split_data = [[split_data[i]] for i in indices]
39
- if isinstance(split_data, list) and len(split_data) == nb_samples:
40
- split_data = [split_data[i] for i in indices]
41
-
42
- data[split_name] = split_data
43
-
44
- if transforms is None:
45
- # do nothing: there is no transform
46
- pass
47
- elif isinstance(transforms, collections.Sequence):
48
- # we have a list of transforms, apply each one of them
49
- for transform in transforms:
50
- data = transform(data)
51
- else:
52
- # anything else should be a functor
53
- data = transforms(data)
54
-
55
- return data
56
-
57
-
58
- def len_batch(batch):
59
- """
60
-
61
- Args:
62
- batch: a data split or a `collections.Sequence`
63
-
64
- Returns:
65
- the number of elements within a data split
66
- """
67
- if isinstance(batch, (collections.Sequence, torch.Tensor)):
68
- return len(batch)
69
-
70
- assert isinstance(batch, collections.Mapping), 'Must be a dict-like structure! got={}'.format(type(batch))
71
-
72
- for name, values in batch.items():
73
- if isinstance(values, (list, tuple)):
74
- return len(values)
75
- if isinstance(values, torch.Tensor) and len(values.shape) != 0:
76
- return values.shape[0]
77
- if isinstance(values, np.ndarray) and len(values.shape) != 0:
78
- return values.shape[0]
79
- return 0
80
-
81
- class SequenceArray(sequence.Sequence):
82
- """
83
- Create a sequence of batches from numpy arrays, lists and :class:`torch.Tensor`
84
- """
85
- def __init__(
86
- self,
87
- split,
88
- sampler=sampler_src.SamplerRandom(),
89
- transforms=None,
90
- use_advanced_indexing=True,
91
- sample_uid_name=sample_uid_name):
92
- """
93
-
94
- Args:
95
- split: a dictionary of tensors. Tensors may be `numpy.ndarray`, `torch.Tensor`, numeric
96
- sampler: the sampler to be used to iterate through the sequence
97
- transforms: a transform or list of transforms to be applied on each batch of data
98
- use_advanced_indexing:
99
- sample_uid_name: if not `None`, create a unique UID per sample so that it is easy to track
100
- particular samples (e.g., during data augmentation)
101
- """
102
- super().__init__(None) # there is no source sequence for this as we get our input from a numpy split
103
- self.split = split
104
- self.nb_samples = None
105
- self.sampler = sampler
106
- self.sampler_iterator = None
107
- self.transforms = transforms
108
- self.use_advanced_indexing = use_advanced_indexing
109
-
110
- # create a unique UID
111
- if sample_uid_name is not None and sample_uid_name not in split:
112
- split[sample_uid_name] = np.asarray(np.arange(len_batch(split)))
113
-
114
- def subsample(self, nb_samples):
115
- # get random indices
116
- subsample_sample = sampler_src.SamplerRandom(batch_size=nb_samples)
117
- subsample_sample.initializer(self.split)
118
-
119
- # extract the indices
120
- indices = next(iter(subsample_sample))
121
- subsampled_split = get_batch_n(
122
- self.split,
123
- len_batch(self.split),
124
- indices,
125
- self.transforms,
126
- # use `use_advanced_indexing` so that we keep the types as close as possible to original
127
- use_advanced_indexing=True
128
- )
129
- return SequenceArray(
130
- subsampled_split,
131
- copy.deepcopy(self.sampler),
132
- transforms=self.transforms,
133
- use_advanced_indexing=self.use_advanced_indexing
134
- )
135
-
136
- def subsample_uids(self, uids, uids_name, new_sampler=None):
137
- uid_values = self.split.get(uids_name)
138
- assert uid_values is not None, 'no UIDs with name={}'.format(uids_name)
139
-
140
- # find the samples that are in `uids`
141
- indices_to_keep = []
142
- uids_set = set(uids)
143
- for index, uid in enumerate(uid_values):
144
- if uid in uids_set:
145
- indices_to_keep.append(index)
146
-
147
- # reorder the `indices_to_keep` following the `uids` ordering
148
- uids_ordering = {uid: index for index, uid in enumerate(uids)}
149
- kvp_index_ordering = []
150
- for index in indices_to_keep:
151
- uid = uid_values[index]
152
- ordering = uids_ordering[uid]
153
- kvp_index_ordering.append((index, ordering))
154
- kvp_uids_ordering = sorted(kvp_index_ordering, key=lambda value: value[1])
155
- indices_to_keep = [index for index, ordering in kvp_uids_ordering]
156
-
157
- # extract the samples
158
- subsampled_split = get_batch_n(
159
- self.split,
160
- len_batch(self.split),
161
- indices_to_keep,
162
- self.transforms,
163
- # use `use_advanced_indexing` so that we keep the types as close as possible to original
164
- use_advanced_indexing=True
165
- )
166
-
167
- if new_sampler is None:
168
- new_sampler = copy.deepcopy(self.sampler)
169
- else:
170
- new_sampler = copy.deepcopy(new_sampler)
171
-
172
- return SequenceArray(
173
- subsampled_split,
174
- new_sampler,
175
- transforms=self.transforms,
176
- use_advanced_indexing=self.use_advanced_indexing
177
- )
178
-
179
- def __iter__(self):
180
- # make sure the sampler is copied so that we can have multiple iterators of the
181
- # same sequence
182
- return SequenceIteratorArray(self, copy.deepcopy(self.sampler))
183
-
184
- def close(self):
185
- pass
186
-
187
-
188
- class SequenceIteratorArray(sequence.SequenceIterator):
189
- """
190
- Iterate the elements of an :class:`SequenceArray` sequence
191
-
192
- Assumptions:
193
- - underlying `base_sequence` doesn't change sizes while iterating
194
- """
195
- def __init__(self, base_sequence, sampler):
196
- super().__init__()
197
- self.base_sequence = base_sequence
198
- self.nb_samples = len_batch(self.base_sequence.split)
199
-
200
- self.sampler = sampler
201
- self.sampler.initializer(self.base_sequence.split)
202
- self.sampler_iterator = iter(self.sampler)
203
-
204
- def __next__(self):
205
- indices = self.sampler_iterator.__next__()
206
- if not isinstance(indices, (np.ndarray, collections.Sequence)):
207
- indices = [indices]
208
-
209
- return get_batch_n(
210
- self.base_sequence.split,
211
- self.nb_samples,
212
- indices,
213
- self.base_sequence.transforms,
214
- self.base_sequence.use_advanced_indexing)
215
-
216
- def close(self):
217
- pass