fyrizi commited on
Commit
369a289
·
verified ·
1 Parent(s): 3c6882e

Delete main/src/normalize_data.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. main/src/normalize_data.py +0 -220
main/src/normalize_data.py DELETED
@@ -1,220 +0,0 @@
1
- import collections
2
- import os
3
-
4
- import numpy as np
5
- import skimage.transform
6
-
7
-
8
-
9
-
10
- def len_batch(batch):
11
- """
12
-
13
- Args:
14
- batch: a data split or a `collections.Sequence`
15
-
16
- Returns:
17
- the number of elements within a data split
18
- """
19
- if isinstance(batch, (collections.Sequence, torch.Tensor)):
20
- return len(batch)
21
-
22
- assert isinstance(batch, collections.Mapping), 'Must be a dict-like structure! got={}'.format(type(batch))
23
-
24
- for name, values in batch.items():
25
- if isinstance(values, (list, tuple)):
26
- return len(values)
27
- if isinstance(values, torch.Tensor) and len(values.shape) != 0:
28
- return values.shape[0]
29
- if isinstance(values, np.ndarray) and len(values.shape) != 0:
30
- return values.shape[0]
31
- return 0
32
-
33
-
34
-
35
- def safe_lookup(dictionary, *keys, default=None):
36
- """
37
- Recursively access nested dictionaries
38
- Args:
39
- dictionary: nested dictionary
40
- *keys: the keys to access within the nested dictionaries
41
- default: the default value if dictionary is ``None`` or it doesn't contain
42
- the keys
43
- Returns:
44
- None if we can't access to all the keys, else dictionary[key_0][key_1][...][key_n]
45
- """
46
- if dictionary is None:
47
- return default
48
-
49
- for key in keys:
50
- dictionary = dictionary.get(key)
51
- if dictionary is None:
52
- return default
53
-
54
- return dictionary
55
-
56
-
57
-
58
- from data_category import DataCategory
59
- from table_sqlite import get_data_types_and_clean_data
60
- from typing import List
61
-
62
-
63
- def resize_arrays(arrays: List[np.ndarray], shape=None) -> np.ndarray:
64
- if shape is None:
65
- shapes = [a.shape for a in arrays]
66
- shape = np.asarray(shapes).mean(axis=0).round().astype(int)
67
-
68
- resized_arrays = []
69
- for array in arrays:
70
- resized_array = skimage.transform.resize(
71
- array,
72
- shape,
73
- order=1,
74
- mode='constant',
75
- anti_aliasing=False,
76
- preserve_range=True)
77
- resized_arrays.append(resized_array)
78
- return np.asarray(resized_arrays)
79
-
80
-
81
- def normalize_data(options, data, table_name):
82
- """
83
- Normalize, subsample and categorize the data
84
-
85
- The following operations are performed:
86
- - convert to numpy arrays
87
- - removed type column and return a specific `type` dictionary
88
- - normalize the path according to deployment: if static html, nothing to do
89
- if deployed, we MUST add the `app name` (the top folder name containing the SQL DB)
90
- as root
91
- - categorize the data as continuous, discrete unordered, discrete ordered, other
92
- - recover the DB type (string) from data values (e.g., float or int)
93
- """
94
- d = collections.OrderedDict()
95
- for name, values in data.items():
96
- d[name] = np.asarray(values)
97
-
98
- for name in list(d.keys()):
99
- dtype = d[name].dtype
100
- if dtype in options.data.types_to_discard:
101
- del d[name]
102
-
103
- types = get_data_types_and_clean_data(d)
104
- type_categories = {}
105
-
106
- # handle the column removal: maybe they are not useful or maybe the data
107
- # can't be parsed
108
- remove_columns = safe_lookup(options.config, table_name, 'data', 'remove_columns', default=[])
109
- if remove_columns is not None:
110
- for n in remove_columns:
111
- if n in data:
112
- del d[n]
113
- if n in types:
114
- del types[n]
115
-
116
- subsampling_factor = safe_lookup(
117
- options.config,
118
- table_name,
119
- 'data',
120
- 'subsampling_factor',
121
- default='1.0')
122
- subsampling_factor = float(subsampling_factor)
123
- assert subsampling_factor <= 1.0, 'sub-sampling factor must be <= 1.0'
124
- if subsampling_factor != 1.0:
125
- nb_samples = int(len_batch(d) * subsampling_factor)
126
- d = {
127
- name: values[:nb_samples] for name, values in d.items()
128
- }
129
-
130
- if options.embedded:
131
- appname = os.path.basename(os.path.dirname(options.db_root))
132
- for n, t in types.items():
133
- if 'BLOB_' in t:
134
- # it is a blob, meaning that it was saved on the local HD
135
- column = []
136
- for i in d[n]:
137
- if i is not None:
138
- column.append(f'{appname}/' + i)
139
- else:
140
- # the data was not present...
141
- column.append(None)
142
- d[n] = column
143
- #d[n] = list(np.core.defchararray.add(np.asarray([f'{appname}/']), d[n]))
144
-
145
- if 'BLOB_IMAGE' in t:
146
- type_categories[n] = DataCategory.Other
147
-
148
- # load the numpy arrays
149
- for name, t in list(types.items()):
150
- if 'BLOB_NUMPY' in t:
151
- loaded_np = []
152
- root = os.path.join(os.path.dirname(options.db_root), '..')
153
- for relative_path in d[name]:
154
- try:
155
- path = os.path.join(root, relative_path)
156
- loaded_np.append(np.load(path))
157
- except:
158
- # loading failed
159
- loaded_np.append(None)
160
-
161
- # expand the array if satisfying the criteria
162
- try:
163
- array = np.asarray(loaded_np)
164
- except ValueError as e:
165
- array = None
166
-
167
- if array is not None and \
168
- len(array.shape) == 2 and \
169
- array.shape[1] <= options.data.unpack_numpy_arrays_with_less_than_x_columns:
170
- for n in range(array.shape[1]):
171
- name_expanded = name + f'_{n}'
172
- value_expanded = array[:, n]
173
- d[name_expanded] = value_expanded
174
- type_categories[name_expanded] = DataCategory.from_numpy_array(value_expanded)
175
- del d[name]
176
- del types[name]
177
- else:
178
- d[name] = loaded_np
179
- type_categories[name] = DataCategory.Other # dimension too high
180
-
181
- # the DB converted all values to string and have lost the original type. Try to
182
- # revert back the type from the column values
183
- for name in list(d.keys()):
184
- t = d[name]
185
-
186
- if name not in type_categories:
187
- # do not take into account the `None`
188
- # so create a list of indices with non `None` row
189
- t = np.asarray(t)
190
- indices_not_none = np.asarray([i for i, v in enumerate(t) if v is not None]) # consider only full row
191
-
192
- try:
193
- t_np = np.asarray(t[indices_not_none], dtype=np.int) # if e have a float, it will raise exception
194
- type_categories[name] = DataCategory.DiscreteOrdered
195
- full = np.ndarray(len(t), dtype=object) # handle the ``None`` values
196
- full[indices_not_none] = t_np
197
- d[name] = list(full)
198
- continue # success, go to next item
199
- except ValueError:
200
- pass
201
-
202
- try:
203
- t_np = np.asarray(t[indices_not_none], dtype=np.float32)
204
- type_categories[name] = DataCategory.Continuous
205
-
206
- full = np.ndarray(len(t), dtype=object) # handle the ``None`` values
207
- full[indices_not_none] = t_np
208
- d[name] = list(full)
209
- continue # success, go to next item
210
- except ValueError:
211
- type_categories[name] = DataCategory.Other
212
-
213
- try:
214
- _ = np.asarray(t[indices_not_none], dtype=np.str) # test data conversion
215
- type_categories[name] = DataCategory.DiscreteUnordered
216
- continue # success, go to next item
217
- except ValueError:
218
- type_categories[name] = DataCategory.Other
219
-
220
- return d, types, type_categories