Delete main/src/normalize_data.py with huggingface_hub
Browse files- main/src/normalize_data.py +0 -220
main/src/normalize_data.py
DELETED
|
@@ -1,220 +0,0 @@
|
|
| 1 |
-
import collections
|
| 2 |
-
import os
|
| 3 |
-
|
| 4 |
-
import numpy as np
|
| 5 |
-
import skimage.transform
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
def len_batch(batch):
|
| 11 |
-
"""
|
| 12 |
-
|
| 13 |
-
Args:
|
| 14 |
-
batch: a data split or a `collections.Sequence`
|
| 15 |
-
|
| 16 |
-
Returns:
|
| 17 |
-
the number of elements within a data split
|
| 18 |
-
"""
|
| 19 |
-
if isinstance(batch, (collections.Sequence, torch.Tensor)):
|
| 20 |
-
return len(batch)
|
| 21 |
-
|
| 22 |
-
assert isinstance(batch, collections.Mapping), 'Must be a dict-like structure! got={}'.format(type(batch))
|
| 23 |
-
|
| 24 |
-
for name, values in batch.items():
|
| 25 |
-
if isinstance(values, (list, tuple)):
|
| 26 |
-
return len(values)
|
| 27 |
-
if isinstance(values, torch.Tensor) and len(values.shape) != 0:
|
| 28 |
-
return values.shape[0]
|
| 29 |
-
if isinstance(values, np.ndarray) and len(values.shape) != 0:
|
| 30 |
-
return values.shape[0]
|
| 31 |
-
return 0
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
def safe_lookup(dictionary, *keys, default=None):
|
| 36 |
-
"""
|
| 37 |
-
Recursively access nested dictionaries
|
| 38 |
-
Args:
|
| 39 |
-
dictionary: nested dictionary
|
| 40 |
-
*keys: the keys to access within the nested dictionaries
|
| 41 |
-
default: the default value if dictionary is ``None`` or it doesn't contain
|
| 42 |
-
the keys
|
| 43 |
-
Returns:
|
| 44 |
-
None if we can't access to all the keys, else dictionary[key_0][key_1][...][key_n]
|
| 45 |
-
"""
|
| 46 |
-
if dictionary is None:
|
| 47 |
-
return default
|
| 48 |
-
|
| 49 |
-
for key in keys:
|
| 50 |
-
dictionary = dictionary.get(key)
|
| 51 |
-
if dictionary is None:
|
| 52 |
-
return default
|
| 53 |
-
|
| 54 |
-
return dictionary
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
from data_category import DataCategory
|
| 59 |
-
from table_sqlite import get_data_types_and_clean_data
|
| 60 |
-
from typing import List
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
def resize_arrays(arrays: List[np.ndarray], shape=None) -> np.ndarray:
|
| 64 |
-
if shape is None:
|
| 65 |
-
shapes = [a.shape for a in arrays]
|
| 66 |
-
shape = np.asarray(shapes).mean(axis=0).round().astype(int)
|
| 67 |
-
|
| 68 |
-
resized_arrays = []
|
| 69 |
-
for array in arrays:
|
| 70 |
-
resized_array = skimage.transform.resize(
|
| 71 |
-
array,
|
| 72 |
-
shape,
|
| 73 |
-
order=1,
|
| 74 |
-
mode='constant',
|
| 75 |
-
anti_aliasing=False,
|
| 76 |
-
preserve_range=True)
|
| 77 |
-
resized_arrays.append(resized_array)
|
| 78 |
-
return np.asarray(resized_arrays)
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
def normalize_data(options, data, table_name):
|
| 82 |
-
"""
|
| 83 |
-
Normalize, subsample and categorize the data
|
| 84 |
-
|
| 85 |
-
The following operations are performed:
|
| 86 |
-
- convert to numpy arrays
|
| 87 |
-
- removed type column and return a specific `type` dictionary
|
| 88 |
-
- normalize the path according to deployment: if static html, nothing to do
|
| 89 |
-
if deployed, we MUST add the `app name` (the top folder name containing the SQL DB)
|
| 90 |
-
as root
|
| 91 |
-
- categorize the data as continuous, discrete unordered, discrete ordered, other
|
| 92 |
-
- recover the DB type (string) from data values (e.g., float or int)
|
| 93 |
-
"""
|
| 94 |
-
d = collections.OrderedDict()
|
| 95 |
-
for name, values in data.items():
|
| 96 |
-
d[name] = np.asarray(values)
|
| 97 |
-
|
| 98 |
-
for name in list(d.keys()):
|
| 99 |
-
dtype = d[name].dtype
|
| 100 |
-
if dtype in options.data.types_to_discard:
|
| 101 |
-
del d[name]
|
| 102 |
-
|
| 103 |
-
types = get_data_types_and_clean_data(d)
|
| 104 |
-
type_categories = {}
|
| 105 |
-
|
| 106 |
-
# handle the column removal: maybe they are not useful or maybe the data
|
| 107 |
-
# can't be parsed
|
| 108 |
-
remove_columns = safe_lookup(options.config, table_name, 'data', 'remove_columns', default=[])
|
| 109 |
-
if remove_columns is not None:
|
| 110 |
-
for n in remove_columns:
|
| 111 |
-
if n in data:
|
| 112 |
-
del d[n]
|
| 113 |
-
if n in types:
|
| 114 |
-
del types[n]
|
| 115 |
-
|
| 116 |
-
subsampling_factor = safe_lookup(
|
| 117 |
-
options.config,
|
| 118 |
-
table_name,
|
| 119 |
-
'data',
|
| 120 |
-
'subsampling_factor',
|
| 121 |
-
default='1.0')
|
| 122 |
-
subsampling_factor = float(subsampling_factor)
|
| 123 |
-
assert subsampling_factor <= 1.0, 'sub-sampling factor must be <= 1.0'
|
| 124 |
-
if subsampling_factor != 1.0:
|
| 125 |
-
nb_samples = int(len_batch(d) * subsampling_factor)
|
| 126 |
-
d = {
|
| 127 |
-
name: values[:nb_samples] for name, values in d.items()
|
| 128 |
-
}
|
| 129 |
-
|
| 130 |
-
if options.embedded:
|
| 131 |
-
appname = os.path.basename(os.path.dirname(options.db_root))
|
| 132 |
-
for n, t in types.items():
|
| 133 |
-
if 'BLOB_' in t:
|
| 134 |
-
# it is a blob, meaning that it was saved on the local HD
|
| 135 |
-
column = []
|
| 136 |
-
for i in d[n]:
|
| 137 |
-
if i is not None:
|
| 138 |
-
column.append(f'{appname}/' + i)
|
| 139 |
-
else:
|
| 140 |
-
# the data was not present...
|
| 141 |
-
column.append(None)
|
| 142 |
-
d[n] = column
|
| 143 |
-
#d[n] = list(np.core.defchararray.add(np.asarray([f'{appname}/']), d[n]))
|
| 144 |
-
|
| 145 |
-
if 'BLOB_IMAGE' in t:
|
| 146 |
-
type_categories[n] = DataCategory.Other
|
| 147 |
-
|
| 148 |
-
# load the numpy arrays
|
| 149 |
-
for name, t in list(types.items()):
|
| 150 |
-
if 'BLOB_NUMPY' in t:
|
| 151 |
-
loaded_np = []
|
| 152 |
-
root = os.path.join(os.path.dirname(options.db_root), '..')
|
| 153 |
-
for relative_path in d[name]:
|
| 154 |
-
try:
|
| 155 |
-
path = os.path.join(root, relative_path)
|
| 156 |
-
loaded_np.append(np.load(path))
|
| 157 |
-
except:
|
| 158 |
-
# loading failed
|
| 159 |
-
loaded_np.append(None)
|
| 160 |
-
|
| 161 |
-
# expand the array if satisfying the criteria
|
| 162 |
-
try:
|
| 163 |
-
array = np.asarray(loaded_np)
|
| 164 |
-
except ValueError as e:
|
| 165 |
-
array = None
|
| 166 |
-
|
| 167 |
-
if array is not None and \
|
| 168 |
-
len(array.shape) == 2 and \
|
| 169 |
-
array.shape[1] <= options.data.unpack_numpy_arrays_with_less_than_x_columns:
|
| 170 |
-
for n in range(array.shape[1]):
|
| 171 |
-
name_expanded = name + f'_{n}'
|
| 172 |
-
value_expanded = array[:, n]
|
| 173 |
-
d[name_expanded] = value_expanded
|
| 174 |
-
type_categories[name_expanded] = DataCategory.from_numpy_array(value_expanded)
|
| 175 |
-
del d[name]
|
| 176 |
-
del types[name]
|
| 177 |
-
else:
|
| 178 |
-
d[name] = loaded_np
|
| 179 |
-
type_categories[name] = DataCategory.Other # dimension too high
|
| 180 |
-
|
| 181 |
-
# the DB converted all values to string and have lost the original type. Try to
|
| 182 |
-
# revert back the type from the column values
|
| 183 |
-
for name in list(d.keys()):
|
| 184 |
-
t = d[name]
|
| 185 |
-
|
| 186 |
-
if name not in type_categories:
|
| 187 |
-
# do not take into account the `None`
|
| 188 |
-
# so create a list of indices with non `None` row
|
| 189 |
-
t = np.asarray(t)
|
| 190 |
-
indices_not_none = np.asarray([i for i, v in enumerate(t) if v is not None]) # consider only full row
|
| 191 |
-
|
| 192 |
-
try:
|
| 193 |
-
t_np = np.asarray(t[indices_not_none], dtype=np.int) # if e have a float, it will raise exception
|
| 194 |
-
type_categories[name] = DataCategory.DiscreteOrdered
|
| 195 |
-
full = np.ndarray(len(t), dtype=object) # handle the ``None`` values
|
| 196 |
-
full[indices_not_none] = t_np
|
| 197 |
-
d[name] = list(full)
|
| 198 |
-
continue # success, go to next item
|
| 199 |
-
except ValueError:
|
| 200 |
-
pass
|
| 201 |
-
|
| 202 |
-
try:
|
| 203 |
-
t_np = np.asarray(t[indices_not_none], dtype=np.float32)
|
| 204 |
-
type_categories[name] = DataCategory.Continuous
|
| 205 |
-
|
| 206 |
-
full = np.ndarray(len(t), dtype=object) # handle the ``None`` values
|
| 207 |
-
full[indices_not_none] = t_np
|
| 208 |
-
d[name] = list(full)
|
| 209 |
-
continue # success, go to next item
|
| 210 |
-
except ValueError:
|
| 211 |
-
type_categories[name] = DataCategory.Other
|
| 212 |
-
|
| 213 |
-
try:
|
| 214 |
-
_ = np.asarray(t[indices_not_none], dtype=np.str) # test data conversion
|
| 215 |
-
type_categories[name] = DataCategory.DiscreteUnordered
|
| 216 |
-
continue # success, go to next item
|
| 217 |
-
except ValueError:
|
| 218 |
-
type_categories[name] = DataCategory.Other
|
| 219 |
-
|
| 220 |
-
return d, types, type_categories
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|