File size: 8,002 Bytes
862e0b5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 | from locale import D_FMT
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from sklearn.model_selection import train_test_split
import scipy.stats as stats
from imblearn.over_sampling import SMOTE, ADASYN, BorderlineSMOTE, RandomOverSampler
from pandas import read_csv
from numpy import dstack
from sklearn.preprocessing import StandardScaler, LabelEncoder
from tensorflow.keras.utils import to_categorical
#from keras.utils import to_categorical
from pandas import read_csv
from numpy import dstack
import pickle
#Dataset Option
WISDM0_UCI1 = 0
Aug_Option = 2
name_of_augmentation_algorithm = ["Original","SMOTE","ADASYN","BorderlineSMOTE","RandomOverSampler"]
if WISDM0_UCI1 == 0:
number_of_each_class = 11000
else:
number_of_each_class = 3500
dataset_name_WISDM0_UCI1 = ["WISDM", 'UCI_HAR']
train0_test1 = ["train", "test"]
save_path = 'HAR_code/HAR_data/' + dataset_name_WISDM0_UCI1[WISDM0_UCI1]
def get_frames(df, frame_size, hop_size):
N_FEATURES = 3
frames = []
labels = []
for i in range(0, len(df) - frame_size, hop_size):
x = df['x'].values[i: i + frame_size]
y = df['y'].values[i: i + frame_size]
z = df['z'].values[i: i + frame_size]
# Retrieve the most often used label in this segment
label = stats.mode(df['label'][i: i + frame_size])[0][0]
frames.append([x, y, z])
labels.append(label)
# Bring the segments into a better shape
frames = np.asarray(frames).reshape(-1, frame_size, N_FEATURES)
labels = np.asarray(labels)
return frames, labels
# load data ============================================================
if(WISDM0_UCI1 == 0):
Fs = 20
frame_size = Fs*4 # 80
hop_size = Fs*2 # 40
file = open('HAR_code/WISDM/WISDM_ar_v1.1/WISDM_ar_v1.1_raw.txt')
lines = file.readlines()
processedList = []
for i, line in enumerate(lines):
try:
line = line.split(',')
last = line[5].split(';')[0]
last = last.strip()
if last == '':
print('break!')
#break;
else: #original has no else
temp = [line[0], line[1], line[2], line[3], line[4], last]
processedList.append(temp)
except:
print('Error at line number: ', i)
columns = ['user', 'activity', 'time', 'x', 'y', 'z']
data = pd.DataFrame(data = processedList, columns = columns)
#data.head()
activities = data['activity'].value_counts().index
activities
df = data.drop(['user', 'time'], axis = 1).copy()
#df.head()
data['activity'].value_counts()
# improve data imbalance problem
label = LabelEncoder()
data['label'] = label.fit_transform(data['activity'])
#data.head()
X = data[['x', 'y', 'z']]
y = data['label']
scaler = StandardScaler()
X = scaler.fit_transform(X)
scaled_X = pd.DataFrame(data = X, columns = ['x', 'y', 'z'])
scaled_X['label'] = y.values
scaled_X.head()
X, y = get_frames(scaled_X, frame_size, hop_size)
X.shape, y.shape
class_name = label.classes_
else:
def load_file(filepath):
dataframe = read_csv(filepath, header=None, delim_whitespace=True)
return dataframe.values
def load_group(filenames, prefix=''):
loaded = list()
for name in filenames:
data = load_file(prefix + name)
loaded.append(data)
loaded = dstack(loaded)
return loaded
def load_dataset_group(group, prefix=''):
filepath = prefix + group + '/Inertial Signals/'
# load all 9 files as a single array
filenames = list()
# total acceleration
filenames += ['total_acc_x_'+group+'.txt', 'total_acc_y_'+group+'.txt', 'total_acc_z_'+group+'.txt']
# body acceleration
filenames += ['body_acc_x_'+group+'.txt', 'body_acc_y_'+group+'.txt', 'body_acc_z_'+group+'.txt']
# body gyroscope
filenames += ['body_gyro_x_'+group+'.txt', 'body_gyro_y_'+group+'.txt', 'body_gyro_z_'+group+'.txt']
# load input data
X = load_group(filenames, filepath)
# load class output
y = load_file(prefix + group + '/y_'+group+'.txt')
return X, y
def load_dataset(prefix=''):
# load all train
trainX, trainy = load_dataset_group('train', prefix + 'HAR_code/UCI/UCI HAR Dataset/')
print(trainX.shape, trainy.shape)
# load all test
testX, testy = load_dataset_group('test', prefix + 'HAR_code/UCI/UCI HAR Dataset/')
print(testX.shape, testy.shape)
# zero-offset class values
trainy = trainy - 1
testy = testy - 1
print(trainX.shape, trainy.shape, testX.shape, testy.shape)
return trainX, trainy, testX, testy
if Aug_Option != 0:
X, y, X_test, y_test = load_dataset()
X=np.concatenate((X,X_test), axis=0)
y=np.concatenate((y,y_test), axis=0)
else:
X_train, y_train, X_test, y_test = load_dataset()
class_name = np.array(["Walking","Walking Upstairs","Walking Downstairs","Sitting","Standing","Laying"])
unique, counts = np.unique(y, return_counts=True)
print('Before augmentation', dict(zip(unique, counts)))
# Oversampling based augmentation ==============================================
if Aug_Option != 0:
X_oneD = np.zeros((len(X),len(X[0])*len(X[0][0])))
for window_idx in range(len(X)):
buf = np.array([])
for timestep_idx in range(len(X[0])):
buf = np.append(buf, X[window_idx][timestep_idx])
X_oneD[window_idx] = buf
print('Data augmentation option : {}'.format(name_of_augmentation_algorithm[Aug_Option]))
strategy = {0:number_of_each_class, 1:number_of_each_class, 2:number_of_each_class, 3:number_of_each_class, 4:number_of_each_class, 5:number_of_each_class}
#이곳 SMOTE대신 ADASYN, BorderlineSMOTE, RandomOverSampler
if Aug_Option == 1:
oversample = SMOTE(k_neighbors=10, sampling_strategy=strategy) # k_neighbors 수가 모델보다 데이터가 부족할 수 있음
elif Aug_Option == 2:
oversample = ADASYN(n_neighbors=10, sampling_strategy=strategy, n_jobs=-1) # k_neighbors 수가 모델보다 데이터가 부족할 수 있음
elif Aug_Option == 3:
oversample = BorderlineSMOTE(k_neighbors=10, sampling_strategy=strategy) # k_neighbors 수가 모델보다 데이터가 부족할 수 있음
elif Aug_Option == 4:
oversample = RandomOverSampler(sampling_strategy=strategy)
X_AUG, y_AUG = oversample.fit_resample(X_oneD, y)
X_AUG = np.reshape(X_AUG,(len(X_AUG),len(X[0]),len(X[0][0])))
X_train, X_test, y_train, y_test = train_test_split(X_AUG, y_AUG, test_size = 0.3, random_state = 0, stratify = y_AUG)
print('X_train.shape : {} X_test.shape : {}'.format(X_train.shape, X_test.shape))
shape_of_data = [(len(X_train),len(X_train[0]),len(X_train[0][0])), (len(X_test),len(X_test[0]),len(X_test[0][0]))]
with open(save_path + "_shape_of_data.p", 'wb') as f:
for line in shape_of_data:
pickle.dump(line, f)
with open(save_path + "_X_train_"+ name_of_augmentation_algorithm[Aug_Option] +".p", 'wb') as f:
for line in X_train:
pickle.dump(line, f)
with open(save_path + "_y_train_"+ name_of_augmentation_algorithm[Aug_Option] +".p", 'wb') as f:
for line in y_train:
pickle.dump(line, f)
with open(save_path + "_X_test_"+ name_of_augmentation_algorithm[Aug_Option] +".p", 'wb') as f:
for line in X_test:
pickle.dump(line, f)
with open(save_path + "_y_test_"+ name_of_augmentation_algorithm[Aug_Option] +".p", 'wb') as f:
for line in y_test:
pickle.dump(line, f)
print('finish')
|