File size: 5,991 Bytes
91f863a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 | import os
import pandas as pd
import numpy as np
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split, StratifiedShuffleSplit
def norm_Clint_scaler(df_data, is_testCode = False):
# df_norm = df_data[(df_data['logP'] >= -2) & (df_data['logP'] <= 8) & (v['Clint'] <= 500) & (df_data['Fup'] >= 0.01) & (df_load['Fup'] <= 0.99)]
df_norm = df_data[(df_data['Clint'] <= 500) & (df_data['Fup'] >= 0.01) & (df_data['Fup'] <= 0.99)]
df_norm['Clint'] = np.log1p(df_norm['Clint'])
if not is_testCode:
df_norm['Clint'], scaler = scaling(df_norm['Clint'])
return df_norm, scaler
else:
df_norm['Clint'], df_mean, df_std= scaling_test(df_norm['Clint'])
return df_norm, [df_mean, df_std]
def norm_shuffledSet_merge(df_train, df_test):
# df_train.dropna(subset=['SMILES','logP','Clint','Fup'], axis=0 ,inplace=True)
# df_test.dropna(subset=['SMILES','logP','Clint','Fup'], axis=0 ,inplace=True)
merged_dataset = pd.concat([df_train, df_test])
merged_dataset.reset_index(drop=True, inplace=True)
# df_merged_norm, merged_scaler = norm_Clint_scaler(merged_dataset)
df_merged_norm, merged_scaler = norm_Clint_scaler(merged_dataset, is_testCode=True)
# df_train_norm = df_merged_norm.iloc[:len(df_train),:]
# df_test_norm = df_merged_norm.iloc[len(df_train):,:]
df_train_norm = df_merged_norm.sample(frac=0.8, random_state=42)
df_test_norm = df_merged_norm.drop(df_train_norm.index)
return df_train_norm, df_test_norm, merged_scaler
def norm_shuffledSet(df_train:pd.DataFrame, df_test:pd.DataFrame, random_seed:int, shuffle_rate = None):
# df_train.dropna(subset=['SMILES','logP','Clint','Fup'], axis=0 ,inplace=True)
# df_test.dropna(subset=['SMILES','logP','Clint','Fup'], axis=0 ,inplace=True)
if shuffle_rate is None:
df_train_norm, train_scaler = norm_Clint_scaler(df_train)
df_test_norm, test_scaler = norm_Clint_scaler(df_test)
else:
valid = df_test.sample(frac=shuffle_rate, random_state=random_seed)
df_test = df_test.drop(valid.index)
# df_test = df_test.loc[~df_test["SMILES"].isin(valid["SMILES"].tolist())]
df_merge_train = pd.concat([df_train, valid])
df_merge_train.reset_index(drop=True, inplace=True)
df_test.reset_index(drop=True, inplace=True)
df_train_norm, train_scaler = norm_Clint_scaler(df_merge_train)
df_test_norm, test_scaler = norm_Clint_scaler(df_test)
# df_train_norm, train_scaler = norm_Clint_scaler(df_merge_train, is_testCode=True)
# df_test_norm, test_scaler = norm_Clint_scaler(df_test, is_testCode=True)
return df_train_norm, train_scaler, df_test_norm, test_scaler
# return df_train_norm, df_test_norm
def normalization(df_load, rm_duplicates:bool = False):
## -- Read dataset drop duplication and Nan data-- ##
if not "SMILES" in df_load.columns:
df_load.rename(columns = {'SMILES_rdkit_final':'SMILES'},inplace=True)
df_load.rename(columns = {'Clint.invitro.':'Clint'},inplace=True)
df_load.dropna(subset=['SMILES','logP','Clint','Fup'], axis=0 ,inplace=True)
## -- Remove duplicates SMILES Data -- ##
if rm_duplicates:
df_load.drop_duplicates(['SMILES'])
## -- Check Attributes Values -- ##
# df_norm = df_load[(df_load['logP'] >= -2) & (df_load['logP'] <= 8) & (df_load['Clint'] <= 500) & (df_load['Fup'] >= 0.01) & (df_load['Fup'] <= 0.99)]
df_norm = df_load[(df_load['Clint'] <= 500) & (df_load['Fup'] >= 0.01) & (df_load['Fup'] <= 0.99)]
# ## -- Clearance Normalization -- ##
df_norm['Clint'] = np.log1p(df_norm['Clint'])
# ## -- F_up Normalization -- ##
df_norm['Fup'] = np.log1p(df_norm['Fup'])
scaling_info = {"Clint": None, "Fup": None, "logP": None}
df_norm['Clint'], scaler_clint = scaling(df_norm['Clint'])
df_norm['Fup'], scaler_fup = scaling(df_norm['Fup'])
df_norm['logP'], scaler_logp = scaling(df_norm['logP'])
return df_norm, scaler_clint
def normalization_logscale(df_load):
## -- Read dataset drop duplication and Nan data-- ##
if not "SMILES" in df_load.columns:
df_load.rename(columns = {'SMILES_rdkit_final':'SMILES'},inplace=True)
df_load.rename(columns = {'Clint.invitro.':'Clint'},inplace=True)
df_load.dropna(subset=['SMILES','logP','Clint','Fup'], axis=0 ,inplace=True)
## -- Check Attributes Values -- ##
# df_norm = df_load[(df_load['logP'] >= -2) & (df_load['logP'] <= 8) & (df_load['Clint'] <= 500) & (df_load['Fup'] >= 0.01) & (df_load['Fup'] <= 0.99)]
df_norm = df_load[(df_load['Clint'] <= 500) & (df_load['Fup'] >= 0.01) & (df_load['Fup'] <= 0.99)]
# ## -- Clearance Normalization -- ##
df_norm['Clint'] = np.log1p(df_norm['Clint'])
return df_norm
def scaling_test(df_data):
dataset_train = df_data.astype(np.float64)
features = dataset_train
data_mean = features.mean(axis=0)
data_std = features.std(axis=0)
features = (features-data_mean)/data_std
dataset_train = features
return dataset_train, data_mean, data_std
def scaling(df_data):
scaler = MinMaxScaler(feature_range = (0,1))
data_reshape = df_data.values.reshape(-1,1)
scaler_fit = scaler.fit(data_reshape)
scaling_data = scaler_fit.transform(data_reshape)
scaling_data = scaling_data.flatten()
return scaling_data, scaler_fit
def inverse_scaling(df_datas, scaler_fit, log_scale=False):
df_datas = df_datas.reshape(-1,1)
inverse_scale = scaler_fit.inverse_transform(df_datas)
if log_scale:
inverse_scale = np.expm1(inverse_scale)
inverse_scale = inverse_scale.flatten()
return inverse_scale
def inverse_normalize(df_norm):
df_norm["Clint"] = np.expm1(df_norm["Clint"]) - 0.01
df_norm['Fup'] = np.log1p(df_norm['Fup']) - 0.001
# df_norm['logP'] = np.log1p(df_norm['logP']) - 0.01
return df_norm |