Upload 7 files

PDC_2023.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e809a36078ca3d39d255aab3edaf2ec5b93e6715442074ec3da43984814e3c7e
+size 30278752

PDC_2024.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8295cd6bb8a03b51474e85e419c47d395682ea44e772816f9c3d1781990d938a
+size 30279144

PDC_2025.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6ef69dc4bb3d1e160b86114def079e038e930d0c32423ff0b226ad23d251488
+size 30279144

dataset.py

@@ -0,0 +1,164 @@
+from os.path import split
+from operator import concat
+import re
+from cProfile import label
+from cgi import test
+from tkinter import Label
+import pandas as pd
+import numpy as np
+import tensorflow as tf
+from utils import smiles2adjoin, molecular_fg
+from rdkit import Chem
+from random import Random
+from collections import defaultdict
+from rdkit.Chem.Scaffolds import MurckoScaffold
+from itertools import compress
+
+str2num = {'<pad>':0 ,'H': 1, 'C': 2, 'N': 3, 'O': 4, 'S': 5, 'F': 6, 'Cl': 7, 'Br': 8, 'P':  9,
+         'I': 10,'Na': 11,'B':12,'Se':13,'Si':14,'<unk>':15,'<mask>':16,'<global>':17}
+num2str =  {i:j for j,i in str2num.items()}
+
+class Graph_Classification_Dataset(object):
+    def __init__(self,path,smiles_field1='Smiles1',smiles_field2='Smiles2',label_field=label, index_field=label, max_len=500,seed=1,batch_size=16,a=1,addH=True):
+        if path.endswith('.txt') or path.endswith('.tsv'):
+            self.df = pd.read_csv(path,sep='\t',encoding='latin1')
+        elif path.endswith('.xlsx'):
+            self.df = pd.read_excel(path)
+        else:
+            self.df = pd.read_csv(path, encoding='latin1')
+        self.smiles_field1 = smiles_field1
+        self.smiles_field2 = smiles_field2
+        self.label_field = label_field
+        self.index_field = index_field
+        self.vocab = str2num
+        self.devocab = num2str
+        self.df = self.df[self.df[smiles_field1].str.len() <= max_len]
+        self.df = self.df[[True if Chem.MolFromSmiles(smi) is not None else False for smi in self.df[smiles_field1]]]
+        self.seed = seed
+        self.batch_size = batch_size
+        self.a = a
+        self.addH = addH
+    
+    def get_data(self):
+        '''Randomized Split Dataset'''
+        data = self.df
+        data = data.fillna(666)
+        train_idx = []
+        idx = data.sample(frac=0.8).index
+        train_idx.extend(idx)
+        train_data = data[data.index.isin(train_idx)]
+        data = data[~data.index.isin(train_idx)]
+        test_idx = []
+        idx = data[~data.index.isin(train_data)].sample(frac=0.5).index
+        test_idx.extend(idx)
+        test_data = data[data.index.isin(test_idx)]
+        val_data = data[~data.index.isin(train_idx+test_idx)]
+        df_train_data = pd.DataFrame(train_data)
+        df_test_data = pd.DataFrame(test_data)
+        df_val_data = pd.DataFrame(val_data)
+
+        self.dataset1 = tf.data.Dataset.from_tensor_slices(
+            (df_train_data[self.smiles_field1], df_train_data[self.label_field], df_train_data[self.smiles_field2], df_train_data[self.index_field]))
+        self.dataset1 = self.dataset1.map(self.tf_numerical_smiles).cache().padded_batch(batch_size=self.batch_size, padded_shapes=(
+            tf.TensorShape([None]), tf.TensorShape([None, None]), tf.TensorShape([self.a]),tf.TensorShape([None]), tf.TensorShape([None, None]), tf.TensorShape([1]))).shuffle(1000).prefetch(50)
+
+        self.dataset2 = tf.data.Dataset.from_tensor_slices((df_test_data[self.smiles_field1], df_test_data[self.label_field], df_test_data[self.smiles_field2], df_test_data[self.index_field]))
+        self.dataset2 = self.dataset2.map(self.tf_numerical_smiles).padded_batch(512, padded_shapes=(
+            tf.TensorShape([None]), tf.TensorShape([None, None]), tf.TensorShape([self.a]),tf.TensorShape([None]), tf.TensorShape([None, None]), tf.TensorShape([1]))).cache().prefetch(100)
+
+        self.dataset3 = tf.data.Dataset.from_tensor_slices((df_val_data[self.smiles_field1], df_val_data[self.label_field], df_val_data[self.smiles_field2], df_val_data[self.index_field]))
+        self.dataset3 = self.dataset3.map(self.tf_numerical_smiles).padded_batch(512, padded_shapes=(
+            tf.TensorShape([None]), tf.TensorShape([None, None]), tf.TensorShape([self.a]),tf.TensorShape([None]), tf.TensorShape([None, None]), tf.TensorShape([1]))).cache().prefetch(100)
+
+        return self.dataset1, self.dataset2, self.dataset3
+
+    def numerical_smiles(self, smiles, label):
+        smiles = smiles.numpy().decode()
+        atoms_list, adjoin_matrix = smiles2adjoin(smiles,explicit_hydrogens=self.addH)
+        atoms_list = ['<global>'] + atoms_list
+        nums_list =  [str2num.get(i,str2num['<unk>']) for i in atoms_list]
+        temp = np.ones((len(nums_list),len(nums_list)))
+        temp[1:, 1:] = adjoin_matrix
+        adjoin_matrix = (1-temp)*(-1e9)
+        x = np.array(nums_list).astype('int64')
+        y = np.array(label).astype('int64')
+        return x, adjoin_matrix, y
+
+    def tf_numerical_smiles(self, smiles1, label, smiles2, index):
+        x1,adjoin_matrix1,y= tf.py_function(self.numerical_smiles, [smiles1,label], [tf.int64, tf.float32 ,tf.int64])
+        x1.set_shape([None])
+        adjoin_matrix1.set_shape([None,None])
+        y.set_shape([None])
+        x2,adjoin_matrix2,index = tf.py_function(self.numerical_smiles, [smiles2,index], [tf.int64, tf.float32 ,tf.int64])
+        x2.set_shape([None])
+        adjoin_matrix2.set_shape([None,None])
+        index.set_shape([None])
+        return x1, adjoin_matrix1, y, x2,adjoin_matrix2, index
+
+class Inference_Dataset(object):
+    def __init__(self,sml_list,max_len=500,addH=True):
+        self.vocab = str2num
+        self.devocab = num2str
+        self.sml_list = [i for i in sml_list if len(i)<max_len]
+        self.addH =  addH
+
+    def get_data(self):
+        self.dataset = tf.data.Dataset.from_tensor_slices((self.sml_list,))
+        self.dataset = self.dataset.map(self.tf_numerical_smiles).padded_batch(512, padded_shapes=(
+            tf.TensorShape([None]), tf.TensorShape([None,None]),tf.TensorShape([1]),tf.TensorShape([None]))).cache().prefetch(20)
+
+        return self.dataset
+
+    def numerical_smiles(self, smiles):
+        smiles_origin = smiles
+        smiles = smiles.numpy().decode()
+        atoms_list, adjoin_matrix = smiles2adjoin(smiles,explicit_hydrogens=self.addH)
+        atoms_list = ['<global>'] + atoms_list
+        nums_list =  [str2num.get(i,str2num['<unk>']) for i in atoms_list]
+        temp = np.ones((len(nums_list),len(nums_list)))
+        temp[1:,1:] = adjoin_matrix
+        adjoin_matrix = (1-temp)*(-1e9)
+        x = np.array(nums_list).astype('int64')
+        return x, adjoin_matrix,[smiles], atoms_list
+
+    def tf_numerical_smiles(self, smiles):
+        x,adjoin_matrix,smiles,atom_list = tf.py_function(self.numerical_smiles, [smiles], [tf.int64, tf.float32,tf.string, tf.string])
+        x.set_shape([None])
+        adjoin_matrix.set_shape([None,None])
+        smiles.set_shape([1])
+        atom_list.set_shape([None])
+        return x, adjoin_matrix,smiles,atom_list
+
+class Inference_Dataset(object):
+    def __init__(self,sml_list,max_len=500,addH=True):
+        self.vocab = str2num
+        self.devocab = num2str
+        self.sml_list = [i for i in sml_list if len(i)<max_len]
+        self.addH =  addH
+
+    def get_data(self):
+        self.dataset = tf.data.Dataset.from_tensor_slices((self.sml_list,))
+        self.dataset = self.dataset.map(self.tf_numerical_smiles).padded_batch(512, padded_shapes=(
+            tf.TensorShape([None]), tf.TensorShape([None,None]),tf.TensorShape([1]),tf.TensorShape([None]))).cache().prefetch(20)
+
+        return self.dataset
+
+    def numerical_smiles(self, smiles):
+        smiles_origin = smiles
+        smiles = smiles.numpy().decode()
+        atoms_list, adjoin_matrix = smiles2adjoin(smiles,explicit_hydrogens=self.addH)
+        atoms_list = ['<global>'] + atoms_list
+        nums_list =  [str2num.get(i,str2num['<unk>']) for i in atoms_list]
+        temp = np.ones((len(nums_list),len(nums_list)))
+        temp[1:,1:] = adjoin_matrix
+        adjoin_matrix = (1-temp)*(-1e9)
+        x = np.array(nums_list).astype('int64')
+        return x, adjoin_matrix,[smiles], atoms_list
+
+    def tf_numerical_smiles(self, smiles):
+        x,adjoin_matrix,smiles,atom_list = tf.py_function(self.numerical_smiles, [smiles], [tf.int64, tf.float32,tf.string, tf.string])
+        x.set_shape([None])
+        adjoin_matrix.set_shape([None,None])
+        smiles.set_shape([1])
+        atom_list.set_shape([None])
+        return x, adjoin_matrix,smiles,atom_list

encoding.py

@@ -0,0 +1,100 @@
+import numpy as np
+
+def zscale_padding(seq_list, padding):
+    zscale = {
+        'A': [0.24, -2.32, 0.60, -0.14, 1.30],
+        'C': [0.84, -1.67, 3.71, 0.18, -2.65],
+        'D': [3.98, 0.93, 1.93, -2.46, 0.75],
+        'E': [3.11, 0.26, -0.11, -0.34, -0.25],
+        'F': [-4.22, 1.94, 1.06, 0.54, -0.62],
+        'G': [2.05, -4.06, 0.36, -0.82, -0.38],
+        'H': [2.47, 1.95, 0.26, 3.90, 0.09],
+        'I': [-3.89, -1.73, -1.71, -0.84, 0.26],
+        'K': [2.29, 0.89, -2.49, 1.49, 0.31],
+        'L': [-4.28, -1.30, -1.49, -0.72, 0.84],
+        'M': [-2.85, -0.22, 0.47, 1.94, -0.98],
+        'N': [3.05, 1.62, 1.04, -1.15, 1.61],
+        'P': [-1.66, 0.27, 1.84, 0.70, 2.00],
+        'Q': [1.75, 0.50, -1.44, -1.34, 0.66],
+        'R': [3.52, 2.50, -3.50, 1.99, -0.17],
+        'S': [2.39, -1.07, 1.15, -1.39, 0.67],
+        'T': [0.75, -2.18, -1.12, -1.46, -0.40],
+        'V': [-2.59, -2.64, -1.54, -0.85, -0.02],
+        'W': [-4.36, 3.94, 0.59, 3.44, -1.59],
+        'Y': [-2.54, 2.44, 0.43, 0.04, -1.47],
+        '-': [0.00, 0.00, 0.00, 0.00, 0.00]  # For gaps or unknown amino acids
+    }
+    feat_list = []
+    for seq in seq_list:
+        feat = [zscale.get(aa, zscale['-']) for aa in seq]
+        feat += [[0] * 5] * (padding - len(seq))
+        feat_list.append(feat)
+    return np.array(feat_list)
+
+
+def one_hot_padding(seq_list, padding):
+    feat_list = []
+    one_hot = {}
+    aa = ['A', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'K', 'L', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'V', 'W', 'Y']
+    for i in range(len(aa)):
+        one_hot[aa[i]] = [0] * 20
+        one_hot[aa[i]][i] = 1
+    for seq in seq_list:
+        feat = [one_hot[aa] for aa in seq] + [[0] * 20] * (padding - len(seq))
+        feat_list.append(feat)
+    return np.array(feat_list)
+
+
+def position_encoding(padding, d=20, b=1000):
+    pos_encoding = np.zeros((padding, d))
+    for pos in range(padding):
+        for i in range(d // 2):
+            pos_encoding[pos, 2 * i] = np.sin(pos / (b ** (2 * i / d)))
+            pos_encoding[pos, 2 * i + 1] = np.cos(pos / (b ** (2 * i / d)))
+    return pos_encoding
+
+
+blosum62 = {
+    'A': [4, -1, -2, -2, 0, -2, -1, 0, -2, -1, -1, -1, -1, -1, -1, 1, 0, 0, -3, -2],
+    'C': [-1, 9, -3, -4, -2, -3, -3, -3, -3, -1, -1, -3, -3, -3, -3, -1, -1, -1, -2, -2],
+    'D': [-2, -3, 6, 2, -3, 1, -1, -1, -1, -4, -3, 1, 0, -3, -3, 0, -1, -3, -4, -3],
+    'E': [-2, -4, 2, 5, -3, 0, -2, -1, 1, -3, -2, 0, 0, -3, -2, 0, -1, -2, -3, -2],
+    'F': [0, -2, -3, -3, 6, -3, -1, -3, -1, 0, 0, -3, -4, -3, -3, -2, -2, -3, 1, 3],
+    'G': [-2, -3, 1, 0, -3, 6, -2, -4, -2, -4, -3, 0, -2, -3, -2, 0, -2, -3, -2, -3],
+    'H': [-1, -3, -1, -2, -1, -2, 8, -3, -1, -3, -2, -1, -2, -1, -2, -2, -2, -2, 2, 2],
+    'I': [0, -3, -1, -1, -3, -4, -3, 4, -3, -3, -3, -1, -3, -3, -3, -3, -1, 2, -3, -1],
+    'K': [-2, -3, -1, 1, -1, -2, -1, -3, 5, -2, -1, 0, -1, 1, 0, -1, 0, -2, -3, -2],
+    'L': [-1, -1, -4, -3, 0, -4, -3, -3, -2, 4, 2, -3, -3, -2, -2, -2, -1, 1, -2, -1],
+    'M': [-1, -1, -3, -2, 0, -3, -2, -3, -1, 2, 5, -2, -2, 0, -1, -2, -1, 1, -1, -1],
+    'N': [-1, -3, 1, 0, -3, 0, -1, -1, 0, -3, -2, 6, 2, -3, -3, 1, 0, -3, -4, -2],
+    'P': [-1, -3, 0, 0, -4, -2, -2, -3, -1, -3, -2, 2, 7, -1, -1, -1, -1, -2, -4, -3],
+    'Q': [-1, -3, -3, -3, -3, -3, -1, -3, 1, -2, 0, -3, -1, 5, 2, -1, 0, -2, -2, -1],
+    'R': [-1, -3, -3, -2, -3, -2, -2, -3, 0, -2, -1, -3, -1, 2, 5, -1, 0, -2, -3, -2],
+    'S': [1, -1, 0, 0, -2, 0, -2, -3, -1, -2, -2, 1, -1, -1, -1, 4, 1, -2, -3, -2],
+    'T': [0, -1, -1, -1, -2, -2, -2, -1, 0, -1, -1, 0, -1, 0, 0, 1, 5, 0, -2, -2],
+    'V': [0, -1, -3, -2, -3, -3, -2, 2, -2, 1, 1, -3, -2, -2, -2, -2, 0, 4, -3, -1],
+    'W': [-3, -2, -4, -3, 1, -2, 2, -3, -3, -2, -1, -4, -4, -2, -3, -3, -2, -3, 11, 2],
+    'Y': [-2, -2, -3, -2, 3, -3, 2, -1, -2, -1, -1, -2, -3, -1, -2, -2, -2, -1, 2, 7],
+    'X': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]  # For unknown amino acids
+}
+
+def blosum62_padding(seq_list, padding):
+    feat_list = []
+    for seq in seq_list:
+        feat = [blosum62.get(aa, blosum62['X']) for aa in seq] + [[0] * 20] * (padding - len(seq))
+        feat_list.append(feat)
+    return np.array(feat_list)
+
+def combined_encoding(seq_list, padding):
+    one_hot_encoded = one_hot_padding(seq_list, padding)
+    pos_encoded = position_encoding(padding)
+    blosum_encoded = blosum62_padding(seq_list, padding)
+    zscale_encoded = zscale_padding(seq_list, padding)
+
+    pos_encoded_expanded = np.expand_dims(pos_encoded, axis=0).repeat(len(seq_list), axis=0)
+
+
+    combined_encoding = np.concatenate(
+        (one_hot_encoded, pos_encoded_expanded, blosum_encoded, zscale_encoded), axis=2
+    )
+    return combined_encoding

model.py

@@ -0,0 +1,240 @@
+
+from cProfile import label
+import tensorflow as tf
+from layers import Attention, MultiHeadAttention
+from tf_keras.models import Model
+from tf_keras.layers import Masking, Dense, LSTM, Bidirectional, Input, Dropout
+
+
+
+def gelu(x):
+    return 0.5 * x * (1.0 + tf.math.erf(x / tf.sqrt(2.)))
+
+
+def scaled_dot_product_attention(q, k, v, mask,adjoin_matrix):
+
+    matmul_qk = tf.matmul(q, k, transpose_b=True)
+
+
+    dk = tf.cast(tf.shape(k)[-1], tf.float32)
+    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
+
+
+    if mask is not None:
+        scaled_attention_logits += (mask * -1e9)
+    if adjoin_matrix is not None:
+        scaled_attention_logits += adjoin_matrix
+
+    attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1) 
+
+    output = tf.matmul(attention_weights, v) 
+
+    return output, attention_weights
+
+
+class MultiHeadAttention(tf.keras.layers.Layer):
+    def __init__(self, d_model, num_heads):
+        super(MultiHeadAttention, self).__init__()
+        self.num_heads = num_heads
+        self.d_model = d_model
+
+        assert d_model % self.num_heads == 0
+
+        self.depth = d_model // self.num_heads
+
+        self.wq = tf.keras.layers.Dense(d_model)
+        self.wk = tf.keras.layers.Dense(d_model)
+        self.wv = tf.keras.layers.Dense(d_model)
+
+        self.dense = tf.keras.layers.Dense(d_model)
+
+    def split_heads(self, x, batch_size):
+        """Split the last dimension into (num_heads, depth).
+        Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth)
+        """
+        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))
+        return tf.transpose(x, perm=[0, 2, 1, 3])
+
+    def call(self, v, k, q, mask,adjoin_matrix):
+        batch_size = tf.shape(q)[0]
+
+        q = self.wq(q) 
+        k = self.wk(k) 
+        v = self.wv(v)  
+
+        q = self.split_heads(q, batch_size)
+        k = self.split_heads(k, batch_size)
+        v = self.split_heads(v, batch_size) 
+
+        scaled_attention, attention_weights = scaled_dot_product_attention(
+            q, k, v, mask,adjoin_matrix)
+
+        scaled_attention = tf.transpose(scaled_attention,
+                                        perm=[0, 2, 1, 3])
+
+        concat_attention = tf.reshape(scaled_attention,
+                                      (batch_size, -1, self.d_model)) 
+
+        output = self.dense(concat_attention)
+
+        return output, attention_weights
+
+def point_wise_feed_forward_network(d_model, dff):
+    return tf.keras.Sequential([
+        tf.keras.layers.Dense(dff, activation=gelu),
+        tf.keras.layers.Dense(d_model) 
+        ])
+
+
+class EncoderLayer(tf.keras.layers.Layer):
+    def __init__(self, d_model, num_heads, dff, rate=0.1):
+        super(EncoderLayer, self).__init__()
+
+        self.mha = MultiHeadAttention(d_model, num_heads)
+        self.ffn = point_wise_feed_forward_network(d_model, dff)
+
+        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
+        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
+
+        self.dropout1 = tf.keras.layers.Dropout(rate)
+        self.dropout2 = tf.keras.layers.Dropout(rate)
+
+    def call(self, x, training, mask,adjoin_matrix):
+        attn_output, attention_weights = self.mha(x, x, x, mask,adjoin_matrix)
+        attn_output = self.dropout1(attn_output, training=training)
+        out1 = self.layernorm1(x + attn_output)
+
+        ffn_output = self.ffn(out1) 
+        ffn_output = self.dropout2(ffn_output, training=training)
+        out2 = self.layernorm2(out1 + ffn_output) 
+
+        return out2,attention_weights
+
+
+
+class Encoder(tf.keras.Model): 
+    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,
+                    maximum_position_encoding, rate=0.1):
+        super(Encoder, self).__init__()
+
+        self.d_model = d_model
+        self.num_layers = num_layers
+
+        self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)
+
+        self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate)
+                           for _ in range(num_layers)]
+
+        self.dropout = tf.keras.layers.Dropout(rate)
+
+    def call(self, x, training, mask,adjoin_matrix):
+        seq_len = tf.shape(x)[1]
+        adjoin_matrix = adjoin_matrix[:,tf.newaxis,:,:]
+        x = self.embedding(x)
+        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
+
+        x = self.dropout(x, training=training)
+
+        for i in range(self.num_layers):
+            x, attention_weights = self.enc_layers[i](x, training=training, mask=mask, adjoin_matrix=adjoin_matrix)
+        return x
+
+class Encoder_test(tf.keras.Model):
+    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,
+                 maximum_position_encoding, rate=0.1):
+        super(Encoder_test, self).__init__()
+
+        self.d_model = d_model
+        self.num_layers = num_layers
+
+        self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)
+
+        self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate)
+                           for _ in range(num_layers)]
+
+        self.dropout = tf.keras.layers.Dropout(rate)
+
+    def call(self, x, training, mask,adjoin_matrix):
+        seq_len = tf.shape(x)[1]
+        adjoin_matrix = adjoin_matrix[:,tf.newaxis,:,:]
+        x = self.embedding(x)
+        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
+
+        x = self.dropout(x, training=training)
+        attention_weights_list = []
+        xs = []
+
+        for i in range(self.num_layers):
+            x,attention_weights = self.enc_layers[i](x, training, mask,adjoin_matrix)
+            attention_weights_list.append(attention_weights)
+            xs.append(x)
+
+        return x,attention_weights_list,xs
+
+class BertModel_test(tf.keras.Model):
+    def __init__(self,num_layers = 6,d_model = 256,dff = 512,num_heads = 8,vocab_size = 18,dropout_rate = 0.1):
+        super(BertModel_test, self).__init__()
+        self.encoder = Encoder_test(num_layers=num_layers,d_model=d_model,
+                        num_heads=num_heads,dff=dff,input_vocab_size=vocab_size,maximum_position_encoding=200,rate=dropout_rate)
+        self.fc1 = tf.keras.layers.Dense(d_model, activation=gelu)
+        self.layernorm = tf.keras.layers.LayerNormalization(-1)
+        self.fc2 = tf.keras.layers.Dense(vocab_size)
+    def call(self,x,adjoin_matrix,mask,training=False):
+        x,att,xs = self.encoder(x,training=training,mask=mask,adjoin_matrix=adjoin_matrix)
+        x = self.fc1(x)
+        x = self.layernorm(x)
+        x = self.fc2(x)
+        return x,att,xs
+
+
+class BertModel(tf.keras.Model):
+    def __init__(self,num_layers = 6,d_model = 256,dff = 512,num_heads = 8,vocab_size = 18,dropout_rate = 0.1):
+        super(BertModel, self).__init__()
+        self.encoder = Encoder(num_layers=num_layers,d_model=d_model,
+                        num_heads=num_heads,dff=dff,input_vocab_size=vocab_size,maximum_position_encoding=200,rate=dropout_rate)
+        self.fc1 = tf.keras.layers.Dense(d_model, activation=gelu)
+        self.layernorm = tf.keras.layers.LayerNormalization(-1)
+        self.fc2 = tf.keras.layers.Dense(vocab_size)
+
+    def call(self,x,adjoin_matrix,mask,training=False):
+        x = self.encoder(x,training=training,mask=mask,adjoin_matrix=adjoin_matrix)
+        x = self.fc1(x)
+        x = self.layernorm(x)
+        x = self.fc2(x)
+        return x
+
+
+class PredictModel(tf.keras.Model):
+    def __init__(self,num_layers = 6,d_model = 256,dff = 512,num_heads = 8,vocab_size =18, a=2, dropout_rate = 0.1,dense_dropout=0.1):
+        super(PredictModel, self).__init__()
+        self.encoder = Encoder(num_layers=num_layers,d_model=d_model,
+                        num_heads=num_heads,dff=dff,input_vocab_size=vocab_size,maximum_position_encoding=200,rate=dropout_rate)
+        self.fc1 = tf.keras.layers.Dense(256,activation=tf.keras.layers.LeakyReLU(0.1))
+        self.dropout1 = tf.keras.layers.Dropout(dense_dropout)
+        self.fc2 = tf.keras.layers.Dense(a)
+        self.t2_processing_network = self.build_layer()
+
+    def build_layer(self, maxlen=71):
+        inputs = Input(shape=(maxlen, 65), name='t2_Input')
+        masking = Masking(mask_value=0.0, input_shape=(maxlen, 65), name='Masking')(inputs)
+        hidden = Bidirectional(LSTM(512, use_bias=True, dropout=0.5, return_sequences=True), name='Bidirectional-LSTM')(
+            masking)
+        hidden = Dropout(0.2, name='Dropout_1')(hidden)
+        hidden = Attention(name='Attention')(hidden)
+        hidden = Dense(512, activation='relu')(hidden)
+
+        model = Model(inputs=inputs, outputs=hidden)
+        return model
+
+    def call(self,x1,adjoin_matrix1,mask1,x2,adjoin_matrix2,mask2,t1,t2, training=False):
+        x1 = self.encoder(x1,training=False,mask=mask1,adjoin_matrix=adjoin_matrix1)
+        x1 = x1[:,0,:]
+        x2 = self.encoder(x2,training=False,mask=mask2,adjoin_matrix=adjoin_matrix2)
+        x2 = x2[:,0,:]
+        x = tf.concat([x1,x2], axis=1)
+        t2 = self.t2_processing_network(t2)
+        x = tf.concat([x, t1, t2], axis=1)
+        x = self.fc1(x)
+        x = self.dropout1(x,training=training)
+        x = self.fc2(x)
+        return x

utils.py

@@ -0,0 +1,110 @@
+from rdkit import Chem
+from rdkit.Chem import rdmolfiles, rdmolops
+import numpy as np
+import openbabel as ob
+import os
+import csv
+from rdkit import RDConfig
+from rdkit.Chem import FragmentCatalog
+
+def fg_list(): # 47 FGs list
+    fName=os.path.join(RDConfig.RDDataDir,'FunctionalGroups.txt')
+    fparams = FragmentCatalog.FragCatParams(1,6,fName)
+    fg_list = []
+    for i in range(fparams.GetNumFuncGroups()):
+        fg_list.append(fparams.GetFuncGroup(i))
+    fg_list.pop(27)
+    
+    x = [Chem.MolToSmiles(_) for _ in fg_list]+['*C=C','*F','*Cl','*Br','*I','[Na+]','*P','*P=O','*[Se]','*[Si]']
+    y = set(x)
+    return list(y)
+
+def obsmitosmile(smi):
+    conv = ob.OBConversion()
+    conv.SetInAndOutFormats("smi", "can")
+    conv.SetOptions("K", conv.OUTOPTIONS)
+    mol = ob.OBMol()
+    conv.ReadString(mol, smi)
+    smile = conv.WriteString(mol)
+    smile = smile.replace('\t\n', '')
+    return smile
+
+def molecular_fg(smiles): # Getting functional groups (including rings) in molecules
+
+    mol = Chem.MolFromSmiles(smiles)
+    if mol is None:
+        print('error')
+        mol = Chem.MolFromSmiles(obsmitosmile(smiles))
+        assert mol is not None, smiles + ' is not valid '
+    a = fg_list()
+
+
+    ssr = Chem.GetSymmSSSR(mol)
+    num_ring = len(ssr)
+    ring_dict = {}
+    for i in range(num_ring):
+        ring_dict[i+1] = list(ssr[i])
+
+    f_g_list = []
+    for i in ring_dict.values():
+        f_g_list.append(i)
+
+    for i in a:
+        patt = Chem.MolFromSmarts(i)
+        flag = mol.HasSubstructMatch(patt)
+        if flag:
+            atomids = mol.GetSubstructMatches(patt)
+            for atomid in atomids:
+                f_g_list.append(list(atomid))
+    return f_g_list
+
+
+def smiles2adjoin(smiles,explicit_hydrogens=True,canonical_atom_order=False): # Converting molecules in SMILES format to atom lists and adjacency matrices
+
+    mol = Chem.MolFromSmiles(smiles)
+    if mol is None:
+        print('error')
+        mol = Chem.MolFromSmiles(obsmitosmile(smiles))
+        assert mol is not None, smiles + ' is not valid '
+
+    if explicit_hydrogens:
+        mol = Chem.AddHs(mol)
+    else:
+        mol = Chem.RemoveHs(mol)
+
+    if canonical_atom_order:
+        new_order = rdmolfiles.CanonicalRankAtoms(mol)
+        mol = rdmolops.RenumberAtoms(mol, new_order)
+    num_atoms = mol.GetNumAtoms()
+
+    atoms_list = []
+    for i in range(num_atoms):
+        atom = mol.GetAtomWithIdx(i)
+        atoms_list.append(atom.GetSymbol())
+
+
+    adjoin_matrix = np.eye(num_atoms)
+    num_bonds = mol.GetNumBonds()
+    
+    for i in range(num_bonds):
+        bond = mol.GetBondWithIdx(i)
+        u = bond.GetBeginAtomIdx()
+        v = bond.GetEndAtomIdx()
+        adjoin_matrix[u,v] = 1.0
+        adjoin_matrix[v,u] = 1.0
+    
+    return atoms_list,adjoin_matrix
+
+
+def get_header(path):
+    with open(path) as f:
+        header = next(csv.reader(f))
+
+    return header
+
+
+def get_task_names(path, use_compound_names=False):
+    index = 2 if use_compound_names else 1
+    task_names = get_header(path)[index:]
+
+    return task_names