Update app.py

app.py

@@ -7,32 +7,16 @@ from tensorflow.keras.models import Model
 from tensorflow.keras import models
 from tensorflow.keras.layers import Input,LSTM,Dense
 #translation
-input_texts_translation=[]
-target_texts_translation=[]
-input_characters_translation=set()
-target_characters_translation=set()
-
-with open('word.txt','r',encoding='utf-8') as f:
-    rows=f.read().split('\n')
-    count=0
-for row in rows[:92]:
-    
-    input_text,target_text = row.split('\t')
-    
-    target_text='\t' + target_text + '\n'
-    input_texts_translation.append(input_text.lower())
-    target_texts_translation.append(target_text.lower())
-   
-    input_characters_translation.update(list(input_text.lower()))
-    target_characters_translation.update(list(target_text.lower()))
-input_characters_translation = sorted(list(input_characters_translation))
-target_characters_translation = sorted(list(target_characters_translation))
+input_texts_translation=['again.', 'arrive.', 'bathroom.', 'believe.', 'can.', 'deaf.', 'fine.', 'go.', 'hello.', 'help.', 'home.', 'how.', 'hungury.', 'sorry.', 'call.', 'later.', 'learn.', 'like.', 'live.', 'meet.', 'my.', 'name.', 'nice.', 'no.', 'please.', 'see.', 'share.', 'sign.', 'slow.', 'takecare.', 'talk.', 'thank you.', 'time.', 'understand.', 'we.', 'what.', 'when.', 'where.', 'who.', 'yes.', 'you.', 'you.', 'you.', 'you.', 'name.', 'good.', 'everning.', 'night.', 'how you.', 'you name what.', 'my name.', 'you live where.', 'i.', 'live.', 'you help.', 'yes help me.', 'you understand.', 'i hungry.', 'good everning.', 'good night.', 'pleased.', 'nice meet you.', 'i fine.', 'home arrive when.', 'where bathroom.', 'believe my.', 'call.', 'call.','deaf.', 'i call later.', 'i deaf.', 'what time.', 'i sorry.', 'on my.', 'my.', 'believe.', 'learn.', 'learn.', 'what are you learning.', 'you learn sign where.', 'i go home.', 'i.', 'i.', 'see you later.', 'meet.', 'meet.', 'we meet.', 'like.', 'i like.', 'talk later.', 'later.', 'later.']
+input_characters_translation=[' ', '.', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'k', 'l', 'm', 'n', 'o', 'p', 'r', 's', 't', 'u', 'v', 'w', 'y']
+target_characters_translation=['\t', '\n', ' ', '.', 'ं', 'आ', 'उ', 'क', 'ख', 'ग', 'घ', 'च', 'ज', 'झ', 'ट', 'ड', 'ण', 'त', 'द', 'न', 'प', 'फ', 'ब', 'भ', 'म', 'य', 'र', 'ल', 'ळ', 'व', 'श', 'स', 'ह', 'ा', 'ि', 'ी', 'ु', 'ू', 'े', 'ो', '्']
+
 
-num_en_chars_translation = len(input_characters_translation)
-num_dec_chars_translation = len(target_characters_translation)
+num_en_chars_translation = 24
+num_dec_chars_translation = 41
 
-max_input_length_translation = max([len(i) for i in input_texts_translation])
-max_target_length_translation = max([len(i) for i in target_texts_translation])
+max_input_length_translation = 22
+max_target_length_translation = 37
 
 cv_translation=CountVectorizer(binary=True,tokenizer=lambda txt: txt.split(),stop_words=None,analyzer='char') 
 
@@ -104,41 +88,25 @@ def bagofcharacter_translation(input_t):
     
         return np.array(en_in_data,dtype="float32")
 #transliteration
-input_texts_transliteration=[]
-target_texts_transliteration=[]
-input_characters_transliteration=set()
-target_characters_transliteration=set()
-with open('transliteration_dataset.txt','r',encoding='utf-8') as f:
-    rows=f.read().split('\n')
-    for row in rows[0:30823]:        
-        input_text_transliteration,target_text_transliteration = row.split('\t')   
-        target_text_transliteration='\t' + target_text_transliteration + '\n'
-        input_texts_transliteration.append(input_text_transliteration.lower())
-        target_texts_transliteration.append(target_text_transliteration.lower())    
-        input_characters_transliteration.update(list(input_text_transliteration.lower()))
-        target_characters_transliteration.update(list(target_text_transliteration.lower()))
-input_characters_transliteration = sorted(list(input_characters_transliteration))
-target_characters_transliteration = sorted(list(target_characters_transliteration))
-
-num_en_chars_transliteration = len(input_characters_transliteration)
-num_dec_chars_transliteration = len(target_characters_transliteration)
-max_input_length_transliteration = max([len(i) for i in input_texts_transliteration])
-max_target_length_transliteration = max([len(i) for i in target_texts_transliteration])
+
+input_characters_transliteration=[' ', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u','v', 'w', 'x', 'y', 'z']
+target_characters_transliteration=['\t', '\n', 'ँ', 'ं', 'ः', 'अ', 'आ', 'इ', 'ई', 'उ', 'ऊ', 'ए', 'ऐ', 'ऑ', 'ओ', 'औ', 'क', 'ख', 'ग', 'घ', 'च', 'छ', 'ज', 'झ', 'ट', 'ठ', 'ड', 'ढ', 'ण', 'त', 'थ', 'द', 'ध', 'न', 'प', 'फ', 'ब', 'भ', 'म', 'य', 'र', 'ल', 'ळ', 'व', 'श', 'ष', 'स', 'ह', '़', 'ा', 'ि', 'ी', 'ु', 'ू', 'ृ', 'े','ै', 'ॉ', 'ो', 'ौ', '्', 'क़', 'ख़', 'ग़', 'ज़', 'ड़', 'ढ़', 'फ़']
+num_en_chars_transliteration = 27
+num_dec_chars_transliteration = 68
+max_input_length_transliteration = 21
+max_target_length_transliteration = 20
 cv_transliteration=CountVectorizer(binary=True,tokenizer=lambda txt: txt.split(),stop_words=None,analyzer='char') 
-#Inference model
-#load the model
+
 model_transliteration = models.load_model("s2s_transliteration")
-#construct encoder model from the output of second layer
-#discard the encoder output and store only states.
+
 enc_outputs_transliteration, state_h_enc_transliteration, state_c_enc_transliteration = model_transliteration.layers[2].output 
-#add input object and state from the layer.
+
 en_model_transliteration = Model(model_transliteration.input[0], [state_h_enc_transliteration, state_c_enc_transliteration])
-#create Input object for hidden and cell state for decoder
-#shape of layer with hidden or latent dimension
+
 dec_state_input_h_transliteration = Input(shape=(256,), name="input_6")
 dec_state_input_c_transliteration = Input(shape=(256,), name="input_7")
 dec_states_inputs_transliteration = [dec_state_input_h_transliteration, dec_state_input_c_transliteration]
-#add input from the encoder output and initialize with states.
+
 dec_lstm_transliteration = model_transliteration.layers[3]
 dec_outputs_transliteration, state_h_dec_transliteration, state_c_dec_transliteration = dec_lstm_transliteration(
     model_transliteration.input[1], initial_state=dec_states_inputs_transliteration
@@ -146,51 +114,41 @@ dec_outputs_transliteration, state_h_dec_transliteration, state_c_dec_transliter
 dec_states_transliteration = [state_h_dec_transliteration, state_c_dec_transliteration]
 dec_dense_transliteration = model_transliteration.layers[4]
 dec_outputs_transliteration = dec_dense_transliteration(dec_outputs_transliteration)
-#create Model with the input of decoder state input and encoder input
-#and decoder output with the decoder states.
+
 dec_model_transliteration = Model(
     [model_transliteration.input[1]] + dec_states_inputs_transliteration, [dec_outputs_transliteration] + dec_states_transliteration
 )
 def decode_sequence_transliteration(input_seq):
-        #create a dictionary with a key as index and value as characters.
+       
         reverse_target_char_index_transliteration = dict(enumerate(target_characters_transliteration))
-        #get the states from the user input sequence
+       
         states_value_transliteration = en_model_transliteration.predict(input_seq)
 
-        #fit target characters and 
-        #initialize every first character to be 1 which is '\t'.
-        #Generate empty target sequence of length 1.
         co=cv_transliteration.fit(target_characters_transliteration) 
         target_seq_transliteration=np.array([co.transform(list("\t")).toarray().tolist()],dtype="float32")
 
-        #if the iteration reaches the end of text than it will be stop the it
-        stop_condition = False
-        #append every predicted character in decoded sentence
+        stop_condition = False  
         decoded_sentence = ""
-
-        while not stop_condition:
-            #get predicted output and discard hidden and cell state.
+        while not stop_condition:            
             output_chars, h, c = dec_model_transliteration.predict([target_seq_transliteration] + states_value_transliteration)
-
-            #get the index and from the dictionary get the character.
             char_index = np.argmax(output_chars[0, -1, :])
             text_char = reverse_target_char_index_transliteration[char_index]
             decoded_sentence += text_char
-                # Exit condition: either hit max length
-            # or find a stop character.
+
             if text_char == "\n" or len(decoded_sentence) > max_target_length_transliteration:
                 stop_condition = True
-            #update target sequence to the current character index.
+           
             target_seq_transliteration = np.zeros((1, 1, num_dec_chars_transliteration))
             target_seq_transliteration[0, 0, char_index] = 1.0
             states_value_transliteration = [h, c]
-        #return the decoded sentence
+      
         return decoded_sentence
 def bagofcharacter_transliteration(input_t):
         cv_transliteration=CountVectorizer(binary=True,tokenizer=lambda txt:
         txt.split(),stop_words=None,analyzer='char') 
         en_in_data=[] ; pad_en=[1]+[0]*(len(input_characters_transliteration)-1)
-    
+        print("hi3")
+         
         cv_inp= cv_transliteration.fit(input_characters_transliteration)
         en_in_data.append(cv_inp.transform(list(input_t)).toarray().tolist())
     
@@ -208,10 +166,12 @@ def translate_to_Konkani(sent):
     for x in input_text:
         term=x+"."
         if term in input_texts_translation:
+            print("hi1")
             en_in_data = bagofcharacter_translation( x.lower()+".")    
             x=decode_sequence_translation(en_in_data)
             output_texts+=" "+ x 
         else:
+            print("hi2")
             en_in_data = bagofcharacter_transliteration( x.lower()+".")    
             x=decode_sequence_transliteration(en_in_data)
             output_texts+=" "+ x