saskarkhadka commited on
Commit
554a609
·
1 Parent(s): 185a1ab

remove summarization options

Browse files
.DS_Store ADDED
Binary file (6.15 kB). View file
 
Dockerfile CHANGED
@@ -28,7 +28,6 @@ COPY outputs/bert/label_encoder.pkl ./outputs/bert/
28
  COPY outputs/lstm ./outputs/lstm
29
  COPY outputs/ml ./outputs/ml
30
  COPY outputs/mlp ./outputs/mlp
31
- COPY outputs/transformer ./outputs/transformer
32
 
33
  COPY app ./app
34
  COPY start.sh .
 
28
  COPY outputs/lstm ./outputs/lstm
29
  COPY outputs/ml ./outputs/ml
30
  COPY outputs/mlp ./outputs/mlp
 
31
 
32
  COPY app ./app
33
  COPY start.sh .
app/.DS_Store ADDED
Binary file (6.15 kB). View file
 
app/config/config.py CHANGED
@@ -30,12 +30,4 @@ class Settings:
30
  ML_PARAMETERS_PATH = os.path.join(ROOT_LEVEL_PATH, 'outputs/ml/parameters.json')
31
  MLP_PARAMETERS_PATH = os.path.join(ROOT_LEVEL_PATH, 'outputs/mlp/parameters.json')
32
 
33
- ### Summarization
34
- mBART_BASE_MODEL_PATH: str = os.getenv('mBART_BASE_MODEL_PATH')
35
- mBART_LORA_SUMMARIZATION_MODEL_PATH: str = os.getenv('mBART_LORA_SUMMARIZATION_MODEL_PATH')
36
- TRANSFORMER_SUMMARIZATION_MODEL_PATH: str = os.path.join(ROOT_LEVEL_PATH, 'outputs/transformer/TransformerSummarizationModel.keras')
37
-
38
- mBART_TOKENIZER_PATH: str = os.getenv('mBART_BASE_MODEL_PATH')
39
- NEPALI_SUMMARIZATION_TOKENIZER: str = os.path.join(ROOT_LEVEL_PATH, 'outputs/transformer/tokenizer/summarization_52000.model')
40
-
41
  settings = Settings()
 
30
  ML_PARAMETERS_PATH = os.path.join(ROOT_LEVEL_PATH, 'outputs/ml/parameters.json')
31
  MLP_PARAMETERS_PATH = os.path.join(ROOT_LEVEL_PATH, 'outputs/mlp/parameters.json')
32
 
 
 
 
 
 
 
 
 
33
  settings = Settings()
app/inference/nep_news_summ_mbart.py DELETED
@@ -1,22 +0,0 @@
1
- from transformers import MBartForConditionalGeneration, MBart50Tokenizer
2
- from peft import PeftModel
3
- from app.config.config import settings
4
- from app.preprocess.preprocess import nepali_perprocessing
5
- import torch
6
- import gc
7
-
8
- torch.classes.__path__ = []
9
-
10
- def generate(text: str, input_sequence_length: int=512, output_sequence_length: int=26):
11
- model = MBartForConditionalGeneration.from_pretrained(settings.mBART_BASE_MODEL_PATH)
12
- tokenizer = MBart50Tokenizer.from_pretrained(settings.mBART_BASE_MODEL_PATH)
13
- tokenizer.src_lang = "ne_NP"
14
- tokenizer.tgt_lang = "ne_NP"
15
- model = PeftModel.from_pretrained(model, settings.mBART_LORA_SUMMARIZATION_MODEL_PATH, token=settings.HF_ACCESS_TOKEN)
16
- text = nepali_perprocessing(text, 'summarization')
17
- inputs = tokenizer(text, truncation=True, padding='max_length', max_length=input_sequence_length, return_tensors='pt')
18
- generated_tokens = model.generate(**inputs, forced_bos_token_id=tokenizer.lang_code_to_id['ne_NP'], max_new_tokens=output_sequence_length)
19
- pred = tokenizer.decode(generated_tokens[0], skip_special_tokens=True)
20
- del model, tokenizer
21
- gc.collect()
22
- return pred
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
app/inference/nep_news_summ_transformer.py DELETED
@@ -1,643 +0,0 @@
1
- import tensorflow as tf
2
- from app.config.config import settings
3
- import keras
4
- import sentencepiece as spm
5
-
6
- @keras.saving.register_keras_serializable(package="Embeddings", name="Embeddings")
7
- class Embeddings(tf.keras.layers.Layer):
8
- def __init__(self, d_model: int, seq_len: int, voc_size: int, dropout_rate: float = 0.1, **kwargs):
9
- super(Embeddings, self).__init__(**kwargs)
10
- self.d_model = d_model
11
- self.seq_len = seq_len
12
- self.voc_size = voc_size
13
- self.dropout_rate = dropout_rate
14
-
15
- self.input_emb = tf.keras.layers.Embedding(self.voc_size, self.d_model, name='Sequence_Embedding')
16
- self.positional_emb = tf.keras.layers.Embedding(self.seq_len, self.d_model, name='Positional_Embedding')
17
- self.dropout = tf.keras.layers.Dropout(self.dropout_rate)
18
-
19
- def build(self, input_shape):
20
- self.input_emb.build(input_shape)
21
- self.positional_emb.build(input_shape)
22
- output_shape = self.input_emb.compute_output_shape(tuple(input_shape))
23
- self.dropout.build(output_shape)
24
-
25
- def compute_output_shape(self, input_shape):
26
- return self.input_emb.compute_output_shape(tuple(input_shape))
27
-
28
- def call(self, inputs, training=False):
29
- # inputs -> (batch, seq_len)
30
- positions = tf.repeat(tf.expand_dims(tf.range(tf.shape(inputs)[1]), 0), [tf.shape(inputs)[0]], axis=0) # (batch, seq_len)
31
- inp_emb = self.input_emb(inputs) # (batch, seq_len, d_model)
32
- pos_emb = self.positional_emb(positions) # (batch, seq_len, d_model)
33
-
34
- return self.dropout(inp_emb + pos_emb, training=training) # (batch, seq_len, d_model)
35
-
36
- def get_config(self):
37
- config = super().get_config().copy()
38
- config.update({
39
- 'd_model': self.d_model,
40
- 'seq_len': self.seq_len,
41
- 'voc_size': self.voc_size,
42
- 'dropout_rate': self.dropout_rate,
43
- })
44
- return config
45
-
46
- @classmethod
47
- def from_config(cls, config):
48
- return cls(**config)
49
-
50
- @keras.saving.register_keras_serializable(package="MultiHeadAttention", name="MultiHeadAttention")
51
- class MultiHeadAttention(tf.keras.layers.Layer):
52
- def __init__(self, d_model: int, h: int, dropout_rate: float = 0.1, **kwargs):
53
- super(MultiHeadAttention, self).__init__(**kwargs)
54
- assert d_model % h == 0
55
- self.d_model = d_model
56
- self.h = h
57
- self.d_k = self.d_model // self.h
58
- self.dropout_rate = dropout_rate
59
-
60
- self.w_q = tf.keras.layers.Dense(self.d_model)
61
- self.w_k = tf.keras.layers.Dense(self.d_model)
62
- self.w_v = tf.keras.layers.Dense(self.d_model)
63
- self.w_o = tf.keras.layers.Dense(self.d_model)
64
-
65
- self.dropout = tf.keras.layers.Dropout(self.dropout_rate)
66
-
67
- def build(self, input_shape):
68
- q, k, v = input_shape
69
- self.w_q.build(q)
70
- self.w_k.build(k)
71
- self.w_v.build(v)
72
- self.w_o.build(q)
73
- self.dropout.build(q)
74
-
75
- def compute_output_shape(self, input_shape):
76
- q, k, v = input_shape
77
- return self.dropout.compute_output_shape(q), (q[0], self.h, q[1], k[1])
78
-
79
- def scaled_dot_product_attention(self, q, k, v, mask=None):
80
- attn_score = q @ tf.transpose(k, perm=[0,1,3,2]) / tf.sqrt(tf.cast(k.shape[-1], dtype=tf.float32))
81
- if mask is not None:
82
- # attn_score += (mask * -1e9)
83
- attn_score = tf.where(mask==0, -1e9, attn_score) # Set very small values where mask = 0
84
-
85
- attn_wts = tf.nn.softmax(attn_score, -1) # (batch, h, seq_len, seq_len) seq_len*seq_len because self attention
86
- outputs = attn_wts @ v # (batch, h, seq_len, d_k)
87
- return outputs, attn_wts
88
-
89
- def call(self, q, k, v, mask=None, training=False):
90
- q = self.w_q(q) # (batch, seq_len, d_model)
91
- k = self.w_k(k)
92
- v = self.w_v(v)
93
-
94
- # Convert (batch, seq_len, d_model) to (batch, h, seq_len, d_k)
95
- # Split d_model into h*d_k and then transpose the 2nd and 3rd dimension
96
- q = tf.transpose(tf.reshape(q, [tf.shape(q)[0], tf.shape(q)[1], self.h, self.d_k]), perm=[0,2,1,3])
97
- k = tf.transpose(tf.reshape(k, [tf.shape(k)[0], tf.shape(k)[1], self.h, self.d_k]), perm=[0,2,1,3])
98
- v = tf.transpose(tf.reshape(v, [tf.shape(v)[0], tf.shape(v)[1], self.h, self.d_k]), perm=[0,2,1,3])
99
-
100
- outputs, attn_weights = self.scaled_dot_product_attention(q, k, v, mask)
101
-
102
- # First Convert (batch, h, seq_len, d_k) to (batch, seq_len, d_model)
103
- # Reverse the above operations
104
- # Run through Dense to get (batch, seq_len, d_model)
105
- outputs = self.w_o(tf.reshape(tf.transpose(outputs, perm=[0,2,1,3]), [tf.shape(outputs)[0], tf.shape(outputs)[2], self.d_model]))
106
-
107
- return self.dropout(outputs, training=training), attn_weights
108
-
109
- def get_config(self):
110
- config = super().get_config().copy()
111
- config.update({
112
- 'd_model': self.d_model,
113
- 'h': self.h,
114
- 'dropout_rate': self.dropout_rate,
115
- })
116
- return config
117
-
118
- @classmethod
119
- def from_config(cls, config):
120
- return cls(**config)
121
-
122
- @keras.saving.register_keras_serializable(package="AddAndNorm", name="AddAndNorm")
123
- class AddAndNorm(tf.keras.layers.Layer):
124
- def __init__(self, **kwargs):
125
- super(AddAndNorm, self).__init__(**kwargs)
126
- self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-6)
127
-
128
- def build(self, input_shape):
129
- self.layer_norm.build(input_shape)
130
-
131
- def compute_output_shape(self, input_shape):
132
- return self.layer_norm.compute_output_shape(input_shape)
133
-
134
- def call(self, skip_conn, output):
135
- return self.layer_norm(skip_conn + output) # (batch, seq_len, d_model)
136
-
137
- def get_config(self):
138
- config = super().get_config().copy()
139
- return config
140
-
141
- @classmethod
142
- def from_config(cls, config):
143
- return cls(**config)
144
-
145
- @keras.saving.register_keras_serializable(package="PositionwiseFeedForwardNetwork", name="PositionwiseFeedForwardNetwork")
146
- class PositionwiseFeedForwardNetwork(tf.keras.layers.Layer):
147
- def __init__(self, d_model: int, d_ff: int, dropout_rate: float = 0.1, **kwargs):
148
- super(PositionwiseFeedForwardNetwork, self).__init__(**kwargs)
149
- self.d_model = d_model
150
- self.d_ff = d_ff
151
- self.dropout_rate = dropout_rate
152
-
153
- self.inner = tf.keras.layers.Dense(self.d_ff, activation='relu')
154
- self.outer = tf.keras.layers.Dense(self.d_model)
155
- self.dropout = tf.keras.layers.Dropout(self.dropout_rate)
156
-
157
- def build(self, input_shape):
158
- self.inner.build(input_shape)
159
- output = self.inner.compute_output_shape(input_shape)
160
- self.outer.build(output)
161
- self.dropout.build(self.outer.compute_output_shape(output))
162
-
163
- def compute_output_shape(self, input_shape):
164
- return self.outer.compute_output_shape(self.inner.compute_output_shape(input_shape))
165
-
166
- def call(self, inputs, training=False):
167
- x = self.inner(inputs) # (batch, seq_len, d_ff)
168
- x = self.outer(x) # (batch, seq_len, d_model)
169
- return self.dropout(x, training=training)
170
-
171
- def get_config(self):
172
- config = super().get_config().copy()
173
- config.update({
174
- 'd_model': self.d_model,
175
- 'd_ff': self.d_ff,
176
- 'dropout_rate': self.dropout_rate,
177
- })
178
- return config
179
-
180
- @classmethod
181
- def from_config(cls, config):
182
- return cls(**config)
183
-
184
- @keras.saving.register_keras_serializable(package="EncoderBlock", name="EncoderBlock")
185
- class EncoderBlock(tf.keras.layers.Layer):
186
- def __init__(self, d_model: int, h:int, d_ff: int, dropout_rate: float = 0.1, **kwargs):
187
- super(EncoderBlock, self).__init__(**kwargs)
188
- self.d_model = d_model
189
- self.h = h
190
- self.d_ff = d_ff
191
- self.dropout_rate = dropout_rate
192
-
193
- self.mhsa = MultiHeadAttention(self.d_model, self.h, self.dropout_rate)
194
- self.add_norm1 = AddAndNorm()
195
- self.pffn = PositionwiseFeedForwardNetwork(self.d_model, self.d_ff, self.dropout_rate)
196
- self.add_norm2 = AddAndNorm()
197
-
198
- def build(self, input_shape):
199
- self.mhsa.build([input_shape, input_shape, input_shape])
200
- self.add_norm1.build(input_shape)
201
- self.pffn.build(input_shape)
202
- self.add_norm2.build(input_shape)
203
-
204
- def compute_output_shape(self, input_shape):
205
- return self.mhsa.compute_output_shape([input_shape, input_shape, input_shape])
206
-
207
- def call(self, inputs, mask=None, training=False):
208
- mhsa_outputs, attn_weights = self.mhsa(inputs, inputs, inputs, mask, training=training)
209
- x = self.add_norm1(inputs, mhsa_outputs)
210
- pffn_outputs = self.pffn(x, training=training)
211
- x = self.add_norm2(x, pffn_outputs)
212
-
213
- return x, attn_weights
214
-
215
- def get_config(self):
216
- config = super().get_config().copy()
217
- config.update({
218
- 'd_model': self.d_model,
219
- 'h': self.h,
220
- 'd_ff': self.d_ff,
221
- 'dropout_rate': self.dropout_rate,
222
- })
223
- return config
224
-
225
- @classmethod
226
- def from_config(cls, config):
227
- return cls(**config)
228
-
229
- @keras.saving.register_keras_serializable(package="Encoder", name="Encoder")
230
- class Encoder(tf.keras.layers.Layer):
231
- def __init__(self, N: int, d_model: int, seq_len: int, voc_size: int, h:int, d_ff: int, dropout_rate: float = 0.1, **kwargs):
232
- super(Encoder, self).__init__(**kwargs)
233
- assert N > 0
234
- self.N = N
235
- self.d_model = d_model
236
- self.seq_len = seq_len
237
- self.voc_size = voc_size
238
- self.h = h
239
- self.d_ff = d_ff
240
- self.dropout_rate = dropout_rate
241
-
242
- self.embedding = Embeddings(self.d_model, self.seq_len, self.voc_size, self.dropout_rate)
243
- self.enc_layers = [EncoderBlock(self.d_model, self.h, self.d_ff, self.dropout_rate) for _ in range(self.N)]
244
-
245
- def build(self, input_shape):
246
- self.embedding.build(input_shape)
247
- output = self.embedding.compute_output_shape(tuple(input_shape))
248
- for encoder in self.enc_layers:
249
- encoder.build(output)
250
-
251
- def compute_output_shape(self, input_shape):
252
- return self.enc_layers[0].compute_output_shape(self.embedding.compute_output_shape(input_shape))
253
-
254
- def call(self, inputs, mask=None, training=False):
255
- attn_weights = None
256
- x = self.embedding(inputs, training=training)
257
- for encoder in self.enc_layers:
258
- x, attn_weights = encoder(x, mask=mask, training=training)
259
- return x, attn_weights
260
-
261
- def get_config(self):
262
- config = super().get_config().copy()
263
- config.update({
264
- 'N': self.N,
265
- 'd_model': self.d_model,
266
- 'seq_len': self.seq_len,
267
- 'voc_size': self.voc_size,
268
- 'h': self.h,
269
- 'd_ff': self.d_ff,
270
- 'dropout_rate': self.dropout_rate,
271
- })
272
- return config
273
-
274
- @classmethod
275
- def from_config(cls, config):
276
- return cls(**config)
277
-
278
- @keras.saving.register_keras_serializable(package="DecoderBlock", name="DecoderBlock")
279
- class DecoderBlock(tf.keras.layers.Layer):
280
- def __init__(self, d_model: int, h:int, d_ff: int, dropout_rate: float = 0.1, **kwargs):
281
- super(DecoderBlock, self).__init__(**kwargs)
282
- self.d_model = d_model
283
- self.h = h
284
- self.d_ff = d_ff
285
- self.dropout_rate = dropout_rate
286
-
287
- self.mhsa = MultiHeadAttention(self.d_model, self.h, self.dropout_rate)
288
- self.add_norm1 = AddAndNorm()
289
- self.mhca = MultiHeadAttention(self.d_model, self.h, self.dropout_rate)
290
- self.add_norm2 = AddAndNorm()
291
- self.pffn = PositionwiseFeedForwardNetwork(self.d_model, self.d_ff, self.dropout_rate)
292
- self.add_norm3 = AddAndNorm()
293
-
294
- def build(self, input_shape):
295
- dec_input_shape, enc_output_shape = input_shape
296
- self.mhsa.build([dec_input_shape, dec_input_shape, dec_input_shape])
297
- self.add_norm1.build(dec_input_shape)
298
- self.mhca.build([dec_input_shape, enc_output_shape, enc_output_shape])
299
- self.add_norm2.build(dec_input_shape)
300
- self.pffn.build(dec_input_shape)
301
- self.add_norm3.build(dec_input_shape)
302
-
303
- def compute_output_shape(self, input_shape):
304
- dec_input_shape, enc_output_shape = input_shape
305
- return self.mhca.compute_output_shape([dec_input_shape, enc_output_shape, enc_output_shape])
306
-
307
- def call(self, inputs, encoder_outputs, decoder_mask=None, encoder_mask=None, training=False):
308
- mhsa_outputs, _ = self.mhsa(inputs, inputs, inputs, mask=decoder_mask, training=training)
309
- x = self.add_norm1(inputs, mhsa_outputs)
310
- mhca_outputs, attn_weights = self.mhca(x, encoder_outputs, encoder_outputs, mask=encoder_mask, training=training)
311
- x = self.add_norm2(x, mhca_outputs)
312
- pffn_outputs = self.pffn(x, training=training)
313
- x = self.add_norm3(x, pffn_outputs)
314
-
315
- return x, attn_weights
316
-
317
- def get_config(self):
318
- config = super().get_config().copy()
319
- config.update({
320
- 'd_model': self.d_model,
321
- 'h': self.h,
322
- 'd_ff': self.d_ff,
323
- 'dropout_rate': self.dropout_rate,
324
- })
325
- return config
326
-
327
- @classmethod
328
- def from_config(cls, config):
329
- return cls(**config)
330
-
331
- @keras.saving.register_keras_serializable(package="Decoder", name="Decoder")
332
- class Decoder(tf.keras.layers.Layer):
333
- def __init__(self, N: int, d_model: int, seq_len: int, voc_size: int, h:int, d_ff: int, dropout_rate: float = 0.1, **kwargs):
334
- super(Decoder, self).__init__(**kwargs)
335
- assert N > 0
336
- self.N = N
337
- self.d_model = d_model
338
- self.seq_len = seq_len
339
- self.voc_size = voc_size
340
- self.h = h
341
- self.d_ff = d_ff
342
- self.dropout_rate = dropout_rate
343
-
344
- self.embedding = Embeddings(self.d_model, self.seq_len, self.voc_size, self.dropout_rate)
345
- self.dec_layers = [DecoderBlock(self.d_model, self.h, self.d_ff, self.dropout_rate) for _ in range(self.N)]
346
-
347
- def build(self, input_shape):
348
- dec_input_shape, enc_output_shape = input_shape
349
- self.embedding.build(dec_input_shape)
350
- output = self.embedding.compute_output_shape(dec_input_shape)
351
- for decoder in self.dec_layers:
352
- decoder.build([output, enc_output_shape])
353
-
354
- def compute_output_shape(self, input_shape):
355
- dec_input_shape, enc_output_shape = input_shape
356
- return self.dec_layers[0].compute_output_shape([self.embedding.compute_output_shape(dec_input_shape), enc_output_shape])
357
-
358
- def call(self, inputs, encoder_outputs, decoder_mask=None, encoder_mask=None, training=False):
359
- attn_weights = None
360
- x = self.embedding(inputs, training=training)
361
- for decoder in self.dec_layers:
362
- x, attn_weights = decoder(x, encoder_outputs, decoder_mask=decoder_mask, encoder_mask=encoder_mask, training=training)
363
- return x, attn_weights
364
-
365
- def get_config(self):
366
- config = super().get_config().copy()
367
- config.update({
368
- 'N': self.N,
369
- 'd_model': self.d_model,
370
- 'seq_len': self.seq_len,
371
- 'voc_size': self.voc_size,
372
- 'h': self.h,
373
- 'd_ff': self.d_ff,
374
- 'dropout_rate': self.dropout_rate,
375
- })
376
- return config
377
-
378
- @classmethod
379
- def from_config(cls, config):
380
- return cls(**config)
381
-
382
- @keras.saving.register_keras_serializable(package="Transformer", name="Transformer")
383
- class Transformer(tf.keras.layers.Layer):
384
- def __init__(
385
- self,
386
- encoder_layers: int,
387
- decoder_layers: int,
388
- d_model: int,
389
- encoder_seq_len: int,
390
- decoder_seq_len: int,
391
- encoder_voc_size: int,
392
- decoder_voc_size: int,
393
- encoder_attention_heads: int,
394
- decoder_attention_heads: int,
395
- encoder_ffn_dim: int,
396
- decoder_ffn_dim: int,
397
- dropout_rate: float = 0.1,
398
- **kwargs
399
- ):
400
- super(Transformer, self).__init__(**kwargs)
401
-
402
- self.encoder_layers = encoder_layers
403
- self.decoder_layers = decoder_layers
404
- assert self.encoder_layers > 0 and self.decoder_layers > 0, "Encoder and Decoder must have atleast 1 layer"
405
-
406
- self.d_model = d_model
407
- self.encoder_seq_len = encoder_seq_len
408
- self.decoder_seq_len = decoder_seq_len
409
- self.encoder_voc_size = encoder_voc_size
410
- self.decoder_voc_size = decoder_voc_size
411
- self.encoder_attention_heads = encoder_attention_heads
412
- self.decoder_attention_heads = decoder_attention_heads
413
- self.encoder_ffn_dim = encoder_ffn_dim
414
- self.decoder_ffn_dim = decoder_ffn_dim
415
- self.dropout_rate = dropout_rate
416
-
417
- self.encoder = Encoder(self.encoder_layers, self.d_model, self.encoder_seq_len, self.encoder_voc_size, self.encoder_attention_heads, self.encoder_ffn_dim, self.dropout_rate)
418
- self.decoder = Decoder(self.decoder_layers, self.d_model, self.decoder_seq_len, self.decoder_voc_size, self.decoder_attention_heads, self.decoder_ffn_dim, self.dropout_rate)
419
- self.projection = tf.keras.layers.Dense(decoder_voc_size)
420
-
421
- def build(self, input_shape):
422
- enc_input_shape, dec_input_shape = input_shape
423
- self.encoder.build(enc_input_shape)
424
- enc_output, _ = self.encoder.compute_output_shape(enc_input_shape)
425
- self.decoder.build([dec_input_shape, enc_output])
426
- dec_output, _ = self.decoder.compute_output_shape([dec_input_shape, enc_output])
427
- self.projection.build(dec_output)
428
-
429
- def compute_output_shape(self, input_shape):
430
- enc_input_shape, dec_input_shape = input_shape
431
- enc_output, enc_attn = self.encoder.compute_output_shape(enc_input_shape)
432
- dec_output, dec_attn = self.decoder.compute_output_shape([dec_input_shape, enc_output])
433
- return self.projection.compute_output_shape(dec_output), enc_attn, dec_attn
434
-
435
- def call(self, encoder_inputs, decoder_inputs, encoder_mask=None, decoder_mask=None, training=False):
436
- enc_outputs, enc_attn_wts = self.encoder(encoder_inputs, mask=encoder_mask, training=training)
437
- dec_outputs, dec_attn_wts = self.decoder(decoder_inputs, enc_outputs, encoder_mask=encoder_mask, decoder_mask=decoder_mask, training=training)
438
- logits = self.projection(dec_outputs)
439
-
440
- return logits, enc_attn_wts, dec_attn_wts
441
-
442
- def get_config(self):
443
- config = super().get_config().copy()
444
- config.update({
445
- 'encoder_layers': self.encoder_layers,
446
- 'decoder_layers': self.decoder_layers,
447
- 'd_model': self.d_model,
448
- 'encoder_seq_len': self.encoder_seq_len,
449
- 'decoder_seq_len': self.decoder_seq_len,
450
- 'encoder_voc_size': self.encoder_voc_size,
451
- 'decoder_voc_size': self.decoder_voc_size,
452
- 'encoder_attention_heads': self.encoder_attention_heads,
453
- 'decoder_attention_heads': self.decoder_attention_heads,
454
- 'encoder_ffn_dim': self.encoder_ffn_dim,
455
- 'decoder_ffn_dim': self.decoder_ffn_dim,
456
- 'dropout_rate': self.dropout_rate,
457
- })
458
- return config
459
-
460
- @classmethod
461
- def from_config(cls, config):
462
- return cls(**config)
463
-
464
- @keras.saving.register_keras_serializable(package="TransformerTrainer", name="TransformerTrainer")
465
- class TransformerTrainer(tf.keras.Model):
466
- def __init__(self, transformer: Transformer, encoder_seq_len: int, decoder_seq_len: int, sos_token_id: int, eos_token_id: int, label_smoothing: float = 0.1, **kwargs):
467
- super(TransformerTrainer, self).__init__(**kwargs)
468
- self.transformer = transformer
469
- self.label_smoothing = label_smoothing
470
- self.encoder_seq_len = encoder_seq_len
471
- self.decoder_seq_len = decoder_seq_len
472
- self.sos_token_id = sos_token_id
473
- self.eos_token_id = eos_token_id
474
- self.loss_tracker = tf.keras.metrics.Mean(name="loss")
475
- self.val_loss_tracker = tf.keras.metrics.Mean(name="val_loss")
476
-
477
- def build(self, input_shape):
478
- enc_input, dec_input, dec_output = input_shape
479
- if not self.transformer.built:
480
- self.transformer.build([enc_input, dec_input])
481
-
482
- def compute_output_shape(self, input_shape):
483
- enc_input, dec_input, dec_output = input_shape
484
- return self.transformer.compute_output_shape([enc_input, dec_input])
485
-
486
- def compute_padding_mask(self, inp):
487
- mask = tf.cast(tf.math.not_equal(inp, 0), tf.float32)
488
- mask = mask[:, tf.newaxis, tf.newaxis, :]
489
- return mask
490
-
491
- def compute_padding_lookahead_mask(self, decoder_inp):
492
- mask = tf.cast(tf.math.equal(decoder_inp, 0), tf.float32)
493
- mask = mask[:, tf.newaxis, tf.newaxis, :]
494
- return tf.cast(tf.maximum(mask, 1 - tf.linalg.band_part(tf.ones((decoder_inp.shape[-1], decoder_inp.shape[-1])), -1, 0)) == 0, tf.float32)
495
-
496
- def call(self, inputs):
497
- encoder_mask = self.compute_padding_mask(inputs)
498
-
499
- decoder_input = tf.fill([1,1], self.sos_token_id)
500
- enc_outputs, enc_attn_wts = self.transformer.encoder(inputs, mask=encoder_mask, training=False)
501
-
502
- for t in range(self.decoder_seq_len):
503
- decoder_mask = self.compute_padding_lookahead_mask(decoder_input)
504
- dec_outputs, dec_attn_wts = self.transformer.decoder(decoder_input, enc_outputs, encoder_mask=encoder_mask, decoder_mask=decoder_mask, training=False)
505
- final_dist = self.transformer.projection(dec_outputs[:,-1])
506
- curr_output = tf.expand_dims(tf.argmax(final_dist, -1, output_type=tf.int32), 1)
507
- if curr_output[0] == self.eos_token_id:
508
- break
509
- decoder_input = tf.concat([decoder_input, curr_output], -1)
510
-
511
- return tf.squeeze(decoder_input, 0)
512
-
513
- @tf.function
514
- def train_step(self, inputs):
515
- encoder_inputs, decoder_inputs, targets = inputs
516
-
517
- loss = None
518
- encoder_mask = self.compute_padding_mask(encoder_inputs)
519
- decoder_mask = self.compute_padding_lookahead_mask(decoder_inputs)
520
-
521
- with tf.GradientTape() as tape:
522
- logits, _, _ = self.transformer(encoder_inputs, decoder_inputs, encoder_mask, decoder_mask, training=True)
523
- loss = self.loss(targets, logits, tf.cast(tf.math.not_equal(targets, 0), tf.float32), self.label_smoothing)
524
-
525
- gradients = tape.gradient(loss, self.transformer.trainable_variables)
526
- self.optimizer.apply_gradients(zip(gradients, self.transformer.trainable_variables))
527
-
528
- self.loss_tracker.update_state(loss)
529
-
530
- return {'loss': self.loss_tracker.result()}
531
-
532
- @tf.function
533
- def test_step(self, inputs):
534
- encoder_inputs, decoder_inputs, targets = inputs
535
- encoder_mask = self.compute_padding_mask(encoder_inputs)
536
- decoder_mask = self.compute_padding_lookahead_mask(decoder_inputs)
537
-
538
- logits, _, _ = self.transformer(encoder_inputs, decoder_inputs, encoder_mask, decoder_mask, training=False)
539
- loss = self.loss(targets, logits, mask=tf.cast(tf.math.not_equal(targets, 0), tf.float32), label_smoothing=0.0)
540
-
541
- self.val_loss_tracker.update_state(loss)
542
-
543
- return {'loss': self.val_loss_tracker.result()}
544
-
545
- def get_config(self):
546
- config = super().get_config().copy()
547
- config.update({
548
- 'transformer': self.transformer,
549
- 'label_smoothing': self.label_smoothing,
550
- 'encoder_seq_len': self.encoder_seq_len,
551
- 'decoder_seq_len': self.decoder_seq_len,
552
- 'sos_token_id': self.sos_token_id,
553
- 'eos_token_id': self.eos_token_id
554
- })
555
- return config
556
-
557
- @classmethod
558
- def from_config(cls, config):
559
- config["transformer"] = tf.keras.layers.deserialize(config["transformer"])
560
- return cls(**config)
561
-
562
- def get_build_config(self):
563
- return {
564
- "input_shape": ((None, self.encoder_seq_len), (None, self.decoder_seq_len), (None, self.decoder_seq_len))
565
- }
566
-
567
- def build_from_config(self, config):
568
- self.build(config["input_shape"])
569
-
570
- @keras.saving.register_keras_serializable(package="WarmupSchedule", name="WarmupSchedule")
571
- class WarmupSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):
572
- def __init__(self, d_model: int, warmup_steps: int=4_000):
573
- super().__init__()
574
- self.d_model = d_model
575
- self.warmup_steps = warmup_steps
576
-
577
- def __call__(self, step):
578
- step = tf.cast(step, tf.float32)
579
- return tf.math.rsqrt(tf.cast(self.d_model, tf.float32)) * tf.math.minimum(tf.math.rsqrt(step), step * tf.math.pow(tf.cast(self.warmup_steps, tf.float32), -1.5))
580
-
581
- def get_config(self):
582
- config = {
583
- 'd_model': self.d_model,
584
- 'warmup_steps': self.warmup_steps,
585
- }
586
- return config
587
-
588
- @classmethod
589
- def from_config(cls, config):
590
- return cls(**config)
591
-
592
- class LearningRateLoggerCallback(tf.keras.callbacks.Callback):
593
- def __init__(self):
594
- super().__init__()
595
-
596
- def on_batch_begin(self, batch, logs=None):
597
- lr = self.model.optimizer.learning_rate
598
- if isinstance(lr, tf.keras.optimizers.schedules.LearningRateSchedule):
599
- step = self.model.optimizer.iterations.numpy()
600
- current_lr = lr(step).numpy()
601
- else:
602
- current_lr = lr.numpy()
603
- print(f"Step {batch}, Learnnig Rate: {current_lr:.6f}")
604
-
605
- @keras.saving.register_keras_serializable(package="cross_entropy_loss", name="cross_entropy_loss")
606
- def cross_entropy_loss(targets, output_dist, mask=None, label_smoothing=0.1):
607
- targets = tf.keras.utils.to_categorical(targets, num_classes=output_dist.shape[-1])
608
- scce_loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True, reduction=None, label_smoothing=label_smoothing)
609
- step_scce_loss = scce_loss(targets, output_dist)
610
- if mask is not None:
611
- step_scce_loss = tf.reduce_mean(tf.math.divide_no_nan(tf.reduce_sum(step_scce_loss*mask, 1), tf.reduce_sum(mask, 1)))
612
- return step_scce_loss
613
-
614
- def generate(news: str, encoder_seq_len: int=384):
615
- model = model = tf.keras.models.load_model(settings.TRANSFORMER_SUMMARIZATION_MODEL_PATH,
616
- custom_objects= {
617
- 'Embeddings': Embeddings,
618
- 'MultiHeadAttention': MultiHeadAttention,
619
- 'AddAndNorm': AddAndNorm,
620
- 'PositionwiseFeedForwardNetwork': PositionwiseFeedForwardNetwork,
621
- 'EncoderBlock': EncoderBlock,
622
- 'Encoder': Encoder,
623
- 'DecoderBlock': DecoderBlock,
624
- 'Decoder': Decoder,
625
- 'Transformer': Transformer,
626
- 'cross_entropy_loss': cross_entropy_loss,
627
- 'TransformerTrainer': TransformerTrainer,
628
- 'WarmupSchedule': WarmupSchedule
629
- }
630
- )
631
- tokenizer = spm.SentencePieceProcessor()
632
- tokenizer.load(settings.NEPALI_SUMMARIZATION_TOKENIZER)
633
-
634
- news_encoded = tokenizer.encode(news)
635
-
636
- if len(news_encoded) >= encoder_seq_len:
637
- news_encoded = news_encoded[:encoder_seq_len]
638
- else:
639
- news_encoded = news_encoded + [tokenizer.pad_id()] * (encoder_seq_len - len(news_encoded))
640
-
641
- news_encoded = tf.convert_to_tensor([news_encoded])
642
-
643
- return tokenizer.decode(model(news_encoded).numpy().tolist())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
app/main.py CHANGED
@@ -3,13 +3,11 @@ import os, uvicorn
3
  from fastapi.middleware.trustedhost import TrustedHostMiddleware
4
  from app.middleware.auth import AuthMiddleware
5
 
6
- from app.models.models import ClassificationRequest, ClassificationResponse, SummarizationRequest, SummarizationResponse
7
  from app.inference import nep_news_cls_bert as bert, \
8
  nep_news_cls_lstm as lstm, \
9
  nep_news_cls_ml as ml, \
10
- nep_news_cls_mlp as mlp, \
11
- nep_news_summ_mbart as mbart, \
12
- nep_news_summ_transformer as transformer
13
 
14
  app = FastAPI()
15
 
@@ -35,18 +33,6 @@ def classify_nepali_news(body: ClassificationRequest) -> ClassificationResponse:
35
 
36
  return ClassificationResponse(prediction=res)
37
 
38
- @app.post('/summarize/nepali-news', status_code=200, response_model=SummarizationResponse)
39
- def classify_nepali_news(body: SummarizationRequest) -> SummarizationResponse:
40
-
41
- if body.model == 'transformer':
42
- res = transformer.generate(body.text)
43
- elif body.model == 'mbart':
44
- res = mbart.generate(body.text)
45
- else:
46
- raise HTTPException(400, "Bad Request")
47
-
48
- return SummarizationResponse(prediction=res)
49
-
50
  if __name__ == "__main__":
51
  port = int(os.environ.get("PORT", 8000))
52
  uvicorn.run(app, host="0.0.0.0", port=port)
 
3
  from fastapi.middleware.trustedhost import TrustedHostMiddleware
4
  from app.middleware.auth import AuthMiddleware
5
 
6
+ from app.models.models import ClassificationRequest, ClassificationResponse
7
  from app.inference import nep_news_cls_bert as bert, \
8
  nep_news_cls_lstm as lstm, \
9
  nep_news_cls_ml as ml, \
10
+ nep_news_cls_mlp as mlp
 
 
11
 
12
  app = FastAPI()
13
 
 
33
 
34
  return ClassificationResponse(prediction=res)
35
 
 
 
 
 
 
 
 
 
 
 
 
 
36
  if __name__ == "__main__":
37
  port = int(os.environ.get("PORT", 8000))
38
  uvicorn.run(app, host="0.0.0.0", port=port)
app/models/models.py CHANGED
@@ -15,13 +15,4 @@ class ClassificationRequest(BaseModel):
15
  model: Optional[ClassificationModels] = "mlp"
16
 
17
  class ClassificationResponse(BaseModel):
18
- prediction: str
19
-
20
- class SummarizationModels(str, Enum):
21
- vtransformer = "transformer"
22
- mbart = 'mbart'
23
- class SummarizationRequest(BaseModel):
24
- text: str
25
- model: Optional[SummarizationModels] = 'mbart'
26
- class SummarizationResponse(BaseModel):
27
  prediction: str
 
15
  model: Optional[ClassificationModels] = "mlp"
16
 
17
  class ClassificationResponse(BaseModel):
 
 
 
 
 
 
 
 
 
18
  prediction: str
app/preprocess/preprocess.py CHANGED
@@ -2,47 +2,26 @@ import string
2
  import re
3
  from nltk.corpus import stopwords
4
 
5
- def nepali_perprocessing(text, task='classification'):
6
  nep_stopwrods = stopwords.words("nepali")
7
- if task == 'classification':
8
- text = (
9
- text.replace("\n", " ")
10
- .replace("\r", " ")
11
- .replace("\t", " ")
12
- .replace("–", " ")
13
- .replace("-", " ")
14
- .replace("।", " ")
15
- .replace("—", " ")
16
- .replace("ॽ", " ")
17
- .replace("?", " ")
18
- .replace("ः", "")
19
- .replace(":", " ")
20
- .replace("“", ' ')
21
- .replace("”", ' ')
22
- .replace("‘", " ")
23
- .replace("’", " ")
24
- .replace('"', " ")
25
- .replace("'", " ")
26
- .replace("0", "०")
27
- .replace("1", "१")
28
- .replace("2", "२")
29
- .replace("3", "३")
30
- .replace("4", "४")
31
- .replace("5", "५")
32
- .replace("6", "६")
33
- .replace("7", "७")
34
- .replace("8", "८")
35
- .replace("9", "९")
36
- )
37
- else:
38
- text = (
39
  text.replace("\n", " ")
40
  .replace("\r", " ")
41
  .replace("\t", " ")
42
- .replace("“", '"')
43
- .replace("”", '"')
44
- .replace("‘", "'")
45
- .replace("’", "'")
 
 
 
 
 
 
 
 
 
 
46
  .replace("0", "०")
47
  .replace("1", "१")
48
  .replace("2", "२")
@@ -54,7 +33,6 @@ def nepali_perprocessing(text, task='classification'):
54
  .replace("8", "८")
55
  .replace("9", "९")
56
  )
57
-
58
  # Remove emojis
59
  regrex_pattern = re.compile(
60
  pattern="["
@@ -95,12 +73,11 @@ def nepali_perprocessing(text, task='classification'):
95
 
96
  text = " ".join(text.split())
97
 
98
- if task == 'classification':
99
- punctuations = list(string.punctuation)
100
- text = "".join([char for char in text if char not in punctuations and char not in "०१२३४५६७८९"])
101
- text = [word.strip() for word in text.split() if word.strip() not in nep_stopwrods]
102
- text = " ".join(text)
103
- text = " ".join(text.split())
104
 
105
  return text
106
 
 
2
  import re
3
  from nltk.corpus import stopwords
4
 
5
+ def nepali_perprocessing(text):
6
  nep_stopwrods = stopwords.words("nepali")
7
+ text = (
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8
  text.replace("\n", " ")
9
  .replace("\r", " ")
10
  .replace("\t", " ")
11
+ .replace("–", " ")
12
+ .replace("-", " ")
13
+ .replace("।", " ")
14
+ .replace("—", " ")
15
+ .replace("ॽ", " ")
16
+ .replace("?", " ")
17
+ .replace("ः", "")
18
+ .replace(":", " ")
19
+ .replace("“", ' ')
20
+ .replace("”", ' ')
21
+ .replace("‘", " ")
22
+ .replace("’", " ")
23
+ .replace('"', " ")
24
+ .replace("'", " ")
25
  .replace("0", "०")
26
  .replace("1", "१")
27
  .replace("2", "२")
 
33
  .replace("8", "८")
34
  .replace("9", "९")
35
  )
 
36
  # Remove emojis
37
  regrex_pattern = re.compile(
38
  pattern="["
 
73
 
74
  text = " ".join(text.split())
75
 
76
+ punctuations = list(string.punctuation)
77
+ text = "".join([char for char in text if char not in punctuations and char not in "०१२३४५६७८९"])
78
+ text = [word.strip() for word in text.split() if word.strip() not in nep_stopwrods]
79
+ text = " ".join(text)
80
+ text = " ".join(text.split())
 
81
 
82
  return text
83
 
outputs/transformer/tokenizer/summarization_52000.model DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:5eb27b8fb8cc3a41e3fd67a51628a689b828f6e46a006cc8ec4ec6d8640e8952
3
- size 1800880
 
 
 
 
outputs/transformer/tokenizer/summarization_52000.vocab DELETED
The diff for this file is too large to render. See raw diff