saskarkhadka commited on
Commit
c8a375c
·
1 Parent(s): 5412e5c

add bloomz summarization

Browse files
app/config/config.py CHANGED
@@ -13,10 +13,13 @@ class Settings:
13
  mBART_LORA_SUMMARIZATION_MODEL_PATH: str = os.getenv('mBART_LORA_SUMMARIZATION_MODEL_PATH')
14
  mT5_BASE_MODEL_PATH: str = os.getenv('mT5_BASE_MODEL_PATH')
15
  mT5_LORA_SUMMARIZATION_MODEL_PATH: str = os.getenv('mT5_LORA_SUMMARIZATION_MODEL_PATH')
 
 
16
  TRANSFORMER_SUMMARIZATION_MODEL_PATH: str = os.path.join(ROOT_LEVEL_PATH, 'outputs/transformer/TransformerSummarizationModel.keras')
17
 
18
  mBART_TOKENIZER_PATH: str = os.getenv('mBART_BASE_MODEL_PATH')
19
  mT5_TOKENIZER_PATH: str = os.getenv('mT5_BASE_MODEL_PATH')
 
20
  NEPALI_SUMMARIZATION_TOKENIZER: str = os.path.join(ROOT_LEVEL_PATH, 'outputs/transformer/tokenizer/summarization_52000.model')
21
 
22
  settings = Settings()
 
13
  mBART_LORA_SUMMARIZATION_MODEL_PATH: str = os.getenv('mBART_LORA_SUMMARIZATION_MODEL_PATH')
14
  mT5_BASE_MODEL_PATH: str = os.getenv('mT5_BASE_MODEL_PATH')
15
  mT5_LORA_SUMMARIZATION_MODEL_PATH: str = os.getenv('mT5_LORA_SUMMARIZATION_MODEL_PATH')
16
+ BLOOMZ_LORA_SUMMARIZATION_MODEL_PATH: str = os.getenv('BLOOMZ_LORA_SUMMARIZATION_MODEL_PATH')
17
+ BLOOMZ_BASE_MODEL_PATH: str = os.getenv('BLOOMZ_BASE_MODEL_PATH')
18
  TRANSFORMER_SUMMARIZATION_MODEL_PATH: str = os.path.join(ROOT_LEVEL_PATH, 'outputs/transformer/TransformerSummarizationModel.keras')
19
 
20
  mBART_TOKENIZER_PATH: str = os.getenv('mBART_BASE_MODEL_PATH')
21
  mT5_TOKENIZER_PATH: str = os.getenv('mT5_BASE_MODEL_PATH')
22
+ BLOOMZ_TOKENIZER_PATH: str = os.getenv('BLOOMZ_BASE_MODEL_PATH')
23
  NEPALI_SUMMARIZATION_TOKENIZER: str = os.path.join(ROOT_LEVEL_PATH, 'outputs/transformer/tokenizer/summarization_52000.model')
24
 
25
  settings = Settings()
app/inference/bloomz.py ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import AutoModelForCausalLM, AutoTokenizer
2
+ from peft import PeftModel
3
+ from app.config.config import settings
4
+ from app.preprocess.preprocess import nepali_perprocessing
5
+ import torch
6
+ import gc
7
+
8
+ torch.classes.__path__ = []
9
+
10
+ def tokenize(news: str, tokenizer, news_sequence_length: int=523, title_prompt_sequence_length: int=3):
11
+ title_prompt_ids = tokenizer(
12
+ [" Headline:"],
13
+ max_length=title_prompt_sequence_length,
14
+ padding=False,
15
+ truncation=True,
16
+ add_special_tokens=False
17
+ )
18
+
19
+ inputs = tokenizer(
20
+ news,
21
+ max_length=news_sequence_length,
22
+ add_special_tokens=True,
23
+ truncation=True,
24
+ padding=False
25
+ )
26
+
27
+ input_ids = inputs["input_ids"][0] + title_prompt_ids['input_ids'][0]
28
+ attn_mask = [1] * len(input_ids)
29
+
30
+ return {
31
+ "input_ids": torch.tensor([input_ids]),
32
+ "attention_mask": torch.tensor([attn_mask]),
33
+ }
34
+
35
+ def generate(text: str, input_sequence_length: int=512, output_sequence_length: int=26, title_prompt_sequence_length: int=3, num_beams: int=3, temperature: float=1, top_k: int=50):
36
+ model = AutoModelForCausalLM.from_pretrained(settings.BLOOMZ_BASE_MODEL_PATH)
37
+ tokenizer = AutoTokenizer.from_pretrained(settings.BLOOMZ_BASE_MODEL_PATH)
38
+ model = PeftModel.from_pretrained(model, settings.BLOOMZ_LORA_SUMMARIZATION_MODEL_PATH, token=settings.HF_ACCESS_TOKEN)
39
+ text = nepali_perprocessing(text)
40
+ inputs = tokenize(text, tokenizer, input_sequence_length, title_prompt_sequence_length)
41
+ generated_tokens = model.generate(
42
+ input_ids=inputs["input_ids"],
43
+ attention_mask=inputs["attention_mask"],
44
+ max_new_tokens=output_sequence_length,
45
+ num_beams=num_beams,
46
+ do_sample=True,
47
+ temperature=temperature,
48
+ top_k=top_k,
49
+ )
50
+ input_token_length = inputs["input_ids"].shape[1]
51
+ summary_tokens = generated_tokens[0][input_token_length:]
52
+ summary = tokenizer.decode(summary_tokens, skip_special_tokens=True).strip()
53
+ del model, tokenizer
54
+ gc.collect()
55
+ return summary
app/main.py CHANGED
@@ -3,6 +3,7 @@ import os, uvicorn
3
  from fastapi.middleware.trustedhost import TrustedHostMiddleware
4
  from app.inference import mbart as mbart
5
  from app.inference import mt5 as mt5
 
6
  from app.middleware.auth import AuthMiddleware
7
  from fastapi.middleware.cors import CORSMiddleware
8
  from app.models.models import SummarizationRequest, SummarizationResponse
@@ -25,6 +26,8 @@ def summarize_nepali_news(body: SummarizationRequest) -> SummarizationResponse:
25
  res = mbart.generate(body.text)
26
  elif body.model == 'mt5':
27
  res = mt5.generate(body.text)
 
 
28
  else:
29
  raise HTTPException(400, "Bad Request")
30
 
 
3
  from fastapi.middleware.trustedhost import TrustedHostMiddleware
4
  from app.inference import mbart as mbart
5
  from app.inference import mt5 as mt5
6
+ from app.inference import bloomz as bloomz
7
  from app.middleware.auth import AuthMiddleware
8
  from fastapi.middleware.cors import CORSMiddleware
9
  from app.models.models import SummarizationRequest, SummarizationResponse
 
26
  res = mbart.generate(body.text)
27
  elif body.model == 'mt5':
28
  res = mt5.generate(body.text)
29
+ elif body.model == 'bloomz':
30
+ res = bloomz.generate(body.text, num_beams=body.num_beams, temperature=body.temperature, top_k=body.top_k)
31
  else:
32
  raise HTTPException(400, "Bad Request")
33
 
app/models/models.py CHANGED
@@ -6,10 +6,14 @@ class SummarizationModels(str, Enum):
6
  transformer = "transformer"
7
  mbart = 'mbart'
8
  mt5 = 'mt5'
 
9
 
10
  class SummarizationRequest(BaseModel):
11
  text: str
12
  model: Optional[SummarizationModels] = 'mbart'
 
 
 
13
 
14
  class SummarizationResponse(BaseModel):
15
  prediction: str
 
6
  transformer = "transformer"
7
  mbart = 'mbart'
8
  mt5 = 'mt5'
9
+ blooomz = 'bloomz'
10
 
11
  class SummarizationRequest(BaseModel):
12
  text: str
13
  model: Optional[SummarizationModels] = 'mbart'
14
+ num_beams: Optional[int] = 3
15
+ temperature: Optional[float] = 1.0
16
+ top_k: Optional[int] = 50
17
 
18
  class SummarizationResponse(BaseModel):
19
  prediction: str