saskarkhadka's picture
restructure and add telemtry
9766e10
Raw History Blame Contribute Delete
4 kB
import joblib
import pickle
import gc
import string
import re
from nltk.corpus import stopwords
from app.core.config import settings
from app.models.models import MLModel
from app.core.context import request_id_ctx
import logging
logger = logging.getLogger(__name__)
class MachineLearning:
def __init__(self, model: MLModel):
self.MAX_SEQUENCE_LENGTH: int = 256
self.model_path: str = {
MLModel.naivebayes: settings.ML_NAIVE_BAYES_MODEL_PATH,
MLModel.gradientboost: settings.ML_GRADIENT_BOOSTING_MODEL_PATH,
MLModel.xgboost: settings.ML_XG_BOOST_MODEL_PATH,
}.get(model)
if self.model_path is None:
logger.error(
"inference_model_not_found",
extra={
"request_id": request_id_ctx.get('not-set'),
"model": model
}
)
raise ValueError(
f"Unknown model: {model}"
)
def preprocess(self, text: str) -> str:
nep_stopwrods = stopwords.words("nepali")
text = (
text.replace("\n", " ")
.replace("\r", " ")
.replace("\t", " ")
.replace("–", " ")
.replace("-", " ")
.replace("।", " ")
.replace("—", " ")
.replace("ॽ", " ")
.replace("?", " ")
.replace("ः", "")
.replace(":", " ")
.replace("“", ' ')
.replace("”", ' ')
.replace("‘", " ")
.replace("’", " ")
.replace('"', " ")
.replace("'", " ")
.replace("0", "०")
.replace("1", "१")
.replace("2", "२")
.replace("3", "३")
.replace("4", "४")
.replace("5", "५")
.replace("6", "६")
.replace("7", "७")
.replace("8", "८")
.replace("9", "९")
)
regrex_pattern = re.compile(
pattern="["
"\U0001f600-\U0001f64f"
"\U0001f300-\U0001f5ff"
"\U0001f680-\U0001f6ff"
"\U0001f1e0-\U0001f1ff"
"\U00002500-\U00002bef"
"\U00002702-\U000027b0"
"\U000024c2-\U0001f251"
"\U0001f926-\U0001f937"
"\U00010000-\U0010ffff"
"\u2640-\u2642"
"\u2600-\u2b55"
"\u200d"
"\u23cf"
"\u23e9"
"\u231a"
"\ufe0f"
"\u3030"
"]+",
flags=re.UNICODE,
)
text = regrex_pattern.sub("", text)
pattern = re.compile(r'<.*?>')
text = pattern.sub('', text)
text = re.sub(r'([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,})', '', text)
text = re.sub(r"\s+", " ", text)
text = re.sub('[a-zA-Z]', '', text)
text = re.sub(r'\([^)]*\)', '', text)
text = " ".join(text.split())
punctuations = list(string.punctuation)
text = "".join([char for char in text if char not in punctuations and char not in "०१२३४५६७८९"])
text = [word.strip() for word in text.split() if word.strip() not in nep_stopwrods]
text = " ".join(text)
text = " ".join(text.split())
return text
def predict(self, text: str) -> str:
model = joblib.load(settings.ML_NAIVE_BAYES_MODEL_PATH)
with open(settings.ML_TFIDF_PATH, 'rb') as f:
tfidf = pickle.load(f)
with open(settings.ML_LABEL_ENCODER, 'rb') as f:
label_encoder = pickle.load(f)
text = self.preprocess(text)
vectorized = tfidf.transform([text])
pred_label_idx = model.predict(vectorized)
pred = label_encoder.inverse_transform(pred_label_idx)[0]
del model, label_encoder, tfidf
gc.collect()
return pred