import joblib import pickle import gc import string import re from nltk.corpus import stopwords from app.core.config import settings from app.models.models import MLModel from app.core.context import request_id_ctx import logging logger = logging.getLogger(__name__) class MachineLearning: def __init__(self, model: MLModel): self.MAX_SEQUENCE_LENGTH: int = 256 self.model_path: str = { MLModel.naivebayes: settings.ML_NAIVE_BAYES_MODEL_PATH, MLModel.gradientboost: settings.ML_GRADIENT_BOOSTING_MODEL_PATH, MLModel.xgboost: settings.ML_XG_BOOST_MODEL_PATH, }.get(model) if self.model_path is None: logger.error( "inference_model_not_found", extra={ "request_id": request_id_ctx.get('not-set'), "model": model } ) raise ValueError( f"Unknown model: {model}" ) def preprocess(self, text: str) -> str: nep_stopwrods = stopwords.words("nepali") text = ( text.replace("\n", " ") .replace("\r", " ") .replace("\t", " ") .replace("–", " ") .replace("-", " ") .replace("।", " ") .replace("—", " ") .replace("ॽ", " ") .replace("?", " ") .replace("ः", "") .replace(":", " ") .replace("“", ' ') .replace("”", ' ') .replace("‘", " ") .replace("’", " ") .replace('"', " ") .replace("'", " ") .replace("0", "०") .replace("1", "१") .replace("2", "२") .replace("3", "३") .replace("4", "४") .replace("5", "५") .replace("6", "६") .replace("7", "७") .replace("8", "८") .replace("9", "९") ) regrex_pattern = re.compile( pattern="[" "\U0001f600-\U0001f64f" "\U0001f300-\U0001f5ff" "\U0001f680-\U0001f6ff" "\U0001f1e0-\U0001f1ff" "\U00002500-\U00002bef" "\U00002702-\U000027b0" "\U000024c2-\U0001f251" "\U0001f926-\U0001f937" "\U00010000-\U0010ffff" "\u2640-\u2642" "\u2600-\u2b55" "\u200d" "\u23cf" "\u23e9" "\u231a" "\ufe0f" "\u3030" "]+", flags=re.UNICODE, ) text = regrex_pattern.sub("", text) pattern = re.compile(r'<.*?>') text = pattern.sub('', text) text = re.sub(r'([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,})', '', text) text = re.sub(r"\s+", " ", text) text = re.sub('[a-zA-Z]', '', text) text = re.sub(r'\([^)]*\)', '', text) text = " ".join(text.split()) punctuations = list(string.punctuation) text = "".join([char for char in text if char not in punctuations and char not in "०१२३४५६७८९"]) text = [word.strip() for word in text.split() if word.strip() not in nep_stopwrods] text = " ".join(text) text = " ".join(text.split()) return text def predict(self, text: str) -> str: model = joblib.load(settings.ML_NAIVE_BAYES_MODEL_PATH) with open(settings.ML_TFIDF_PATH, 'rb') as f: tfidf = pickle.load(f) with open(settings.ML_LABEL_ENCODER, 'rb') as f: label_encoder = pickle.load(f) text = self.preprocess(text) vectorized = tfidf.transform([text]) pred_label_idx = model.predict(vectorized) pred = label_encoder.inverse_transform(pred_label_idx)[0] del model, label_encoder, tfidf gc.collect() return pred