Download app/inference/ml.py from saskarkhadka/classification-backend: direct link, hf CLI and curl.
- Browser
- Download file 4 kB
-
https://huggingface.co/spaces/saskarkhadka/classification-backend/resolve/main/app/inference/ml.py
- Command line
-
hf download hf://spaces/saskarkhadka/classification-backend/app/inference/ml.py
-
curl -L -o ml.py https://huggingface.co/spaces/saskarkhadka/classification-backend/resolve/main/app/inference/ml.py
4 kB
| import joblib | |
| import pickle | |
| import gc | |
| import string | |
| import re | |
| from nltk.corpus import stopwords | |
| from app.core.config import settings | |
| from app.models.models import MLModel | |
| from app.core.context import request_id_ctx | |
| import logging | |
| logger = logging.getLogger(__name__) | |
| class MachineLearning: | |
| def __init__(self, model: MLModel): | |
| self.MAX_SEQUENCE_LENGTH: int = 256 | |
| self.model_path: str = { | |
| MLModel.naivebayes: settings.ML_NAIVE_BAYES_MODEL_PATH, | |
| MLModel.gradientboost: settings.ML_GRADIENT_BOOSTING_MODEL_PATH, | |
| MLModel.xgboost: settings.ML_XG_BOOST_MODEL_PATH, | |
| }.get(model) | |
| if self.model_path is None: | |
| logger.error( | |
| "inference_model_not_found", | |
| extra={ | |
| "request_id": request_id_ctx.get('not-set'), | |
| "model": model | |
| } | |
| ) | |
| raise ValueError( | |
| f"Unknown model: {model}" | |
| ) | |
| def preprocess(self, text: str) -> str: | |
| nep_stopwrods = stopwords.words("nepali") | |
| text = ( | |
| text.replace("\n", " ") | |
| .replace("\r", " ") | |
| .replace("\t", " ") | |
| .replace("–", " ") | |
| .replace("-", " ") | |
| .replace("।", " ") | |
| .replace("—", " ") | |
| .replace("ॽ", " ") | |
| .replace("?", " ") | |
| .replace("ः", "") | |
| .replace(":", " ") | |
| .replace("“", ' ') | |
| .replace("”", ' ') | |
| .replace("‘", " ") | |
| .replace("’", " ") | |
| .replace('"', " ") | |
| .replace("'", " ") | |
| .replace("0", "०") | |
| .replace("1", "१") | |
| .replace("2", "२") | |
| .replace("3", "३") | |
| .replace("4", "४") | |
| .replace("5", "५") | |
| .replace("6", "६") | |
| .replace("7", "७") | |
| .replace("8", "८") | |
| .replace("9", "९") | |
| ) | |
| regrex_pattern = re.compile( | |
| pattern="[" | |
| "\U0001f600-\U0001f64f" | |
| "\U0001f300-\U0001f5ff" | |
| "\U0001f680-\U0001f6ff" | |
| "\U0001f1e0-\U0001f1ff" | |
| "\U00002500-\U00002bef" | |
| "\U00002702-\U000027b0" | |
| "\U000024c2-\U0001f251" | |
| "\U0001f926-\U0001f937" | |
| "\U00010000-\U0010ffff" | |
| "\u2640-\u2642" | |
| "\u2600-\u2b55" | |
| "\u200d" | |
| "\u23cf" | |
| "\u23e9" | |
| "\u231a" | |
| "\ufe0f" | |
| "\u3030" | |
| "]+", | |
| flags=re.UNICODE, | |
| ) | |
| text = regrex_pattern.sub("", text) | |
| pattern = re.compile(r'<.*?>') | |
| text = pattern.sub('', text) | |
| text = re.sub(r'([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,})', '', text) | |
| text = re.sub(r"\s+", " ", text) | |
| text = re.sub('[a-zA-Z]', '', text) | |
| text = re.sub(r'\([^)]*\)', '', text) | |
| text = " ".join(text.split()) | |
| punctuations = list(string.punctuation) | |
| text = "".join([char for char in text if char not in punctuations and char not in "०१२३४५६७८९"]) | |
| text = [word.strip() for word in text.split() if word.strip() not in nep_stopwrods] | |
| text = " ".join(text) | |
| text = " ".join(text.split()) | |
| return text | |
| def predict(self, text: str) -> str: | |
| model = joblib.load(settings.ML_NAIVE_BAYES_MODEL_PATH) | |
| with open(settings.ML_TFIDF_PATH, 'rb') as f: | |
| tfidf = pickle.load(f) | |
| with open(settings.ML_LABEL_ENCODER, 'rb') as f: | |
| label_encoder = pickle.load(f) | |
| text = self.preprocess(text) | |
| vectorized = tfidf.transform([text]) | |
| pred_label_idx = model.predict(vectorized) | |
| pred = label_encoder.inverse_transform(pred_label_idx)[0] | |
| del model, label_encoder, tfidf | |
| gc.collect() | |
| return pred |