Commit ·
9766e10
1
Parent(s): 554a609
restructure and add telemtry
Browse files- .gitignore +2 -1
- Dockerfile +4 -11
- app/.DS_Store +0 -0
- app/{config → core}/config.py +5 -2
- app/core/constants.py +18 -0
- app/core/context.py +3 -0
- app/core/exception.py +111 -0
- app/core/logging.py +29 -0
- app/inference/bilistm.py +111 -0
- app/inference/inference.py +84 -0
- app/inference/mbert.py +105 -0
- app/inference/ml.py +124 -0
- app/inference/mlp.py +108 -0
- app/inference/nep_news_cls_bert.py +0 -23
- app/inference/nep_news_cls_lstm.py +0 -28
- app/inference/nep_news_cls_ml.py +0 -26
- app/inference/nep_news_cls_mlp.py +0 -34
- app/main.py +36 -22
- app/middleware/auth.py +37 -16
- app/middleware/request_tag.py +25 -0
- app/models/models.py +20 -8
- app/preprocess/download_nltk_stopwords.py +0 -3
- app/preprocess/preprocess.py +0 -84
- app/{config → router}/__init__.py +0 -0
- app/router/router.py +6 -0
- app/{preprocess → routes}/__init__.py +0 -0
- app/routes/v1/__init__.py +0 -0
- app/routes/v1/classify.py +13 -0
- pyproject.toml +128 -0
- requirements.txt +34 -1
- start.sh +1 -2
- uv.lock +0 -0
.gitignore
CHANGED
|
@@ -1,4 +1,5 @@
|
|
| 1 |
.venv
|
| 2 |
__pycache__
|
| 3 |
.env
|
| 4 |
-
docker_start.sh
|
|
|
|
|
|
| 1 |
.venv
|
| 2 |
__pycache__
|
| 3 |
.env
|
| 4 |
+
docker_start.sh
|
| 5 |
+
start_dev.sh
|
Dockerfile
CHANGED
|
@@ -8,13 +8,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
| 8 |
|
| 9 |
COPY requirements.txt .
|
| 10 |
|
| 11 |
-
RUN
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
FROM python:3.11-slim AS prod
|
| 16 |
-
|
| 17 |
-
WORKDIR /usr/src/app
|
| 18 |
|
| 19 |
RUN mkdir -p /outputs/bert \
|
| 20 |
/outputs/ml \
|
|
@@ -22,8 +18,6 @@ RUN mkdir -p /outputs/bert \
|
|
| 22 |
/outputs/lstm \
|
| 23 |
/outputs/transformer
|
| 24 |
|
| 25 |
-
COPY --from=builder /.venv ./.venv
|
| 26 |
-
|
| 27 |
COPY outputs/bert/label_encoder.pkl ./outputs/bert/
|
| 28 |
COPY outputs/lstm ./outputs/lstm
|
| 29 |
COPY outputs/ml ./outputs/ml
|
|
@@ -32,9 +26,8 @@ COPY outputs/mlp ./outputs/mlp
|
|
| 32 |
COPY app ./app
|
| 33 |
COPY start.sh .
|
| 34 |
|
| 35 |
-
ENV PATH="/usr/src/app/.venv/bin:$PATH"
|
| 36 |
-
|
| 37 |
RUN useradd -m appuser && \
|
|
|
|
| 38 |
chmod +x start.sh
|
| 39 |
|
| 40 |
USER appuser
|
|
|
|
| 8 |
|
| 9 |
COPY requirements.txt .
|
| 10 |
|
| 11 |
+
RUN pip3 install --upgrade pip && \
|
| 12 |
+
pip3 install --no-cache-dir -r requirements.txt && \
|
| 13 |
+
opentelemetry-bootstrap -a install
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
|
| 15 |
RUN mkdir -p /outputs/bert \
|
| 16 |
/outputs/ml \
|
|
|
|
| 18 |
/outputs/lstm \
|
| 19 |
/outputs/transformer
|
| 20 |
|
|
|
|
|
|
|
| 21 |
COPY outputs/bert/label_encoder.pkl ./outputs/bert/
|
| 22 |
COPY outputs/lstm ./outputs/lstm
|
| 23 |
COPY outputs/ml ./outputs/ml
|
|
|
|
| 26 |
COPY app ./app
|
| 27 |
COPY start.sh .
|
| 28 |
|
|
|
|
|
|
|
| 29 |
RUN useradd -m appuser && \
|
| 30 |
+
chown -R appuser:appuser /usr/src/app && \
|
| 31 |
chmod +x start.sh
|
| 32 |
|
| 33 |
USER appuser
|
app/.DS_Store
CHANGED
|
Binary files a/app/.DS_Store and b/app/.DS_Store differ
|
|
|
app/{config → core}/config.py
RENAMED
|
@@ -5,10 +5,13 @@ load_dotenv()
|
|
| 5 |
class Settings:
|
| 6 |
APP_LEVEL_PATH: str = os.path.split(os.path.dirname(os.path.abspath(__file__)))[0]
|
| 7 |
ROOT_LEVEL_PATH: str = os.path.split(APP_LEVEL_PATH)[0]
|
| 8 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
HF_ACCESS_TOKEN = os.getenv("HF_TOKEN")
|
| 10 |
|
| 11 |
-
### Classification
|
| 12 |
BERT_TOKENIZER_PATH = os.getenv("BERT_MODEL_PATH")
|
| 13 |
LSTM_TOKENIZER_PATH = os.path.join(ROOT_LEVEL_PATH, 'outputs/lstm/tokenizer.json')
|
| 14 |
ML_TFIDF_PATH = os.path.join(ROOT_LEVEL_PATH, 'outputs/ml/tfidf_vectorizer.pkl')
|
|
|
|
| 5 |
class Settings:
|
| 6 |
APP_LEVEL_PATH: str = os.path.split(os.path.dirname(os.path.abspath(__file__)))[0]
|
| 7 |
ROOT_LEVEL_PATH: str = os.path.split(APP_LEVEL_PATH)[0]
|
| 8 |
+
|
| 9 |
+
APP_NAME: str = os.getenv('APP_NAME', "Classification-Backend")
|
| 10 |
+
ENVIRONMENT: str = os.getenv('ENVIRONMENT', "development")
|
| 11 |
+
PORT: int = os.getenv('PORT', 8000)
|
| 12 |
+
|
| 13 |
HF_ACCESS_TOKEN = os.getenv("HF_TOKEN")
|
| 14 |
|
|
|
|
| 15 |
BERT_TOKENIZER_PATH = os.getenv("BERT_MODEL_PATH")
|
| 16 |
LSTM_TOKENIZER_PATH = os.path.join(ROOT_LEVEL_PATH, 'outputs/lstm/tokenizer.json')
|
| 17 |
ML_TFIDF_PATH = os.path.join(ROOT_LEVEL_PATH, 'outputs/ml/tfidf_vectorizer.pkl')
|
app/core/constants.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
REQUEST_ID_HEADER = "X-Request-ID"
|
| 2 |
+
|
| 3 |
+
SECURITY_HEADERS = {
|
| 4 |
+
"X-Content-Type-Options": "nosniff",
|
| 5 |
+
"X-Frame-Options": "DENY",
|
| 6 |
+
"Referrer-Policy": "strict-origin-when-cross-origin",
|
| 7 |
+
"Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload",
|
| 8 |
+
|
| 9 |
+
"Cache-Control": "no-store, no-cache, must-revalidate max-age=0"
|
| 10 |
+
}
|
| 11 |
+
|
| 12 |
+
CSP_HEADER = {
|
| 13 |
+
"Content-Security-Policy": "default-src 'none'; frame-ancestors 'none';",
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
UNAUTHORIZED_HEADERS = {
|
| 17 |
+
"WWW-Authenticate": "Bearer realm='api'"
|
| 18 |
+
}
|
app/core/context.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import contextvars
|
| 2 |
+
|
| 3 |
+
request_id_ctx = contextvars.ContextVar("request_id", default=None)
|
app/core/exception.py
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import Request, Response
|
| 2 |
+
from fastapi.encoders import jsonable_encoder
|
| 3 |
+
from fastapi.responses import JSONResponse
|
| 4 |
+
from fastapi.exceptions import RequestValidationError
|
| 5 |
+
from starlette.exceptions import HTTPException
|
| 6 |
+
from fastapi.utils import is_body_allowed_for_status_code
|
| 7 |
+
from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY
|
| 8 |
+
from app.core.constants import (
|
| 9 |
+
REQUEST_ID_HEADER,
|
| 10 |
+
SECURITY_HEADERS,
|
| 11 |
+
CSP_HEADER
|
| 12 |
+
)
|
| 13 |
+
import logging
|
| 14 |
+
|
| 15 |
+
logger = logging.getLogger(__name__)
|
| 16 |
+
|
| 17 |
+
async def http_exception_handler(
|
| 18 |
+
request: Request,
|
| 19 |
+
exception: HTTPException
|
| 20 |
+
) -> Response:
|
| 21 |
+
headers = getattr(exception, "headers", None) or {}
|
| 22 |
+
request_id = getattr(request.state, "request_id", 'not-set')
|
| 23 |
+
|
| 24 |
+
logger.warning(
|
| 25 |
+
"http_exception",
|
| 26 |
+
extra={
|
| 27 |
+
"path": request.url.path,
|
| 28 |
+
"status_code": exception.status_code,
|
| 29 |
+
"request_id": request_id,
|
| 30 |
+
"method": request.method,
|
| 31 |
+
"error_type": type(exception).__name__,
|
| 32 |
+
"error": str(exception.detail),
|
| 33 |
+
},
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
if not is_body_allowed_for_status_code(exception.status_code):
|
| 37 |
+
return Response(status_code=exception.status_code, headers=headers)
|
| 38 |
+
|
| 39 |
+
return JSONResponse(
|
| 40 |
+
status_code=exception.status_code,
|
| 41 |
+
content={
|
| 42 |
+
"detail": exception.detail,
|
| 43 |
+
"request_id": request_id,
|
| 44 |
+
},
|
| 45 |
+
headers= headers | {
|
| 46 |
+
REQUEST_ID_HEADER: request_id,
|
| 47 |
+
**SECURITY_HEADERS,
|
| 48 |
+
**CSP_HEADER
|
| 49 |
+
}
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
async def request_validation_error_handler(
|
| 53 |
+
request: Request,
|
| 54 |
+
exception: RequestValidationError
|
| 55 |
+
) -> Response:
|
| 56 |
+
headers = getattr(exception, "headers", None) or {}
|
| 57 |
+
request_id = getattr(request.state, "request_id", 'not-set')
|
| 58 |
+
|
| 59 |
+
logger.warning(
|
| 60 |
+
"request_validation_exception",
|
| 61 |
+
extra={
|
| 62 |
+
"path": request.url.path,
|
| 63 |
+
"request_id": request_id,
|
| 64 |
+
"method": request.method,
|
| 65 |
+
"error_type": type(exception).__name__,
|
| 66 |
+
"error": str(exception.errors())
|
| 67 |
+
},
|
| 68 |
+
)
|
| 69 |
+
return JSONResponse(
|
| 70 |
+
status_code=HTTP_422_UNPROCESSABLE_ENTITY,
|
| 71 |
+
content={
|
| 72 |
+
"detail": jsonable_encoder(exception.errors()),
|
| 73 |
+
"request_id": request_id,
|
| 74 |
+
},
|
| 75 |
+
headers= headers | {
|
| 76 |
+
REQUEST_ID_HEADER: request_id,
|
| 77 |
+
**SECURITY_HEADERS,
|
| 78 |
+
**CSP_HEADER
|
| 79 |
+
}
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
async def global_exception_handler(
|
| 84 |
+
request: Request,
|
| 85 |
+
exception: Exception
|
| 86 |
+
) -> Response:
|
| 87 |
+
headers = getattr(exception, "headers", None) or {}
|
| 88 |
+
request_id = getattr(request.state, "request_id", 'not-set')
|
| 89 |
+
|
| 90 |
+
logger.exception(
|
| 91 |
+
"unhandled_exception",
|
| 92 |
+
extra={
|
| 93 |
+
"path": request.url.path,
|
| 94 |
+
"request_id": request_id,
|
| 95 |
+
"method": request.method,
|
| 96 |
+
"error_type": type(exception).__name__
|
| 97 |
+
},
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
return JSONResponse(
|
| 101 |
+
status_code=500,
|
| 102 |
+
content={
|
| 103 |
+
"detail": "Internal Server Error",
|
| 104 |
+
"request_id": request_id,
|
| 105 |
+
},
|
| 106 |
+
headers=headers | {
|
| 107 |
+
REQUEST_ID_HEADER: request_id,
|
| 108 |
+
**SECURITY_HEADERS,
|
| 109 |
+
**CSP_HEADER
|
| 110 |
+
}
|
| 111 |
+
)
|
app/core/logging.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
from pythonjsonlogger import jsonlogger
|
| 3 |
+
from app.core.config import settings
|
| 4 |
+
|
| 5 |
+
class AppNameFilter(logging.Filter):
|
| 6 |
+
def filter(self, record):
|
| 7 |
+
record.app_name = settings.APP_NAME
|
| 8 |
+
return True
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def setup_logging():
|
| 12 |
+
logger = logging.getLogger()
|
| 13 |
+
logger.setLevel(logging.INFO)
|
| 14 |
+
|
| 15 |
+
formatter = jsonlogger.JsonFormatter(
|
| 16 |
+
fmt="%(app_name)s %(levelname)s %(name)s %(message)s %(request_id)s",
|
| 17 |
+
rename_fields={
|
| 18 |
+
"levelname": "level",
|
| 19 |
+
"name": "logger",
|
| 20 |
+
}
|
| 21 |
+
)
|
| 22 |
+
|
| 23 |
+
for handler in logger.handlers:
|
| 24 |
+
handler.addFilter(AppNameFilter())
|
| 25 |
+
handler.setFormatter(formatter)
|
| 26 |
+
|
| 27 |
+
logging.getLogger("uvicorn").setLevel(logging.WARNING)
|
| 28 |
+
logging.getLogger("uvicorn.error").setLevel(logging.WARNING)
|
| 29 |
+
logging.getLogger("uvicorn.access").setLevel(logging.WARNING)
|
app/inference/bilistm.py
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import tensorflow as tf
|
| 2 |
+
import keras
|
| 3 |
+
import json
|
| 4 |
+
import gc
|
| 5 |
+
import pickle
|
| 6 |
+
import string
|
| 7 |
+
import re
|
| 8 |
+
from nltk.corpus import stopwords
|
| 9 |
+
from app.core.config import settings
|
| 10 |
+
|
| 11 |
+
class BiLstm:
|
| 12 |
+
|
| 13 |
+
def __init__(self):
|
| 14 |
+
self.MAX_SEQUENCE_LENGTH: int = 256
|
| 15 |
+
|
| 16 |
+
def preprocess(self, text: str) -> str:
|
| 17 |
+
nep_stopwrods = stopwords.words("nepali")
|
| 18 |
+
text = (
|
| 19 |
+
text.replace("\n", " ")
|
| 20 |
+
.replace("\r", " ")
|
| 21 |
+
.replace("\t", " ")
|
| 22 |
+
.replace("–", " ")
|
| 23 |
+
.replace("-", " ")
|
| 24 |
+
.replace("।", " ")
|
| 25 |
+
.replace("—", " ")
|
| 26 |
+
.replace("ॽ", " ")
|
| 27 |
+
.replace("?", " ")
|
| 28 |
+
.replace("ः", "")
|
| 29 |
+
.replace(":", " ")
|
| 30 |
+
.replace("“", ' ')
|
| 31 |
+
.replace("”", ' ')
|
| 32 |
+
.replace("‘", " ")
|
| 33 |
+
.replace("’", " ")
|
| 34 |
+
.replace('"', " ")
|
| 35 |
+
.replace("'", " ")
|
| 36 |
+
.replace("0", "०")
|
| 37 |
+
.replace("1", "१")
|
| 38 |
+
.replace("2", "२")
|
| 39 |
+
.replace("3", "३")
|
| 40 |
+
.replace("4", "४")
|
| 41 |
+
.replace("5", "५")
|
| 42 |
+
.replace("6", "६")
|
| 43 |
+
.replace("7", "७")
|
| 44 |
+
.replace("8", "८")
|
| 45 |
+
.replace("9", "९")
|
| 46 |
+
)
|
| 47 |
+
regrex_pattern = re.compile(
|
| 48 |
+
pattern="["
|
| 49 |
+
"\U0001f600-\U0001f64f"
|
| 50 |
+
"\U0001f300-\U0001f5ff"
|
| 51 |
+
"\U0001f680-\U0001f6ff"
|
| 52 |
+
"\U0001f1e0-\U0001f1ff"
|
| 53 |
+
"\U00002500-\U00002bef"
|
| 54 |
+
"\U00002702-\U000027b0"
|
| 55 |
+
"\U000024c2-\U0001f251"
|
| 56 |
+
"\U0001f926-\U0001f937"
|
| 57 |
+
"\U00010000-\U0010ffff"
|
| 58 |
+
"\u2640-\u2642"
|
| 59 |
+
"\u2600-\u2b55"
|
| 60 |
+
"\u200d"
|
| 61 |
+
"\u23cf"
|
| 62 |
+
"\u23e9"
|
| 63 |
+
"\u231a"
|
| 64 |
+
"\ufe0f"
|
| 65 |
+
"\u3030"
|
| 66 |
+
"]+",
|
| 67 |
+
flags=re.UNICODE,
|
| 68 |
+
)
|
| 69 |
+
text = regrex_pattern.sub("", text)
|
| 70 |
+
pattern = re.compile(r'<.*?>')
|
| 71 |
+
text = pattern.sub('', text)
|
| 72 |
+
text = re.sub(r'([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,})', '', text)
|
| 73 |
+
text = re.sub(r"\s+", " ", text)
|
| 74 |
+
text = re.sub('[a-zA-Z]', '', text)
|
| 75 |
+
text = re.sub(r'\([^)]*\)', '', text)
|
| 76 |
+
text = " ".join(text.split())
|
| 77 |
+
|
| 78 |
+
punctuations = list(string.punctuation)
|
| 79 |
+
text = "".join([char for char in text if char not in punctuations and char not in "०१२३४५६७८९"])
|
| 80 |
+
text = [word.strip() for word in text.split() if word.strip() not in nep_stopwrods]
|
| 81 |
+
text = " ".join(text)
|
| 82 |
+
text = " ".join(text.split())
|
| 83 |
+
|
| 84 |
+
return text
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def predict(self, text: str) -> str:
|
| 88 |
+
model = keras.models.load_model(settings.LSTM_MODEL_PATH)
|
| 89 |
+
with open(settings.LSTM_TOKENIZER_PATH, encoding='utf-8') as f:
|
| 90 |
+
data = json.load(f)
|
| 91 |
+
tokenizer = tf.keras.preprocessing.text.tokenizer_from_json(data)
|
| 92 |
+
with open(settings.LSTM_LABEL_ENCODER, 'rb') as f:
|
| 93 |
+
label_encoder = pickle.load(f)
|
| 94 |
+
|
| 95 |
+
text = self.preprocess(text)
|
| 96 |
+
|
| 97 |
+
sequence = tokenizer.texts_to_sequences([text])[0]
|
| 98 |
+
|
| 99 |
+
if len(sequence) > self.MAX_SEQUENCE_LENGTH:
|
| 100 |
+
sequence = sequence[:self.MAX_SEQUENCE_LENGTH]
|
| 101 |
+
else:
|
| 102 |
+
sequence += [0] * (self.MAX_SEQUENCE_LENGTH - len(sequence))
|
| 103 |
+
|
| 104 |
+
probs = model.predict(tf.convert_to_tensor([sequence], dtype=tf.int32))[0]
|
| 105 |
+
pred_label_idx = probs.argmax()
|
| 106 |
+
pred = label_encoder.inverse_transform([pred_label_idx])[0]
|
| 107 |
+
|
| 108 |
+
del model, tokenizer, label_encoder
|
| 109 |
+
gc.collect()
|
| 110 |
+
|
| 111 |
+
return pred
|
app/inference/inference.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from app.models.models import ClassificationModel, MLModel, InferenceResult
|
| 2 |
+
from app.inference.mbert import MBert
|
| 3 |
+
from app.inference.mlp import MLP
|
| 4 |
+
from app.inference.bilistm import BiLstm
|
| 5 |
+
from app.inference.ml import MachineLearning
|
| 6 |
+
from app.core.context import request_id_ctx
|
| 7 |
+
import logging
|
| 8 |
+
import time
|
| 9 |
+
|
| 10 |
+
logger = logging.getLogger(__name__)
|
| 11 |
+
|
| 12 |
+
class InferenceService:
|
| 13 |
+
|
| 14 |
+
def __init__(self):
|
| 15 |
+
self.models: dict[
|
| 16 |
+
ClassificationModel,
|
| 17 |
+
MBert | MLP | BiLstm | MachineLearning
|
| 18 |
+
] = {
|
| 19 |
+
ClassificationModel.bert: MBert(),
|
| 20 |
+
ClassificationModel.bilstm: BiLstm(),
|
| 21 |
+
ClassificationModel.mlp: MLP(),
|
| 22 |
+
ClassificationModel.naivebayes: MachineLearning(model=MLModel.naivebayes),
|
| 23 |
+
ClassificationModel.gradientboost: MachineLearning(model=MLModel.gradientboost),
|
| 24 |
+
ClassificationModel.xgboost: MachineLearning(model=MLModel.xgboost),
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
def run_inference(
|
| 28 |
+
self,
|
| 29 |
+
model: ClassificationModel,
|
| 30 |
+
text: str,
|
| 31 |
+
) -> str:
|
| 32 |
+
request_id = request_id_ctx.get('not-set')
|
| 33 |
+
|
| 34 |
+
logger.info(
|
| 35 |
+
"starting_inference_run",
|
| 36 |
+
extra={
|
| 37 |
+
"request_id": request_id,
|
| 38 |
+
"model": model,
|
| 39 |
+
"text_length": len(text.split()),
|
| 40 |
+
}
|
| 41 |
+
)
|
| 42 |
+
start_time = time.perf_counter()
|
| 43 |
+
|
| 44 |
+
instance = self.models.get(model)
|
| 45 |
+
|
| 46 |
+
if instance is None:
|
| 47 |
+
logger.error(
|
| 48 |
+
"inference_model_not_found",
|
| 49 |
+
extra={
|
| 50 |
+
"request_id": request_id,
|
| 51 |
+
"model": model,
|
| 52 |
+
}
|
| 53 |
+
)
|
| 54 |
+
raise ValueError(
|
| 55 |
+
f"Unknown model: {model}"
|
| 56 |
+
)
|
| 57 |
+
try:
|
| 58 |
+
prediction: str = instance.predict(text=text)
|
| 59 |
+
logger.info(
|
| 60 |
+
"inference_completed_successfully",
|
| 61 |
+
extra={
|
| 62 |
+
"request_id": request_id,
|
| 63 |
+
"model": model,
|
| 64 |
+
"duration": round((time.perf_counter() - start_time) * 1000),
|
| 65 |
+
"words_processed": min(instance.MAX_SEQUENCE_LENGTH, len(text.split()))
|
| 66 |
+
}
|
| 67 |
+
)
|
| 68 |
+
return InferenceResult(
|
| 69 |
+
prediction = prediction
|
| 70 |
+
)
|
| 71 |
+
except Exception as e:
|
| 72 |
+
logger.info(
|
| 73 |
+
"inference_failed",
|
| 74 |
+
extra={
|
| 75 |
+
"request_id": request_id,
|
| 76 |
+
"model": model,
|
| 77 |
+
"duration": round((time.perf_counter() - start_time) * 1000),
|
| 78 |
+
"error": str(e),
|
| 79 |
+
}
|
| 80 |
+
)
|
| 81 |
+
raise e
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
inference_service = InferenceService()
|
app/inference/mbert.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from transformers import BertTokenizer, BertForSequenceClassification
|
| 2 |
+
from app.core.config import settings
|
| 3 |
+
import pickle
|
| 4 |
+
import torch
|
| 5 |
+
import gc
|
| 6 |
+
import string
|
| 7 |
+
import pickle
|
| 8 |
+
import re
|
| 9 |
+
from nltk.corpus import stopwords
|
| 10 |
+
|
| 11 |
+
torch.classes.__path__ = []
|
| 12 |
+
|
| 13 |
+
class MBert:
|
| 14 |
+
|
| 15 |
+
def __init__(self):
|
| 16 |
+
self.MAX_SEQUENCE_LENGTH: int = 512
|
| 17 |
+
|
| 18 |
+
def preprocess(self, text: str) -> str:
|
| 19 |
+
nep_stopwrods = stopwords.words("nepali")
|
| 20 |
+
text = (
|
| 21 |
+
text.replace("\n", " ")
|
| 22 |
+
.replace("\r", " ")
|
| 23 |
+
.replace("\t", " ")
|
| 24 |
+
.replace("–", " ")
|
| 25 |
+
.replace("-", " ")
|
| 26 |
+
.replace("।", " ")
|
| 27 |
+
.replace("—", " ")
|
| 28 |
+
.replace("ॽ", " ")
|
| 29 |
+
.replace("?", " ")
|
| 30 |
+
.replace("ः", "")
|
| 31 |
+
.replace(":", " ")
|
| 32 |
+
.replace("“", ' ')
|
| 33 |
+
.replace("”", ' ')
|
| 34 |
+
.replace("‘", " ")
|
| 35 |
+
.replace("’", " ")
|
| 36 |
+
.replace('"', " ")
|
| 37 |
+
.replace("'", " ")
|
| 38 |
+
.replace("0", "०")
|
| 39 |
+
.replace("1", "१")
|
| 40 |
+
.replace("2", "२")
|
| 41 |
+
.replace("3", "३")
|
| 42 |
+
.replace("4", "४")
|
| 43 |
+
.replace("5", "५")
|
| 44 |
+
.replace("6", "६")
|
| 45 |
+
.replace("7", "७")
|
| 46 |
+
.replace("8", "८")
|
| 47 |
+
.replace("9", "९")
|
| 48 |
+
)
|
| 49 |
+
regrex_pattern = re.compile(
|
| 50 |
+
pattern="["
|
| 51 |
+
"\U0001f600-\U0001f64f"
|
| 52 |
+
"\U0001f300-\U0001f5ff"
|
| 53 |
+
"\U0001f680-\U0001f6ff"
|
| 54 |
+
"\U0001f1e0-\U0001f1ff"
|
| 55 |
+
"\U00002500-\U00002bef"
|
| 56 |
+
"\U00002702-\U000027b0"
|
| 57 |
+
"\U000024c2-\U0001f251"
|
| 58 |
+
"\U0001f926-\U0001f937"
|
| 59 |
+
"\U00010000-\U0010ffff"
|
| 60 |
+
"\u2640-\u2642"
|
| 61 |
+
"\u2600-\u2b55"
|
| 62 |
+
"\u200d"
|
| 63 |
+
"\u23cf"
|
| 64 |
+
"\u23e9"
|
| 65 |
+
"\u231a"
|
| 66 |
+
"\ufe0f"
|
| 67 |
+
"\u3030"
|
| 68 |
+
"]+",
|
| 69 |
+
flags=re.UNICODE,
|
| 70 |
+
)
|
| 71 |
+
text = regrex_pattern.sub("", text)
|
| 72 |
+
pattern = re.compile(r'<.*?>')
|
| 73 |
+
text = pattern.sub('', text)
|
| 74 |
+
text = re.sub(r'([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,})', '', text)
|
| 75 |
+
text = re.sub(r"\s+", " ", text)
|
| 76 |
+
text = re.sub('[a-zA-Z]', '', text)
|
| 77 |
+
text = re.sub(r'\([^)]*\)', '', text)
|
| 78 |
+
text = " ".join(text.split())
|
| 79 |
+
|
| 80 |
+
punctuations = list(string.punctuation)
|
| 81 |
+
text = "".join([char for char in text if char not in punctuations and char not in "०१२३४५६७८९"])
|
| 82 |
+
text = [word.strip() for word in text.split() if word.strip() not in nep_stopwrods]
|
| 83 |
+
text = " ".join(text)
|
| 84 |
+
text = " ".join(text.split())
|
| 85 |
+
|
| 86 |
+
return text
|
| 87 |
+
|
| 88 |
+
def predict(self, text: str) -> str:
|
| 89 |
+
model = BertForSequenceClassification.from_pretrained(settings.BERT_MODEL_PATH, token=settings.HF_ACCESS_TOKEN)
|
| 90 |
+
tokenizer = BertTokenizer.from_pretrained(settings.BERT_TOKENIZER_PATH, token=settings.HF_ACCESS_TOKEN)
|
| 91 |
+
|
| 92 |
+
with open(settings.BERT_LABEL_ENCODER, 'rb') as f:
|
| 93 |
+
label_encoder = pickle.load(f)
|
| 94 |
+
|
| 95 |
+
text = self.preprocess(text)
|
| 96 |
+
inputs = tokenizer(text, padding=True, truncation=True, max_length=self.MAX_SEQUENCE_LENGTH, return_tensors='pt')
|
| 97 |
+
outputs = model(**inputs)
|
| 98 |
+
probs = outputs[0].softmax(1)
|
| 99 |
+
pred_label_idx = probs.argmax()
|
| 100 |
+
pred = label_encoder.inverse_transform([pred_label_idx.cpu().numpy()])[0]
|
| 101 |
+
|
| 102 |
+
del model, tokenizer, label_encoder
|
| 103 |
+
gc.collect()
|
| 104 |
+
|
| 105 |
+
return pred
|
app/inference/ml.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import joblib
|
| 2 |
+
import pickle
|
| 3 |
+
import gc
|
| 4 |
+
import string
|
| 5 |
+
import re
|
| 6 |
+
from nltk.corpus import stopwords
|
| 7 |
+
from app.core.config import settings
|
| 8 |
+
from app.models.models import MLModel
|
| 9 |
+
from app.core.context import request_id_ctx
|
| 10 |
+
import logging
|
| 11 |
+
|
| 12 |
+
logger = logging.getLogger(__name__)
|
| 13 |
+
|
| 14 |
+
class MachineLearning:
|
| 15 |
+
|
| 16 |
+
def __init__(self, model: MLModel):
|
| 17 |
+
self.MAX_SEQUENCE_LENGTH: int = 256
|
| 18 |
+
|
| 19 |
+
self.model_path: str = {
|
| 20 |
+
MLModel.naivebayes: settings.ML_NAIVE_BAYES_MODEL_PATH,
|
| 21 |
+
MLModel.gradientboost: settings.ML_GRADIENT_BOOSTING_MODEL_PATH,
|
| 22 |
+
MLModel.xgboost: settings.ML_XG_BOOST_MODEL_PATH,
|
| 23 |
+
}.get(model)
|
| 24 |
+
|
| 25 |
+
if self.model_path is None:
|
| 26 |
+
logger.error(
|
| 27 |
+
"inference_model_not_found",
|
| 28 |
+
extra={
|
| 29 |
+
"request_id": request_id_ctx.get('not-set'),
|
| 30 |
+
"model": model
|
| 31 |
+
}
|
| 32 |
+
)
|
| 33 |
+
raise ValueError(
|
| 34 |
+
f"Unknown model: {model}"
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
def preprocess(self, text: str) -> str:
|
| 38 |
+
nep_stopwrods = stopwords.words("nepali")
|
| 39 |
+
text = (
|
| 40 |
+
text.replace("\n", " ")
|
| 41 |
+
.replace("\r", " ")
|
| 42 |
+
.replace("\t", " ")
|
| 43 |
+
.replace("–", " ")
|
| 44 |
+
.replace("-", " ")
|
| 45 |
+
.replace("।", " ")
|
| 46 |
+
.replace("—", " ")
|
| 47 |
+
.replace("ॽ", " ")
|
| 48 |
+
.replace("?", " ")
|
| 49 |
+
.replace("ः", "")
|
| 50 |
+
.replace(":", " ")
|
| 51 |
+
.replace("“", ' ')
|
| 52 |
+
.replace("”", ' ')
|
| 53 |
+
.replace("‘", " ")
|
| 54 |
+
.replace("’", " ")
|
| 55 |
+
.replace('"', " ")
|
| 56 |
+
.replace("'", " ")
|
| 57 |
+
.replace("0", "०")
|
| 58 |
+
.replace("1", "१")
|
| 59 |
+
.replace("2", "२")
|
| 60 |
+
.replace("3", "३")
|
| 61 |
+
.replace("4", "४")
|
| 62 |
+
.replace("5", "५")
|
| 63 |
+
.replace("6", "६")
|
| 64 |
+
.replace("7", "७")
|
| 65 |
+
.replace("8", "८")
|
| 66 |
+
.replace("9", "९")
|
| 67 |
+
)
|
| 68 |
+
regrex_pattern = re.compile(
|
| 69 |
+
pattern="["
|
| 70 |
+
"\U0001f600-\U0001f64f"
|
| 71 |
+
"\U0001f300-\U0001f5ff"
|
| 72 |
+
"\U0001f680-\U0001f6ff"
|
| 73 |
+
"\U0001f1e0-\U0001f1ff"
|
| 74 |
+
"\U00002500-\U00002bef"
|
| 75 |
+
"\U00002702-\U000027b0"
|
| 76 |
+
"\U000024c2-\U0001f251"
|
| 77 |
+
"\U0001f926-\U0001f937"
|
| 78 |
+
"\U00010000-\U0010ffff"
|
| 79 |
+
"\u2640-\u2642"
|
| 80 |
+
"\u2600-\u2b55"
|
| 81 |
+
"\u200d"
|
| 82 |
+
"\u23cf"
|
| 83 |
+
"\u23e9"
|
| 84 |
+
"\u231a"
|
| 85 |
+
"\ufe0f"
|
| 86 |
+
"\u3030"
|
| 87 |
+
"]+",
|
| 88 |
+
flags=re.UNICODE,
|
| 89 |
+
)
|
| 90 |
+
text = regrex_pattern.sub("", text)
|
| 91 |
+
pattern = re.compile(r'<.*?>')
|
| 92 |
+
text = pattern.sub('', text)
|
| 93 |
+
text = re.sub(r'([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,})', '', text)
|
| 94 |
+
text = re.sub(r"\s+", " ", text)
|
| 95 |
+
text = re.sub('[a-zA-Z]', '', text)
|
| 96 |
+
text = re.sub(r'\([^)]*\)', '', text)
|
| 97 |
+
text = " ".join(text.split())
|
| 98 |
+
|
| 99 |
+
punctuations = list(string.punctuation)
|
| 100 |
+
text = "".join([char for char in text if char not in punctuations and char not in "०१२३४५६७८९"])
|
| 101 |
+
text = [word.strip() for word in text.split() if word.strip() not in nep_stopwrods]
|
| 102 |
+
text = " ".join(text)
|
| 103 |
+
text = " ".join(text.split())
|
| 104 |
+
|
| 105 |
+
return text
|
| 106 |
+
|
| 107 |
+
def predict(self, text: str) -> str:
|
| 108 |
+
model = joblib.load(settings.ML_NAIVE_BAYES_MODEL_PATH)
|
| 109 |
+
|
| 110 |
+
with open(settings.ML_TFIDF_PATH, 'rb') as f:
|
| 111 |
+
tfidf = pickle.load(f)
|
| 112 |
+
|
| 113 |
+
with open(settings.ML_LABEL_ENCODER, 'rb') as f:
|
| 114 |
+
label_encoder = pickle.load(f)
|
| 115 |
+
|
| 116 |
+
text = self.preprocess(text)
|
| 117 |
+
vectorized = tfidf.transform([text])
|
| 118 |
+
pred_label_idx = model.predict(vectorized)
|
| 119 |
+
pred = label_encoder.inverse_transform(pred_label_idx)[0]
|
| 120 |
+
|
| 121 |
+
del model, label_encoder, tfidf
|
| 122 |
+
gc.collect()
|
| 123 |
+
|
| 124 |
+
return pred
|
app/inference/mlp.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import tensorflow as tf
|
| 2 |
+
import keras
|
| 3 |
+
import json
|
| 4 |
+
import gc
|
| 5 |
+
import pickle
|
| 6 |
+
import string
|
| 7 |
+
import re
|
| 8 |
+
from nltk.corpus import stopwords
|
| 9 |
+
from app.core.config import settings
|
| 10 |
+
|
| 11 |
+
class MLP:
|
| 12 |
+
|
| 13 |
+
def __init__(self):
|
| 14 |
+
self.MAX_SEQUENCE_LENGTH: int = 256
|
| 15 |
+
|
| 16 |
+
def preprocess(self, text: str) -> str:
|
| 17 |
+
nep_stopwrods = stopwords.words("nepali")
|
| 18 |
+
text = (
|
| 19 |
+
text.replace("\n", " ")
|
| 20 |
+
.replace("\r", " ")
|
| 21 |
+
.replace("\t", " ")
|
| 22 |
+
.replace("–", " ")
|
| 23 |
+
.replace("-", " ")
|
| 24 |
+
.replace("।", " ")
|
| 25 |
+
.replace("—", " ")
|
| 26 |
+
.replace("ॽ", " ")
|
| 27 |
+
.replace("?", " ")
|
| 28 |
+
.replace("ः", "")
|
| 29 |
+
.replace(":", " ")
|
| 30 |
+
.replace("“", ' ')
|
| 31 |
+
.replace("”", ' ')
|
| 32 |
+
.replace("‘", " ")
|
| 33 |
+
.replace("’", " ")
|
| 34 |
+
.replace('"', " ")
|
| 35 |
+
.replace("'", " ")
|
| 36 |
+
.replace("0", "०")
|
| 37 |
+
.replace("1", "१")
|
| 38 |
+
.replace("2", "२")
|
| 39 |
+
.replace("3", "३")
|
| 40 |
+
.replace("4", "४")
|
| 41 |
+
.replace("5", "५")
|
| 42 |
+
.replace("6", "६")
|
| 43 |
+
.replace("7", "७")
|
| 44 |
+
.replace("8", "८")
|
| 45 |
+
.replace("9", "९")
|
| 46 |
+
)
|
| 47 |
+
regrex_pattern = re.compile(
|
| 48 |
+
pattern="["
|
| 49 |
+
"\U0001f600-\U0001f64f"
|
| 50 |
+
"\U0001f300-\U0001f5ff"
|
| 51 |
+
"\U0001f680-\U0001f6ff"
|
| 52 |
+
"\U0001f1e0-\U0001f1ff"
|
| 53 |
+
"\U00002500-\U00002bef"
|
| 54 |
+
"\U00002702-\U000027b0"
|
| 55 |
+
"\U000024c2-\U0001f251"
|
| 56 |
+
"\U0001f926-\U0001f937"
|
| 57 |
+
"\U00010000-\U0010ffff"
|
| 58 |
+
"\u2640-\u2642"
|
| 59 |
+
"\u2600-\u2b55"
|
| 60 |
+
"\u200d"
|
| 61 |
+
"\u23cf"
|
| 62 |
+
"\u23e9"
|
| 63 |
+
"\u231a"
|
| 64 |
+
"\ufe0f"
|
| 65 |
+
"\u3030"
|
| 66 |
+
"]+",
|
| 67 |
+
flags=re.UNICODE,
|
| 68 |
+
)
|
| 69 |
+
text = regrex_pattern.sub("", text)
|
| 70 |
+
pattern = re.compile(r'<.*?>')
|
| 71 |
+
text = pattern.sub('', text)
|
| 72 |
+
text = re.sub(r'([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,})', '', text)
|
| 73 |
+
text = re.sub(r"\s+", " ", text)
|
| 74 |
+
text = re.sub('[a-zA-Z]', '', text)
|
| 75 |
+
text = re.sub(r'\([^)]*\)', '', text)
|
| 76 |
+
text = " ".join(text.split())
|
| 77 |
+
|
| 78 |
+
punctuations = list(string.punctuation)
|
| 79 |
+
text = "".join([char for char in text if char not in punctuations and char not in "०१२३४५६७८९"])
|
| 80 |
+
text = [word.strip() for word in text.split() if word.strip() not in nep_stopwrods]
|
| 81 |
+
text = " ".join(text)
|
| 82 |
+
text = " ".join(text.split())
|
| 83 |
+
|
| 84 |
+
return text
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def predict(self, text: str) -> str:
|
| 88 |
+
model = keras.models.load_model(settings.MLP_MODEL_PATH)
|
| 89 |
+
with open(settings.MLP_TOKENIZER_PATH, encoding='utf-8') as f:
|
| 90 |
+
data = json.load(f)
|
| 91 |
+
tokenizer = tf.keras.preprocessing.text.tokenizer_from_json(data)
|
| 92 |
+
with open(settings.MLP_LABEL_ENCODER, 'rb') as f:
|
| 93 |
+
label_encoder = pickle.load(f)
|
| 94 |
+
text = self.preprocess(text)
|
| 95 |
+
sequence = tokenizer.texts_to_sequences([text])[0]
|
| 96 |
+
|
| 97 |
+
if len(sequence) > self.MAX_SEQUENCE_LENGTH:
|
| 98 |
+
sequence = sequence[:self.MAX_SEQUENCE_LENGTH]
|
| 99 |
+
else:
|
| 100 |
+
sequence += [0] * (self.MAX_SEQUENCE_LENGTH - len(sequence))
|
| 101 |
+
probs = model.predict(tf.convert_to_tensor([sequence], dtype=tf.int32))[0]
|
| 102 |
+
pred_label_idx = probs.argmax()
|
| 103 |
+
pred = label_encoder.inverse_transform([pred_label_idx])[0]
|
| 104 |
+
|
| 105 |
+
del model, tokenizer, label_encoder
|
| 106 |
+
gc.collect()
|
| 107 |
+
|
| 108 |
+
return pred
|
app/inference/nep_news_cls_bert.py
DELETED
|
@@ -1,23 +0,0 @@
|
|
| 1 |
-
from transformers import BertTokenizer, BertForSequenceClassification
|
| 2 |
-
from app.config.config import settings
|
| 3 |
-
import pickle
|
| 4 |
-
from app.preprocess.preprocess import nepali_perprocessing
|
| 5 |
-
import torch
|
| 6 |
-
import gc
|
| 7 |
-
|
| 8 |
-
torch.classes.__path__ = []
|
| 9 |
-
|
| 10 |
-
def predict(text, input_sequence_length=256):
|
| 11 |
-
model = BertForSequenceClassification.from_pretrained(settings.BERT_MODEL_PATH, token=settings.HF_ACCESS_TOKEN)
|
| 12 |
-
tokenizer = BertTokenizer.from_pretrained(settings.BERT_TOKENIZER_PATH, token=settings.HF_ACCESS_TOKEN)
|
| 13 |
-
with open(settings.BERT_LABEL_ENCODER, 'rb') as f:
|
| 14 |
-
label_encoder = pickle.load(f)
|
| 15 |
-
text = nepali_perprocessing(text)
|
| 16 |
-
inputs = tokenizer(text, padding=True, truncation=True, max_length=input_sequence_length, return_tensors='pt')
|
| 17 |
-
outputs = model(**inputs)
|
| 18 |
-
probs = outputs[0].softmax(1)
|
| 19 |
-
pred_label_idx = probs.argmax()
|
| 20 |
-
pred = label_encoder.inverse_transform([pred_label_idx.cpu().numpy()])[0]
|
| 21 |
-
del model, tokenizer, label_encoder
|
| 22 |
-
gc.collect()
|
| 23 |
-
return pred
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app/inference/nep_news_cls_lstm.py
DELETED
|
@@ -1,28 +0,0 @@
|
|
| 1 |
-
import tensorflow as tf
|
| 2 |
-
import keras
|
| 3 |
-
import codecs
|
| 4 |
-
from app.config.config import settings
|
| 5 |
-
import json
|
| 6 |
-
import gc
|
| 7 |
-
import pickle
|
| 8 |
-
from app.preprocess.preprocess import nepali_perprocessing
|
| 9 |
-
|
| 10 |
-
def predict(text, input_sequence_length=256):
|
| 11 |
-
model = keras.models.load_model(settings.LSTM_MODEL_PATH)
|
| 12 |
-
with codecs.open(settings.LSTM_TOKENIZER_PATH, encoding='utf-8') as f:
|
| 13 |
-
data = json.load(f)
|
| 14 |
-
tokenizer = tf.keras.preprocessing.text.tokenizer_from_json(data)
|
| 15 |
-
with open(settings.LSTM_LABEL_ENCODER, 'rb') as f:
|
| 16 |
-
label_encoder = pickle.load(f)
|
| 17 |
-
text = nepali_perprocessing(text)
|
| 18 |
-
sequence = tokenizer.texts_to_sequences([text])[0]
|
| 19 |
-
if len(sequence) > input_sequence_length:
|
| 20 |
-
sequence = sequence[:input_sequence_length]
|
| 21 |
-
else:
|
| 22 |
-
sequence += [0] * (input_sequence_length - len(sequence))
|
| 23 |
-
probs = model.predict(tf.convert_to_tensor([sequence], dtype=tf.int32))[0]
|
| 24 |
-
pred_label_idx = probs.argmax()
|
| 25 |
-
pred = label_encoder.inverse_transform([pred_label_idx])[0]
|
| 26 |
-
del model, tokenizer, label_encoder
|
| 27 |
-
gc.collect()
|
| 28 |
-
return pred
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app/inference/nep_news_cls_ml.py
DELETED
|
@@ -1,26 +0,0 @@
|
|
| 1 |
-
import joblib, pickle
|
| 2 |
-
from app.preprocess.preprocess import nepali_perprocessing
|
| 3 |
-
from app.config.config import settings
|
| 4 |
-
import gc
|
| 5 |
-
|
| 6 |
-
def predict(text, input_sequence_length=256, model='naivebayes'):
|
| 7 |
-
if model == 'naivebayes':
|
| 8 |
-
model = joblib.load(settings.ML_NAIVE_BAYES_MODEL_PATH)
|
| 9 |
-
elif model == 'gradientboost':
|
| 10 |
-
model = joblib.load(settings.ML_GRADIENT_BOOSTING_MODEL_PATH)
|
| 11 |
-
else:
|
| 12 |
-
model = joblib.load(settings.ML_XG_BOOST_MODEL_PATH)
|
| 13 |
-
|
| 14 |
-
with open(settings.ML_TFIDF_PATH, 'rb') as f:
|
| 15 |
-
tfidf = pickle.load(f)
|
| 16 |
-
|
| 17 |
-
with open(settings.ML_LABEL_ENCODER, 'rb') as f:
|
| 18 |
-
label_encoder = pickle.load(f)
|
| 19 |
-
|
| 20 |
-
text = nepali_perprocessing(text)
|
| 21 |
-
vectorized = tfidf.transform([text])
|
| 22 |
-
pred_label_idx = model.predict(vectorized)
|
| 23 |
-
pred = label_encoder.inverse_transform(pred_label_idx)[0]
|
| 24 |
-
del model, label_encoder, tfidf
|
| 25 |
-
gc.collect()
|
| 26 |
-
return pred
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app/inference/nep_news_cls_mlp.py
DELETED
|
@@ -1,34 +0,0 @@
|
|
| 1 |
-
import tensorflow as tf
|
| 2 |
-
import keras
|
| 3 |
-
import codecs
|
| 4 |
-
from app.config.config import settings
|
| 5 |
-
import json
|
| 6 |
-
import gc
|
| 7 |
-
import pickle
|
| 8 |
-
from app.preprocess.preprocess import nepali_perprocessing
|
| 9 |
-
|
| 10 |
-
def predict(text, input_sequence_length=256):
|
| 11 |
-
model = keras.models.load_model(settings.MLP_MODEL_PATH)
|
| 12 |
-
with codecs.open(settings.MLP_TOKENIZER_PATH, encoding='utf-8') as f:
|
| 13 |
-
data = json.load(f)
|
| 14 |
-
tokenizer = tf.keras.preprocessing.text.tokenizer_from_json(data)
|
| 15 |
-
with open(settings.MLP_LABEL_ENCODER, 'rb') as f:
|
| 16 |
-
label_encoder = pickle.load(f)
|
| 17 |
-
text = nepali_perprocessing(text)
|
| 18 |
-
sequence = tokenizer.texts_to_sequences([text])[0]
|
| 19 |
-
|
| 20 |
-
if len(sequence) > input_sequence_length:
|
| 21 |
-
sequence = sequence[:input_sequence_length]
|
| 22 |
-
else:
|
| 23 |
-
sequence += [0] * (input_sequence_length - len(sequence))
|
| 24 |
-
probs = model.predict(tf.convert_to_tensor([sequence], dtype=tf.int32))[0]
|
| 25 |
-
pred_label_idx = probs.argmax()
|
| 26 |
-
pred = label_encoder.inverse_transform([pred_label_idx])[0]
|
| 27 |
-
del model, tokenizer, label_encoder
|
| 28 |
-
gc.collect()
|
| 29 |
-
return pred
|
| 30 |
-
|
| 31 |
-
def get_parameters():
|
| 32 |
-
with codecs.open(settings.MLP_PARAMETERS_PATH, encoding='utf-8') as f:
|
| 33 |
-
data = json.load(f)
|
| 34 |
-
return data
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app/main.py
CHANGED
|
@@ -1,38 +1,52 @@
|
|
| 1 |
from fastapi import FastAPI, HTTPException
|
| 2 |
-
import
|
| 3 |
-
from
|
| 4 |
-
from app.middleware.
|
| 5 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
from app.models.models import ClassificationRequest, ClassificationResponse
|
| 7 |
-
from app.inference import
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
|
| 12 |
app = FastAPI()
|
| 13 |
|
| 14 |
-
app.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
@app.get('/')
|
| 17 |
def home():
|
| 18 |
return {'message': 'pong'}
|
| 19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
@app.post('/classify/nepali-news', status_code=200, response_model=ClassificationResponse)
|
| 21 |
def classify_nepali_news(body: ClassificationRequest) -> ClassificationResponse:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
|
| 23 |
-
|
| 24 |
-
res = bert.predict(body.text)
|
| 25 |
-
elif body.model == 'bilstm':
|
| 26 |
-
res = lstm.predict(body.text)
|
| 27 |
-
elif body.model == 'mlp':
|
| 28 |
-
res = mlp.predict(body.text)
|
| 29 |
-
elif body.model in ['naivebayes', 'gradientboost', 'xgboost']:
|
| 30 |
-
res = ml.predict(body.text, model=body.model)
|
| 31 |
-
else:
|
| 32 |
-
raise HTTPException(400, "Bad Request")
|
| 33 |
-
|
| 34 |
-
return ClassificationResponse(prediction=res)
|
| 35 |
|
| 36 |
if __name__ == "__main__":
|
| 37 |
-
port = int(
|
| 38 |
uvicorn.run(app, host="0.0.0.0", port=port)
|
|
|
|
| 1 |
from fastapi import FastAPI, HTTPException
|
| 2 |
+
import uvicorn
|
| 3 |
+
from app.middleware.auth import auth_middleware
|
| 4 |
+
from app.middleware.request_tag import request_id_middleware
|
| 5 |
+
from starlette.exceptions import HTTPException
|
| 6 |
+
from fastapi.exceptions import RequestValidationError
|
| 7 |
+
from app.core.exception import (
|
| 8 |
+
http_exception_handler,
|
| 9 |
+
request_validation_error_handler,
|
| 10 |
+
global_exception_handler
|
| 11 |
+
)
|
| 12 |
from app.models.models import ClassificationRequest, ClassificationResponse
|
| 13 |
+
from app.inference.inference import inference_service
|
| 14 |
+
from app.core.config import settings
|
| 15 |
+
from app.core.logging import setup_logging
|
| 16 |
+
from app.router.router import api_router
|
| 17 |
+
import nltk
|
| 18 |
+
|
| 19 |
+
nltk.download('stopwords')
|
| 20 |
+
|
| 21 |
+
setup_logging()
|
| 22 |
|
| 23 |
app = FastAPI()
|
| 24 |
|
| 25 |
+
app.add_exception_handler(HTTPException, http_exception_handler)
|
| 26 |
+
app.add_exception_handler(RequestValidationError, request_validation_error_handler)
|
| 27 |
+
app.add_exception_handler(Exception, global_exception_handler)
|
| 28 |
+
|
| 29 |
+
# app.middleware("http")(auth_middleware)
|
| 30 |
+
app.middleware("http")(request_id_middleware)
|
| 31 |
|
| 32 |
@app.get('/')
|
| 33 |
def home():
|
| 34 |
return {'message': 'pong'}
|
| 35 |
|
| 36 |
+
@app.get('/err')
|
| 37 |
+
def err():
|
| 38 |
+
return ClassificationResponse()
|
| 39 |
+
|
| 40 |
@app.post('/classify/nepali-news', status_code=200, response_model=ClassificationResponse)
|
| 41 |
def classify_nepali_news(body: ClassificationRequest) -> ClassificationResponse:
|
| 42 |
+
result: ClassificationResponse = inference_service.run_inference(
|
| 43 |
+
model=body.model,
|
| 44 |
+
text=body.text,
|
| 45 |
+
)
|
| 46 |
+
return result
|
| 47 |
|
| 48 |
+
app.include_router(api_router, prefix='/api')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
|
| 50 |
if __name__ == "__main__":
|
| 51 |
+
port = int(settings.PORT)
|
| 52 |
uvicorn.run(app, host="0.0.0.0", port=port)
|
app/middleware/auth.py
CHANGED
|
@@ -1,22 +1,43 @@
|
|
|
|
|
| 1 |
from fastapi import Request, Response
|
| 2 |
-
from starlette.middleware.base import BaseHTTPMiddleware
|
| 3 |
from starlette.responses import JSONResponse
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
import os
|
| 5 |
|
| 6 |
-
|
| 7 |
-
async def dispatch(self, request: Request, call_next):
|
| 8 |
-
if request.url.path in ["/", "/docs", "/openapi.json"]:
|
| 9 |
-
response = await call_next(request)
|
| 10 |
-
return response
|
| 11 |
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
if token != os.getenv("SERVER_AUTH"):
|
| 19 |
-
return JSONResponse(status_code=401, content={"detail": "Unathorized"})
|
| 20 |
-
|
| 21 |
-
response = await call_next(request)
|
| 22 |
-
return response
|
|
|
|
| 1 |
+
import logging
|
| 2 |
from fastapi import Request, Response
|
|
|
|
| 3 |
from starlette.responses import JSONResponse
|
| 4 |
+
from app.core.constants import (
|
| 5 |
+
REQUEST_ID_HEADER,
|
| 6 |
+
SECURITY_HEADERS,
|
| 7 |
+
UNAUTHORIZED_HEADERS
|
| 8 |
+
)
|
| 9 |
+
from app.core.context import request_id_ctx
|
| 10 |
import os
|
| 11 |
|
| 12 |
+
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
+
async def auth_middleware(request: Request, call_next) -> Response:
|
| 15 |
+
if request.url.path in ["/", "/openapi.json"]:
|
| 16 |
+
return await call_next(request)
|
| 17 |
+
|
| 18 |
+
request_id = request_id_ctx.get('not-set')
|
| 19 |
+
auth_header = request.headers.get("Authorization")
|
| 20 |
+
if not auth_header or not auth_header.startswith("Bearer "):
|
| 21 |
+
return JSONResponse(
|
| 22 |
+
status_code=401,
|
| 23 |
+
content={"detail": "Unauthorized", "request_id": request_id},
|
| 24 |
+
headers= {
|
| 25 |
+
REQUEST_ID_HEADER: request_id,
|
| 26 |
+
**SECURITY_HEADERS,
|
| 27 |
+
**UNAUTHORIZED_HEADERS
|
| 28 |
+
}
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
+
token = auth_header.split(" ")[1]
|
| 32 |
+
if token != os.getenv("SERVER_AUTH"):
|
| 33 |
+
return JSONResponse(
|
| 34 |
+
status_code=401,
|
| 35 |
+
content={"detail": "Unauthorized", "request_id": request_id},
|
| 36 |
+
headers= {
|
| 37 |
+
REQUEST_ID_HEADER: request_id,
|
| 38 |
+
**SECURITY_HEADERS,
|
| 39 |
+
**UNAUTHORIZED_HEADERS
|
| 40 |
+
}
|
| 41 |
+
)
|
| 42 |
|
| 43 |
+
return await call_next(request)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app/middleware/request_tag.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import uuid
|
| 2 |
+
import logging
|
| 3 |
+
from fastapi import Request, Response
|
| 4 |
+
from app.core.context import request_id_ctx
|
| 5 |
+
from app.core.constants import (
|
| 6 |
+
REQUEST_ID_HEADER,
|
| 7 |
+
SECURITY_HEADERS,
|
| 8 |
+
CSP_HEADER
|
| 9 |
+
)
|
| 10 |
+
|
| 11 |
+
logger = logging.getLogger(__name__)
|
| 12 |
+
|
| 13 |
+
async def request_id_middleware(request: Request, call_next):
|
| 14 |
+
request_id = request.headers.get(REQUEST_ID_HEADER) or str(uuid.uuid4())
|
| 15 |
+
request.state.request_id = request_id
|
| 16 |
+
token = request_id_ctx.set(request_id)
|
| 17 |
+
|
| 18 |
+
try:
|
| 19 |
+
response: Response = await call_next(request)
|
| 20 |
+
response.headers[REQUEST_ID_HEADER] = request_id
|
| 21 |
+
response.headers.update(SECURITY_HEADERS)
|
| 22 |
+
response.headers.update(CSP_HEADER)
|
| 23 |
+
return response
|
| 24 |
+
finally:
|
| 25 |
+
request_id_ctx.reset(token)
|
app/models/models.py
CHANGED
|
@@ -1,18 +1,30 @@
|
|
| 1 |
-
from pydantic import BaseModel
|
| 2 |
-
from typing import Optional
|
| 3 |
from enum import Enum
|
| 4 |
-
|
| 5 |
-
class
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
bilstm = "bilstm"
|
| 7 |
bert = "bert"
|
| 8 |
mlp = "mlp"
|
| 9 |
naivebayes = "naivebayes"
|
| 10 |
gradientboost = "gradientboost"
|
| 11 |
xgboost = "xgboost"
|
| 12 |
-
|
| 13 |
class ClassificationRequest(BaseModel):
|
| 14 |
text: str
|
| 15 |
-
model:
|
|
|
|
|
|
|
|
|
|
| 16 |
|
| 17 |
-
class ClassificationResponse(
|
| 18 |
-
|
|
|
|
| 1 |
+
from pydantic import BaseModel, Field
|
|
|
|
| 2 |
from enum import Enum
|
| 3 |
+
|
| 4 |
+
class MLModel(str, Enum):
|
| 5 |
+
naivebayes = "naivebayes"
|
| 6 |
+
gradientboost = "gradientboost"
|
| 7 |
+
xgboost = "xgboost"
|
| 8 |
+
|
| 9 |
+
class NeuralModel(str, Enum):
|
| 10 |
+
bilstm = "bilstm"
|
| 11 |
+
bert = "bert"
|
| 12 |
+
mlp = "mlp"
|
| 13 |
+
|
| 14 |
+
class ClassificationModel(str, Enum):
|
| 15 |
bilstm = "bilstm"
|
| 16 |
bert = "bert"
|
| 17 |
mlp = "mlp"
|
| 18 |
naivebayes = "naivebayes"
|
| 19 |
gradientboost = "gradientboost"
|
| 20 |
xgboost = "xgboost"
|
| 21 |
+
|
| 22 |
class ClassificationRequest(BaseModel):
|
| 23 |
text: str
|
| 24 |
+
model: ClassificationModel = Field(default=ClassificationModel.mlp)
|
| 25 |
+
|
| 26 |
+
class InferenceResult(BaseModel):
|
| 27 |
+
prediction: str
|
| 28 |
|
| 29 |
+
class ClassificationResponse(InferenceResult):
|
| 30 |
+
pass
|
app/preprocess/download_nltk_stopwords.py
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
import nltk
|
| 2 |
-
|
| 3 |
-
nltk.download('stopwords')
|
|
|
|
|
|
|
|
|
|
|
|
app/preprocess/preprocess.py
DELETED
|
@@ -1,84 +0,0 @@
|
|
| 1 |
-
import string
|
| 2 |
-
import re
|
| 3 |
-
from nltk.corpus import stopwords
|
| 4 |
-
|
| 5 |
-
def nepali_perprocessing(text):
|
| 6 |
-
nep_stopwrods = stopwords.words("nepali")
|
| 7 |
-
text = (
|
| 8 |
-
text.replace("\n", " ")
|
| 9 |
-
.replace("\r", " ")
|
| 10 |
-
.replace("\t", " ")
|
| 11 |
-
.replace("–", " ")
|
| 12 |
-
.replace("-", " ")
|
| 13 |
-
.replace("।", " ")
|
| 14 |
-
.replace("—", " ")
|
| 15 |
-
.replace("ॽ", " ")
|
| 16 |
-
.replace("?", " ")
|
| 17 |
-
.replace("ः", "")
|
| 18 |
-
.replace(":", " ")
|
| 19 |
-
.replace("“", ' ')
|
| 20 |
-
.replace("”", ' ')
|
| 21 |
-
.replace("‘", " ")
|
| 22 |
-
.replace("’", " ")
|
| 23 |
-
.replace('"', " ")
|
| 24 |
-
.replace("'", " ")
|
| 25 |
-
.replace("0", "०")
|
| 26 |
-
.replace("1", "१")
|
| 27 |
-
.replace("2", "२")
|
| 28 |
-
.replace("3", "३")
|
| 29 |
-
.replace("4", "४")
|
| 30 |
-
.replace("5", "५")
|
| 31 |
-
.replace("6", "६")
|
| 32 |
-
.replace("7", "७")
|
| 33 |
-
.replace("8", "८")
|
| 34 |
-
.replace("9", "९")
|
| 35 |
-
)
|
| 36 |
-
# Remove emojis
|
| 37 |
-
regrex_pattern = re.compile(
|
| 38 |
-
pattern="["
|
| 39 |
-
"\U0001f600-\U0001f64f" # emoticons
|
| 40 |
-
"\U0001f300-\U0001f5ff" # symbols & pictographs
|
| 41 |
-
"\U0001f680-\U0001f6ff" # transport & map symbols
|
| 42 |
-
"\U0001f1e0-\U0001f1ff" # flags (iOS)
|
| 43 |
-
"\U00002500-\U00002bef" # chinese char
|
| 44 |
-
"\U00002702-\U000027b0"
|
| 45 |
-
"\U000024c2-\U0001f251"
|
| 46 |
-
"\U0001f926-\U0001f937"
|
| 47 |
-
"\U00010000-\U0010ffff"
|
| 48 |
-
"\u2640-\u2642"
|
| 49 |
-
"\u2600-\u2b55"
|
| 50 |
-
"\u200d"
|
| 51 |
-
"\u23cf"
|
| 52 |
-
"\u23e9"
|
| 53 |
-
"\u231a"
|
| 54 |
-
"\ufe0f" # dingbats
|
| 55 |
-
"\u3030"
|
| 56 |
-
"]+",
|
| 57 |
-
flags=re.UNICODE,
|
| 58 |
-
)
|
| 59 |
-
text = regrex_pattern.sub("", text)
|
| 60 |
-
|
| 61 |
-
pattern = re.compile(r'<.*?>') # HTML
|
| 62 |
-
text = pattern.sub('', text)
|
| 63 |
-
|
| 64 |
-
text = re.sub(r'([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,})', '', text) # Email
|
| 65 |
-
|
| 66 |
-
# Remove multiple white spaces
|
| 67 |
-
text = re.sub(r"\s+", " ", text)
|
| 68 |
-
|
| 69 |
-
text = re.sub('[a-zA-Z]', '', text)
|
| 70 |
-
|
| 71 |
-
# Text inside paranthesis
|
| 72 |
-
text = re.sub(r'\([^)]*\)', '', text)
|
| 73 |
-
|
| 74 |
-
text = " ".join(text.split())
|
| 75 |
-
|
| 76 |
-
punctuations = list(string.punctuation)
|
| 77 |
-
text = "".join([char for char in text if char not in punctuations and char not in "०१२३४५६७८९"])
|
| 78 |
-
text = [word.strip() for word in text.split() if word.strip() not in nep_stopwrods]
|
| 79 |
-
text = " ".join(text)
|
| 80 |
-
text = " ".join(text.split())
|
| 81 |
-
|
| 82 |
-
return text
|
| 83 |
-
|
| 84 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app/{config → router}/__init__.py
RENAMED
|
File without changes
|
app/router/router.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter
|
| 2 |
+
from app.routes.v1.classify import router as v1_classify
|
| 3 |
+
|
| 4 |
+
api_router = APIRouter()
|
| 5 |
+
|
| 6 |
+
api_router.include_router(v1_classify, prefix="/v1", tags=["v1-classify"])
|
app/{preprocess → routes}/__init__.py
RENAMED
|
File without changes
|
app/routes/v1/__init__.py
ADDED
|
File without changes
|
app/routes/v1/classify.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter
|
| 2 |
+
from app.models.models import ClassificationRequest, ClassificationResponse
|
| 3 |
+
from app.inference.inference import inference_service
|
| 4 |
+
|
| 5 |
+
router = APIRouter()
|
| 6 |
+
|
| 7 |
+
@router.post('/classify', status_code=200, response_model=ClassificationResponse)
|
| 8 |
+
def classify(body: ClassificationRequest) -> ClassificationResponse:
|
| 9 |
+
result: ClassificationResponse = inference_service.run_inference(
|
| 10 |
+
model=body.model,
|
| 11 |
+
text=body.text,
|
| 12 |
+
)
|
| 13 |
+
return result
|
pyproject.toml
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[project]
|
| 2 |
+
name = "classification-backend"
|
| 3 |
+
version = "0.1.0"
|
| 4 |
+
requires-python = ">=3.11.11"
|
| 5 |
+
dependencies = [
|
| 6 |
+
"absl-py==2.3.0",
|
| 7 |
+
"accelerate==1.8.1",
|
| 8 |
+
"annotated-types==0.7.0",
|
| 9 |
+
"anyio==4.9.0",
|
| 10 |
+
"asgiref==3.11.1",
|
| 11 |
+
"astunparse==1.6.3",
|
| 12 |
+
"certifi==2025.6.15",
|
| 13 |
+
"charset-normalizer==3.4.2",
|
| 14 |
+
"click==8.2.1",
|
| 15 |
+
"dnspython==2.7.0",
|
| 16 |
+
"email-validator==2.2.0",
|
| 17 |
+
"fastapi==0.115.14",
|
| 18 |
+
"fastapi-cli==0.0.7",
|
| 19 |
+
"filelock==3.18.0",
|
| 20 |
+
"flatbuffers==25.2.10",
|
| 21 |
+
"fsspec==2025.5.1",
|
| 22 |
+
"gast==0.6.0",
|
| 23 |
+
"google-pasta==0.2.0",
|
| 24 |
+
"googleapis-common-protos==1.75.0",
|
| 25 |
+
"grpcio==1.81.1",
|
| 26 |
+
"h11==0.16.0",
|
| 27 |
+
"h5py==3.14.0",
|
| 28 |
+
"hf-xet==1.1.5",
|
| 29 |
+
"httpcore==1.0.9",
|
| 30 |
+
"httptools==0.6.4",
|
| 31 |
+
"httpx==0.28.1",
|
| 32 |
+
"huggingface-hub==0.33.2",
|
| 33 |
+
"idna==3.10",
|
| 34 |
+
"jinja2==3.1.6",
|
| 35 |
+
"joblib==1.5.1",
|
| 36 |
+
"keras==3.10.0",
|
| 37 |
+
"libclang==18.1.1",
|
| 38 |
+
"markdown==3.8.2",
|
| 39 |
+
"markdown-it-py==3.0.0",
|
| 40 |
+
"markupsafe==3.0.2",
|
| 41 |
+
"mdurl==0.1.2",
|
| 42 |
+
"ml-dtypes==0.5.1",
|
| 43 |
+
"mpmath==1.3.0",
|
| 44 |
+
"namex==0.1.0",
|
| 45 |
+
"networkx==3.5",
|
| 46 |
+
"nltk==3.9.1",
|
| 47 |
+
"numpy==2.1.3",
|
| 48 |
+
"opentelemetry-api==1.43.0",
|
| 49 |
+
"opentelemetry-distro==0.64b0",
|
| 50 |
+
"opentelemetry-exporter-otlp==1.43.0",
|
| 51 |
+
"opentelemetry-exporter-otlp-proto-common==1.43.0",
|
| 52 |
+
"opentelemetry-exporter-otlp-proto-grpc==1.43.0",
|
| 53 |
+
"opentelemetry-exporter-otlp-proto-http==1.43.0",
|
| 54 |
+
"opentelemetry-instrumentation==0.64b0",
|
| 55 |
+
"opentelemetry-instrumentation-asgi==0.64b0",
|
| 56 |
+
"opentelemetry-instrumentation-asyncio==0.64b0",
|
| 57 |
+
"opentelemetry-instrumentation-click==0.64b0",
|
| 58 |
+
"opentelemetry-instrumentation-dbapi==0.64b0",
|
| 59 |
+
"opentelemetry-instrumentation-exceptions==0.64b0",
|
| 60 |
+
"opentelemetry-instrumentation-fastapi==0.64b0",
|
| 61 |
+
"opentelemetry-instrumentation-grpc==0.64b0",
|
| 62 |
+
"opentelemetry-instrumentation-httpx==0.64b0",
|
| 63 |
+
"opentelemetry-instrumentation-jinja2==0.64b0",
|
| 64 |
+
"opentelemetry-instrumentation-logging==0.64b0",
|
| 65 |
+
"opentelemetry-instrumentation-requests==0.64b0",
|
| 66 |
+
"opentelemetry-instrumentation-sqlite3==0.64b0",
|
| 67 |
+
"opentelemetry-instrumentation-starlette==0.64b0",
|
| 68 |
+
"opentelemetry-instrumentation-system-metrics==0.64b0",
|
| 69 |
+
"opentelemetry-instrumentation-threading==0.64b0",
|
| 70 |
+
"opentelemetry-instrumentation-tortoiseorm==0.64b0",
|
| 71 |
+
"opentelemetry-instrumentation-urllib==0.64b0",
|
| 72 |
+
"opentelemetry-instrumentation-urllib3==0.64b0",
|
| 73 |
+
"opentelemetry-instrumentation-wsgi==0.64b0",
|
| 74 |
+
"opentelemetry-proto==1.43.0",
|
| 75 |
+
"opentelemetry-sdk==1.43.0",
|
| 76 |
+
"opentelemetry-semantic-conventions==0.64b0",
|
| 77 |
+
"opentelemetry-util-http==0.64b0",
|
| 78 |
+
"opt-einsum==3.4.0",
|
| 79 |
+
"optree==0.16.0",
|
| 80 |
+
"packaging==25.0",
|
| 81 |
+
"peft==0.14.0",
|
| 82 |
+
"pip==26.1.2",
|
| 83 |
+
"protobuf==5.29.5",
|
| 84 |
+
"psutil==7.0.0",
|
| 85 |
+
"pydantic==2.11.7",
|
| 86 |
+
"pydantic-core==2.33.2",
|
| 87 |
+
"pygments==2.19.2",
|
| 88 |
+
"python-dotenv==1.1.1",
|
| 89 |
+
"python-json-logger==4.1.0",
|
| 90 |
+
"python-multipart==0.0.20",
|
| 91 |
+
"pyyaml==6.0.2",
|
| 92 |
+
"regex==2024.11.6",
|
| 93 |
+
"requests==2.32.4",
|
| 94 |
+
"rich==14.0.0",
|
| 95 |
+
"rich-toolkit==0.14.8",
|
| 96 |
+
"safetensors==0.5.3",
|
| 97 |
+
"scikit-learn==1.6.1",
|
| 98 |
+
"scipy==1.16.0",
|
| 99 |
+
"sentencepiece==0.2.0",
|
| 100 |
+
"setuptools==82.0.1",
|
| 101 |
+
"shellingham==1.5.4",
|
| 102 |
+
"six==1.17.0",
|
| 103 |
+
"sniffio==1.3.1",
|
| 104 |
+
"starlette==0.46.2",
|
| 105 |
+
"sympy==1.14.0",
|
| 106 |
+
"tensorboard==2.19.0",
|
| 107 |
+
"tensorboard-data-server==0.7.2",
|
| 108 |
+
"tensorflow==2.19.0",
|
| 109 |
+
"tensorflow-io-gcs-filesystem==0.37.1",
|
| 110 |
+
"termcolor==3.1.0",
|
| 111 |
+
"threadpoolctl==3.6.0",
|
| 112 |
+
"tokenizers==0.21.2",
|
| 113 |
+
"torch==2.7.1",
|
| 114 |
+
"tqdm==4.67.1",
|
| 115 |
+
"transformers==4.51.3",
|
| 116 |
+
"typer==0.16.0",
|
| 117 |
+
"typing-extensions==4.14.0",
|
| 118 |
+
"typing-inspection==0.4.1",
|
| 119 |
+
"urllib3==2.5.0",
|
| 120 |
+
"uvicorn==0.35.0",
|
| 121 |
+
"uvloop==0.21.0",
|
| 122 |
+
"watchfiles==1.1.0",
|
| 123 |
+
"websockets==15.0.1",
|
| 124 |
+
"werkzeug==3.1.3",
|
| 125 |
+
"wheel==0.47.0",
|
| 126 |
+
"wrapt==1.17.2",
|
| 127 |
+
"xgboost==3.0.0",
|
| 128 |
+
]
|
requirements.txt
CHANGED
|
@@ -2,6 +2,7 @@ absl-py==2.3.0
|
|
| 2 |
accelerate==1.8.1
|
| 3 |
annotated-types==0.7.0
|
| 4 |
anyio==4.9.0
|
|
|
|
| 5 |
astunparse==1.6.3
|
| 6 |
certifi==2025.6.15
|
| 7 |
charset-normalizer==3.4.2
|
|
@@ -15,7 +16,8 @@ flatbuffers==25.2.10
|
|
| 15 |
fsspec==2025.5.1
|
| 16 |
gast==0.6.0
|
| 17 |
google-pasta==0.2.0
|
| 18 |
-
|
|
|
|
| 19 |
h11==0.16.0
|
| 20 |
h5py==3.14.0
|
| 21 |
hf-xet==1.1.5
|
|
@@ -38,6 +40,36 @@ namex==0.1.0
|
|
| 38 |
networkx==3.5
|
| 39 |
nltk==3.9.1
|
| 40 |
numpy==2.1.3
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
opt_einsum==3.4.0
|
| 42 |
optree==0.16.0
|
| 43 |
packaging==25.0
|
|
@@ -48,6 +80,7 @@ pydantic==2.11.7
|
|
| 48 |
pydantic_core==2.33.2
|
| 49 |
Pygments==2.19.2
|
| 50 |
python-dotenv==1.1.1
|
|
|
|
| 51 |
python-multipart==0.0.20
|
| 52 |
PyYAML==6.0.2
|
| 53 |
regex==2024.11.6
|
|
|
|
| 2 |
accelerate==1.8.1
|
| 3 |
annotated-types==0.7.0
|
| 4 |
anyio==4.9.0
|
| 5 |
+
asgiref==3.11.1
|
| 6 |
astunparse==1.6.3
|
| 7 |
certifi==2025.6.15
|
| 8 |
charset-normalizer==3.4.2
|
|
|
|
| 16 |
fsspec==2025.5.1
|
| 17 |
gast==0.6.0
|
| 18 |
google-pasta==0.2.0
|
| 19 |
+
googleapis-common-protos==1.75.0
|
| 20 |
+
grpcio==1.81.1
|
| 21 |
h11==0.16.0
|
| 22 |
h5py==3.14.0
|
| 23 |
hf-xet==1.1.5
|
|
|
|
| 40 |
networkx==3.5
|
| 41 |
nltk==3.9.1
|
| 42 |
numpy==2.1.3
|
| 43 |
+
opentelemetry-api==1.43.0
|
| 44 |
+
opentelemetry-distro==0.64b0
|
| 45 |
+
opentelemetry-exporter-otlp==1.43.0
|
| 46 |
+
opentelemetry-exporter-otlp-proto-common==1.43.0
|
| 47 |
+
opentelemetry-exporter-otlp-proto-grpc==1.43.0
|
| 48 |
+
opentelemetry-exporter-otlp-proto-http==1.43.0
|
| 49 |
+
opentelemetry-instrumentation==0.64b0
|
| 50 |
+
opentelemetry-instrumentation-asgi==0.64b0
|
| 51 |
+
opentelemetry-instrumentation-asyncio==0.64b0
|
| 52 |
+
opentelemetry-instrumentation-click==0.64b0
|
| 53 |
+
opentelemetry-instrumentation-dbapi==0.64b0
|
| 54 |
+
opentelemetry-instrumentation-exceptions==0.64b0
|
| 55 |
+
opentelemetry-instrumentation-fastapi==0.64b0
|
| 56 |
+
opentelemetry-instrumentation-grpc==0.64b0
|
| 57 |
+
opentelemetry-instrumentation-httpx==0.64b0
|
| 58 |
+
opentelemetry-instrumentation-jinja2==0.64b0
|
| 59 |
+
opentelemetry-instrumentation-logging==0.64b0
|
| 60 |
+
opentelemetry-instrumentation-requests==0.64b0
|
| 61 |
+
opentelemetry-instrumentation-sqlite3==0.64b0
|
| 62 |
+
opentelemetry-instrumentation-starlette==0.64b0
|
| 63 |
+
opentelemetry-instrumentation-system-metrics==0.64b0
|
| 64 |
+
opentelemetry-instrumentation-threading==0.64b0
|
| 65 |
+
opentelemetry-instrumentation-tortoiseorm==0.64b0
|
| 66 |
+
opentelemetry-instrumentation-urllib==0.64b0
|
| 67 |
+
opentelemetry-instrumentation-urllib3==0.64b0
|
| 68 |
+
opentelemetry-instrumentation-wsgi==0.64b0
|
| 69 |
+
opentelemetry-proto==1.43.0
|
| 70 |
+
opentelemetry-sdk==1.43.0
|
| 71 |
+
opentelemetry-semantic-conventions==0.64b0
|
| 72 |
+
opentelemetry-util-http==0.64b0
|
| 73 |
opt_einsum==3.4.0
|
| 74 |
optree==0.16.0
|
| 75 |
packaging==25.0
|
|
|
|
| 80 |
pydantic_core==2.33.2
|
| 81 |
Pygments==2.19.2
|
| 82 |
python-dotenv==1.1.1
|
| 83 |
+
python-json-logger==4.1.0
|
| 84 |
python-multipart==0.0.20
|
| 85 |
PyYAML==6.0.2
|
| 86 |
regex==2024.11.6
|
start.sh
CHANGED
|
@@ -1,5 +1,4 @@
|
|
| 1 |
#!/bin/bash
|
| 2 |
|
| 3 |
export PYTHONPATH=$PWD
|
| 4 |
-
python app/
|
| 5 |
-
python app/main.py
|
|
|
|
| 1 |
#!/bin/bash
|
| 2 |
|
| 3 |
export PYTHONPATH=$PWD
|
| 4 |
+
exec opentelemetry-instrument python app/main.py
|
|
|
uv.lock
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|