feat: initial deployment

Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y \
+    curl \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip install uv --no-cache-dir
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY src/ src/
+COPY api/ api/
+COPY app/ app/
+COPY configs/ configs/
+COPY app.py .
+
+RUN mkdir -p artifacts/models/distilbert \
+    artifacts/vectorizers \
+    artifacts/monitoring \
+    data/raw
+
+ENV PYTHONPATH=/app
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+EXPOSE 7860
+
+CMD ["python", "app.py"]

README.md

@@ -1,10 +1,20 @@
 ---
 title: Intent Classifier
-emoji: 🏃
+emoji: 🧠
 colorFrom: blue
-colorTo: purple
+colorTo: indigo
 sdk: docker
+app_port: 7860
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Intent Classifier
+
+End-to-end intent classification system with full MLOps pipeline.
+
+Classifies user queries into 151 intent categories using CLINC150 dataset.
+Compares 6 model architectures: LogReg, SVM, TextCNN, RNN, LSTM, DistilBERT.
+
+## Source
+
+[github.com/princ0301/intent-classifier](https://github.com/princ0301/intent-classifier)

api/main.py

@@ -0,0 +1,54 @@
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+
+from api.routes import models, monitoring, predict
+from api.schemas import HealthResponse
+from src.serving.ab_router import ABRouter
+from src.serving.predictor import Predictor
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    print("loading predictors...")
+    app.state.predictors = {
+        "classical": Predictor("classical"),
+        "svm": Predictor("svm"),
+        "transformer": Predictor("transformer"),
+    }
+    print("loading AB router...")
+    app.state.ab_router = ABRouter()
+    print("startup complete")
+
+    yield
+
+    app.state.predictors.clear()
+
+
+app = FastAPI(
+    title="Intent Classifier API",
+    description="End-to-end intent classification with classical ML, neural networks, and fine-tuned transformers",
+    version="1.0.0",
+    lifespan=lifespan,
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+app.include_router(predict.router, tags=["predict"])
+app.include_router(models.router, tags=["models"])
+app.include_router(monitoring.router, tags=["monitoring"])
+
+
+@app.get("/health", response_model=HealthResponse)
+def health(request: Request) -> HealthResponse:
+    return HealthResponse(
+        status="ok",
+        models_loaded=list(request.app.state.predictors.keys()),
+    )

api/routes/models.py

@@ -0,0 +1,22 @@
+from fastapi import APIRouter, Request
+
+from api.schemas import ModelInfo, ModelsResponse
+
+router = APIRouter()
+
+REGISTERED_MODELS = [
+    {"name": "intent-classifier-logreg", "alias": "challenger"},
+    {"name": "intent-classifier-svm", "alias": "challenger"},
+    {"name": "intent-classifier-distilbert", "alias": "champion"},
+]
+
+
+@router.get("/models", response_model=ModelsResponse)
+def list_models() -> ModelsResponse:
+    return ModelsResponse(models=[ModelInfo(name=m["name"], alias=m["alias"]) for m in REGISTERED_MODELS])
+
+
+@router.get("/models/loaded")
+def list_loaded_models(request: Request) -> dict:
+    predictors = request.app.state.predictors
+    return {"loaded_models": list(predictors.keys())}

api/routes/monitoring.py

@@ -0,0 +1,47 @@
+from fastapi import APIRouter, HTTPException, Request
+
+from api.schemas import ABStatsResponse, DriftResponse
+from src.monitoring.drift import (
+    get_confidence_drift,
+    get_drift_summary,
+    get_oos_rate_drift,
+    load_current_data,
+    load_reference_data,
+    run_drift_report,
+)
+
+router = APIRouter()
+
+
+@router.get("/monitoring/drift", response_model=DriftResponse)
+def get_drift() -> DriftResponse:
+    try:
+        reference_df = load_reference_data()
+        current_df = load_current_data()
+    except FileNotFoundError as e:
+        raise HTTPException(status_code=404, detail=str(e))
+
+    snapshot = run_drift_report(reference_df, current_df)
+    drift_summary = get_drift_summary(snapshot)
+    confidence_drift = get_confidence_drift(reference_df, current_df)
+    oos_rate_drift = get_oos_rate_drift(reference_df, current_df)
+
+    return DriftResponse(
+        drift_summary=drift_summary,
+        confidence_drift=confidence_drift,
+        oos_rate_drift=oos_rate_drift,
+    )
+
+
+@router.get("/monitoring/ab-stats", response_model=ABStatsResponse)
+def get_ab_stats(request: Request) -> ABStatsResponse:
+    ab_router = request.app.state.ab_router
+    stats = ab_router.get_stats()
+    return ABStatsResponse(**stats)
+
+
+@router.post("/monitoring/reset")
+def reset_monitoring(request: Request) -> dict:
+    ab_router = request.app.state.ab_router
+    ab_router.reset_stats()
+    return {"status": "reset"}

api/routes/predict.py

@@ -0,0 +1,78 @@
+from fastapi import APIRouter, HTTPException, Request
+
+from api.schemas import (
+    BatchRequest,
+    BatchResponse,
+    PredictRequest,
+    PredictResponse,
+    TopIntent,
+)
+from src.monitoring.drift import append_current_data
+
+router = APIRouter()
+
+
+def _to_predict_response(result, ab_variant: str | None = None) -> PredictResponse:
+    return PredictResponse(
+        intent=result.intent,
+        confidence=result.confidence,
+        top5=[TopIntent(**item) for item in result.top5],
+        latency_ms=result.latency_ms,
+        is_oos=result.is_oos,
+        model_used=result.model_used,
+        ab_variant=ab_variant,
+    )
+
+
+@router.post("/predict", response_model=PredictResponse)
+def predict(payload: PredictRequest, request: Request) -> PredictResponse:
+    predictors = request.app.state.predictors
+    ab_router = request.app.state.ab_router
+
+    if payload.model_type:
+        if payload.model_type not in predictors:
+            raise HTTPException(status_code=400, detail=f"unknown model_type: {payload.model_type}")
+        result = predictors[payload.model_type].predict(payload.text)
+        response = _to_predict_response(result)
+    else:
+        result_dict = ab_router.predict(payload.text)
+        response = PredictResponse(
+            intent=result_dict["intent"],
+            confidence=result_dict["confidence"],
+            top5=[TopIntent(**item) for item in result_dict["top5"]],
+            latency_ms=result_dict["latency_ms"],
+            is_oos=result_dict["is_oos"],
+            model_used=result_dict["model_used"],
+            ab_variant=result_dict["ab_variant"],
+        )
+
+    append_current_data(
+        texts=[payload.text],
+        predictions=[response.intent],
+        confidences=[response.confidence],
+    )
+
+    return response
+
+
+@router.post("/predict/batch", response_model=BatchResponse)
+def predict_batch(payload: BatchRequest, request: Request) -> BatchResponse:
+    predictors = request.app.state.predictors
+
+    model_type = payload.model_type or "transformer"
+    if model_type not in predictors:
+        raise HTTPException(status_code=400, detail=f"unknown model_type: {model_type}")
+
+    predictor = predictors[model_type]
+    results = predictor.predict_batch(payload.texts)
+    responses = [_to_predict_response(r) for r in results]
+
+    total_latency = sum(r.latency_ms for r in responses)
+
+    append_current_data(
+        texts=payload.texts,
+        predictions=[r.intent for r in responses],
+        confidences=[r.confidence for r in responses],
+    )
+
+    return BatchResponse(predictions=responses, total_latency_ms=round(total_latency, 3))

api/schemas.py

@@ -0,0 +1,58 @@
+from pydantic import BaseModel, Field
+
+
+class PredictRequest(BaseModel):
+    text: str = Field(..., min_length=1, max_length=500)
+    model_type: str | None = None
+
+
+class TopIntent(BaseModel):
+    intent: str
+    confidence: float
+
+
+class PredictResponse(BaseModel):
+    intent: str
+    confidence: float
+    top5: list[TopIntent]
+    latency_ms: float
+    is_oos: bool
+    model_used: str
+    ab_variant: str | None = None
+
+
+class BatchRequest(BaseModel):
+    texts: list[str] = Field(..., min_length=1, max_length=100)
+    model_type: str | None = None
+
+
+class BatchResponse(BaseModel):
+    predictions: list[PredictResponse]
+    total_latency_ms: float
+
+
+class ModelInfo(BaseModel):
+    name: str
+    alias: str
+    version: str | None = None
+
+
+class ModelsResponse(BaseModel):
+    models: list[ModelInfo]
+
+
+class HealthResponse(BaseModel):
+    status: str
+    models_loaded: list[str]
+
+
+class DriftResponse(BaseModel):
+    drift_summary: dict
+    confidence_drift: dict
+    oos_rate_drift: dict
+
+
+class ABStatsResponse(BaseModel):
+    model_a: dict
+    model_b: dict
+    split: float

app.py

@@ -0,0 +1,95 @@
+import os
+import subprocess
+import threading
+import time
+
+
+def start_api():
+    subprocess.run(
+        [
+            "uvicorn",
+            "api.main:app",
+            "--host",
+            "0.0.0.0",
+            "--port",
+            "8000",
+        ],
+        check=False,
+    )
+
+
+def wait_for_api(timeout: int = 120) -> bool:
+    import requests
+
+    start = time.time()
+    while time.time() - start < timeout:
+        try:
+            requests.get("http://localhost:8000/health", timeout=2)
+            return True
+        except Exception:
+            time.sleep(2)
+    return False
+
+
+def download_models():
+    from src.storage.s3 import download_artifact
+
+    print("downloading models from S3...")
+
+    os.makedirs("artifacts/models/distilbert", exist_ok=True)
+    os.makedirs("artifacts/vectorizers", exist_ok=True)
+
+    download_artifact("classical/logreg.pkl", "artifacts/models/logreg.pkl")
+    download_artifact("classical/svm.pkl", "artifacts/models/svm.pkl")
+    download_artifact("classical/tfidf.pkl", "artifacts/vectorizers/tfidf.pkl")
+    download_artifact("neural/vocab.pkl", "artifacts/models/vocab.pkl")
+    download_artifact("transformer/distilbert/config.json", "artifacts/models/distilbert/config.json")
+    download_artifact("transformer/distilbert/model.safetensors", "artifacts/models/distilbert/model.safetensors")
+    download_artifact("transformer/distilbert/tokenizer.json", "artifacts/models/distilbert/tokenizer.json")
+    download_artifact("transformer/distilbert/tokenizer_config.json", "artifacts/models/distilbert/tokenizer_config.json")
+    download_artifact("transformer/distilbert/special_tokens_map.json", "artifacts/models/distilbert/special_tokens_map.json")
+    download_artifact("transformer/distilbert/vocab.txt", "artifacts/models/distilbert/vocab.txt")
+
+    print("models downloaded")
+
+
+def generate_label_map():
+    from src.data.loader import load_clinc150, save_splits
+    from src.data.preprocessor import preprocess
+
+    label_map_path = "artifacts/label_map.json"
+    if os.path.exists(label_map_path):
+        print("label map already exists")
+        return
+
+    print("generating label map...")
+    splits = load_clinc150("plus")
+    save_splits(splits)
+    preprocess(splits)
+    print("label map generated")
+
+
+if __name__ == "__main__":
+    download_models()
+    generate_label_map()
+
+    api_thread = threading.Thread(target=start_api, daemon=True)
+    api_thread.start()
+
+    print("waiting for API to start...")
+    if wait_for_api():
+        print("API ready")
+    else:
+        print("API startup timed out — continuing anyway")
+
+    os.execv(
+        "/usr/bin/streamlit",
+        [
+            "streamlit",
+            "run",
+            "app/streamlit_app.py",
+            "--server.port=7860",
+            "--server.address=0.0.0.0",
+            "--server.headless=true",
+        ],
+    )

app/api_client.py

@@ -0,0 +1,62 @@
+import os
+
+import requests
+import streamlit as st
+
+API_BASE_URL = os.getenv("API_BASE_URL", "http://localhost:8000")
+
+
+def api_predict(text: str, model_type: str | None = None) -> dict:
+    payload = {"text": text}
+    if model_type:
+        payload["model_type"] = model_type
+    response = requests.post(f"{API_BASE_URL}/predict", json=payload, timeout=30)
+    response.raise_for_status()
+    return response.json()
+
+
+def api_health() -> dict:
+    response = requests.get(f"{API_BASE_URL}/health", timeout=10)
+    response.raise_for_status()
+    return response.json()
+
+
+def api_models() -> dict:
+    response = requests.get(f"{API_BASE_URL}/models", timeout=10)
+    response.raise_for_status()
+    return response.json()
+
+
+def api_ab_stats() -> dict:
+    response = requests.get(f"{API_BASE_URL}/monitoring/ab-stats", timeout=10)
+    response.raise_for_status()
+    return response.json()
+
+
+def api_drift() -> dict:
+    response = requests.get(f"{API_BASE_URL}/monitoring/drift", timeout=30)
+    response.raise_for_status()
+    return response.json()
+
+
+def api_reset_monitoring() -> dict:
+    response = requests.post(f"{API_BASE_URL}/monitoring/reset", timeout=10)
+    response.raise_for_status()
+    return response.json()
+
+
+def check_api_connection() -> bool:
+    try:
+        api_health()
+        return True
+    except requests.exceptions.RequestException:
+        return False
+
+
+def require_api():
+    if not check_api_connection():
+        st.error(
+            f"Cannot connect to API at {API_BASE_URL}. "
+            "Start it with: `uv run uvicorn api.main:app --reload --port 8000`"
+        )
+        st.stop()

app/pages/compare.py

@@ -0,0 +1,100 @@
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+import pandas as pd
+import streamlit as st
+
+from app.api_client import api_predict, require_api
+
+require_api()
+
+st.title("Compare Models")
+st.caption("Side-by-side comparison of all trained models on CLINC150 test set.")
+
+RESULTS = {
+    "logreg": {
+        "accuracy": 0.8058,
+        "macro_f1": 0.8416,
+        "weighted_f1": 0.7985,
+        "latency_p50_ms": 0.12,
+    },
+    "svm": {
+        "accuracy": 0.8002,
+        "macro_f1": 0.8474,
+        "weighted_f1": 0.7795,
+        "latency_p50_ms": 0.14,
+    },
+    "textcnn": {
+        "accuracy": 0.7740,
+        "macro_f1": 0.8244,
+        "weighted_f1": 0.7646,
+        "latency_p50_ms": 97.6,
+    },
+    "rnn": {
+        "accuracy": 0.1818,
+        "macro_f1": 0.0020,
+        "weighted_f1": 0.0560,
+        "latency_p50_ms": 124.5,
+    },
+    "lstm": {
+        "accuracy": 0.7669,
+        "macro_f1": 0.8113,
+        "weighted_f1": 0.7555,
+        "latency_p50_ms": 249.2,
+    },
+    "distilbert": {
+        "accuracy": 0.8765,
+        "macro_f1": 0.9019,
+        "weighted_f1": 0.8722,
+        "latency_p50_ms": 9.6,
+    },
+}
+
+df = pd.DataFrame(RESULTS).T
+df.index.name = "model"
+
+st.subheader("Metrics Table")
+st.dataframe(
+    df.style.format(
+        {
+            "accuracy": "{:.2%}",
+            "macro_f1": "{:.2%}",
+            "weighted_f1": "{:.2%}",
+            "latency_p50_ms": "{:.2f} ms",
+        }
+    ).background_gradient(subset=["accuracy", "macro_f1", "weighted_f1"], cmap="Greens"),
+    use_container_width=True,
+)
+
+st.subheader("Accuracy & F1 Comparison")
+chart_df = df[["accuracy", "macro_f1", "weighted_f1"]]
+st.bar_chart(chart_df)
+
+st.subheader("Latency Comparison")
+st.bar_chart(df[["latency_p50_ms"]])
+
+st.divider()
+st.subheader("Side-by-Side Prediction")
+st.caption("Run the same input through multiple models at once.")
+
+text = st.text_input("Enter a query to compare", placeholder="e.g. cancel my hotel reservation")
+selected_models = st.multiselect(
+    "Models to compare",
+    options=["classical", "svm", "transformer"],
+    default=["classical", "transformer"],
+)
+
+if st.button("Compare", type="primary", disabled=not (text and selected_models)):
+    cols = st.columns(len(selected_models))
+    for col, model_type in zip(cols, selected_models):
+        with col:
+            with st.spinner(f"running {model_type}..."):
+                result = api_predict(text, model_type)
+            st.markdown(f"**{model_type}**")
+            st.metric("Intent", result["intent"])
+            st.metric("Confidence", f"{result['confidence']:.2%}")
+            st.metric("Latency", f"{result['latency_ms']:.1f} ms")
+            if result["is_oos"]:
+                st.warning("flagged OOS")

app/pages/monitoring.py

@@ -0,0 +1,90 @@
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+import streamlit as st
+
+from app.api_client import api_ab_stats, api_drift, api_reset_monitoring, require_api
+
+require_api()
+
+st.title("Monitoring Dashboard")
+st.caption("Live A/B test stats and data drift detection.")
+
+st.subheader("A/B Test Stats")
+
+if st.button("Refresh stats"):
+    st.rerun()
+
+try:
+    stats = api_ab_stats()
+
+    col1, col2 = st.columns(2)
+
+    with col1:
+        st.markdown(f"### Model A — `{stats['model_a']['model_name']}`")
+        st.metric("Requests", stats["model_a"]["request_count"])
+        st.metric("Avg Latency", f"{stats['model_a']['avg_latency_ms']:.2f} ms")
+        st.metric("Avg Confidence", f"{stats['model_a']['avg_confidence']:.2%}")
+        st.metric("OOS Rate", f"{stats['model_a']['oos_rate']:.2%}")
+
+    with col2:
+        st.markdown(f"### Model B — `{stats['model_b']['model_name']}`")
+        st.metric("Requests", stats["model_b"]["request_count"])
+        st.metric("Avg Latency", f"{stats['model_b']['avg_latency_ms']:.2f} ms")
+        st.metric("Avg Confidence", f"{stats['model_b']['avg_confidence']:.2%}")
+        st.metric("OOS Rate", f"{stats['model_b']['oos_rate']:.2%}")
+
+    st.caption(f"Configured split: {stats['split']:.0%} to model B")
+
+except Exception as e:
+    st.warning(f"No A/B stats yet. Make some predictions on the Predict page first. ({e})")
+
+if st.button("Reset A/B stats"):
+    api_reset_monitoring()
+    st.success("Stats reset.")
+    st.rerun()
+
+st.divider()
+st.subheader("Data Drift Report")
+st.caption("Compares reference traffic (initial baseline) against recent production traffic.")
+
+if st.button("Run drift check"):
+    try:
+        with st.spinner("computing drift..."):
+            drift = api_drift()
+
+        col1, col2, col3 = st.columns(3)
+        col1.metric(
+            "Drifted Columns",
+            int(drift["drift_summary"].get("drifted_columns_count", 0)),
+        )
+        col2.metric(
+            "Confidence Drop",
+            f"{drift['confidence_drift']['confidence_drop']:.2%}",
+            delta=f"{-drift['confidence_drift']['confidence_drop']:.2%}",
+        )
+        col3.metric(
+            "OOS Rate Increase",
+            f"{drift['oos_rate_drift']['oos_rate_increase']:.2%}",
+        )
+
+        if drift["confidence_drift"]["is_degraded"]:
+            st.error("Confidence has degraded significantly vs reference traffic.")
+        else:
+            st.success("Confidence is stable vs reference traffic.")
+
+        if drift["oos_rate_drift"]["is_anomalous"]:
+            st.error("OOS rate has increased anomalously — possible topic drift in incoming queries.")
+        else:
+            st.success("OOS rate is within normal range.")
+
+        with st.expander("Raw drift details"):
+            st.json(drift)
+
+    except Exception as e:
+        st.warning(
+            "No reference/current data available yet. Run `verify_phase11.py` first to "
+            f"generate baseline monitoring data. ({e})"
+        )

app/pages/predict.py

@@ -0,0 +1,67 @@
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+import pandas as pd
+import streamlit as st
+
+from app.api_client import api_predict, require_api
+
+require_api()
+
+st.title("Predict Intent")
+st.caption("Test the intent classifier live, switching between models or letting the A/B router decide.")
+
+col1, col2 = st.columns([3, 1])
+
+with col1:
+    text = st.text_input("Enter a query", placeholder="e.g. what is my account balance")
+
+with col2:
+    model_choice = st.selectbox(
+        "Model",
+        options=["A/B Router", "Classical (LogReg)", "SVM", "Transformer (DistilBERT)"],
+    )
+
+model_map = {
+    "A/B Router": None,
+    "Classical (LogReg)": "classical",
+    "SVM": "svm",
+    "Transformer (DistilBERT)": "transformer",
+}
+
+if st.button("Predict", type="primary", disabled=not text):
+    with st.spinner("predicting..."):
+        result = api_predict(text, model_map[model_choice])
+
+    col_a, col_b, col_c = st.columns(3)
+    col_a.metric("Intent", result["intent"])
+    col_b.metric("Confidence", f"{result['confidence']:.2%}")
+    col_c.metric("Latency", f"{result['latency_ms']:.1f} ms")
+
+    if result["is_oos"]:
+        st.warning("This query was flagged as out-of-scope (low confidence).")
+
+    if result.get("ab_variant"):
+        st.info(f"Served by A/B variant **{result['ab_variant']}** using model `{result['model_used']}`")
+    else:
+        st.info(f"Served by model `{result['model_used']}`")
+
+    st.subheader("Top 5 Predictions")
+    df = pd.DataFrame(result["top5"])
+    df["confidence"] = df["confidence"].astype(float)
+    st.bar_chart(df.set_index("intent")["confidence"], horizontal=True)
+
+st.divider()
+st.caption("Sample queries to try:")
+samples = [
+    "what is my account balance",
+    "book a flight to new york",
+    "set an alarm for 7am",
+    "tell me about quantum physics",
+    "write me a poem about the ocean",
+]
+cols = st.columns(len(samples))
+for col, sample in zip(cols, samples):
+    col.code(sample, language=None)

app/streamlit_app.py

@@ -0,0 +1,14 @@
+import streamlit as st
+
+st.set_page_config(
+    page_title="Intent Classifier",
+    page_icon=":material/psychology:",
+    layout="wide",
+)
+
+predict_page = st.Page("pages/predict.py", title="Predict", icon=":material/chat:")
+compare_page = st.Page("pages/compare.py", title="Compare Models", icon=":material/bar_chart:")
+monitoring_page = st.Page("pages/monitoring.py", title="Monitoring", icon=":material/monitoring:")
+
+pg = st.navigation([predict_page, compare_page, monitoring_page])
+pg.run()

artifacts/label_map.json

@@ -0,0 +1,153 @@
+{
+  "accept_reservations": 0,
+  "account_blocked": 1,
+  "alarm": 2,
+  "application_status": 3,
+  "apr": 4,
+  "are_you_a_bot": 5,
+  "balance": 6,
+  "bill_balance": 7,
+  "bill_due": 8,
+  "book_flight": 9,
+  "book_hotel": 10,
+  "calculator": 11,
+  "calendar": 12,
+  "calendar_update": 13,
+  "calories": 14,
+  "cancel": 15,
+  "cancel_reservation": 16,
+  "car_rental": 17,
+  "card_declined": 18,
+  "carry_on": 19,
+  "change_accent": 20,
+  "change_ai_name": 21,
+  "change_language": 22,
+  "change_speed": 23,
+  "change_user_name": 24,
+  "change_volume": 25,
+  "confirm_reservation": 26,
+  "cook_time": 27,
+  "credit_limit": 28,
+  "credit_limit_change": 29,
+  "credit_score": 30,
+  "current_location": 31,
+  "damaged_card": 32,
+  "date": 33,
+  "definition": 34,
+  "direct_deposit": 35,
+  "directions": 36,
+  "distance": 37,
+  "do_you_have_pets": 38,
+  "exchange_rate": 39,
+  "expiration_date": 40,
+  "find_phone": 41,
+  "flight_status": 42,
+  "flip_coin": 43,
+  "food_last": 44,
+  "freeze_account": 45,
+  "fun_fact": 46,
+  "gas": 47,
+  "gas_type": 48,
+  "goodbye": 49,
+  "greeting": 50,
+  "how_busy": 51,
+  "how_old_are_you": 52,
+  "improve_credit_score": 53,
+  "income": 54,
+  "ingredient_substitution": 55,
+  "ingredients_list": 56,
+  "insurance": 57,
+  "insurance_change": 58,
+  "interest_rate": 59,
+  "international_fees": 60,
+  "international_visa": 61,
+  "jump_start": 62,
+  "last_maintenance": 63,
+  "lost_luggage": 64,
+  "make_call": 65,
+  "maybe": 66,
+  "meal_suggestion": 67,
+  "meaning_of_life": 68,
+  "measurement_conversion": 69,
+  "meeting_schedule": 70,
+  "min_payment": 71,
+  "mpg": 72,
+  "new_card": 73,
+  "next_holiday": 74,
+  "next_song": 75,
+  "no": 76,
+  "nutrition_info": 77,
+  "oil_change_how": 78,
+  "oil_change_when": 79,
+  "oos": 80,
+  "order": 81,
+  "order_checks": 82,
+  "order_status": 83,
+  "pay_bill": 84,
+  "payday": 85,
+  "pin_change": 86,
+  "play_music": 87,
+  "plug_type": 88,
+  "pto_balance": 89,
+  "pto_request": 90,
+  "pto_request_status": 91,
+  "pto_used": 92,
+  "recipe": 93,
+  "redeem_rewards": 94,
+  "reminder": 95,
+  "reminder_update": 96,
+  "repeat": 97,
+  "replacement_card_duration": 98,
+  "report_fraud": 99,
+  "report_lost_card": 100,
+  "reset_settings": 101,
+  "restaurant_reservation": 102,
+  "restaurant_reviews": 103,
+  "restaurant_suggestion": 104,
+  "rewards_balance": 105,
+  "roll_dice": 106,
+  "rollover_401k": 107,
+  "routing": 108,
+  "schedule_maintenance": 109,
+  "schedule_meeting": 110,
+  "share_location": 111,
+  "shopping_list": 112,
+  "shopping_list_update": 113,
+  "smart_home": 114,
+  "spelling": 115,
+  "spending_history": 116,
+  "sync_device": 117,
+  "taxes": 118,
+  "tell_joke": 119,
+  "text": 120,
+  "thank_you": 121,
+  "time": 122,
+  "timer": 123,
+  "timezone": 124,
+  "tire_change": 125,
+  "tire_pressure": 126,
+  "todo_list": 127,
+  "todo_list_update": 128,
+  "traffic": 129,
+  "transactions": 130,
+  "transfer": 131,
+  "translate": 132,
+  "travel_alert": 133,
+  "travel_notification": 134,
+  "travel_suggestion": 135,
+  "uber": 136,
+  "update_playlist": 137,
+  "user_name": 138,
+  "vaccines": 139,
+  "w2": 140,
+  "weather": 141,
+  "what_are_your_hobbies": 142,
+  "what_can_i_ask_you": 143,
+  "what_is_your_name": 144,
+  "what_song": 145,
+  "where_are_you_from": 146,
+  "whisper_mode": 147,
+  "who_do_you_work_for": 148,
+  "who_made_you": 149,
+  "yes": 150
+}

configs/classical.yaml

@@ -0,0 +1,25 @@
+model:
+  type: logistic_regression
+  C: 1.0
+  max-iter: 1000
+  random_state: 42
+
+tfidf:
+  max-features: 50000
+  ngram_range: [1, 2]
+  min_df: 2
+
+data:
+  dataset: clinc_oos
+  subset: plus
+  test_size: 0.1
+  val_size: 0.1
+  random_state: 42
+
+mlflow:
+  experiment_name: intent-classifier
+  run_name: logistic_regression
+
+s3:
+  bucket: intent-classifier145
+  prefix: classical

configs/neural.yaml

@@ -0,0 +1,41 @@
+model:
+  textcnn:
+    embedding_dim: 128
+    num_filters: 128
+    kernel_sizes: [2, 3, 4]
+    dropout: 0.3
+
+  rnn:
+    embedding_dim: 128
+    hidden_dim: 256
+    num_layers: 2
+    dropout: 0.3
+
+  lstm:
+    embedding_dim: 128
+    hidden_dim: 256
+    num_layers: 2
+    dropout: 0.3
+
+training:
+  epochs: 50
+  batch_size: 64
+  learning_rate: 0.001
+  random_state: 42
+  patience: 5
+  grad_clip: 1.0
+
+data:
+  dataset: clinc_oos
+  subset: plus
+  test_size: 0.1
+  val_size: 0.1
+  random_state: 42
+
+mlflow:
+  experiment_name: intent-classifier
+  run_name: textcnn
+
+s3:
+  bucket: intent-classifier145
+  prefix: neural

configs/transformer.yaml

@@ -0,0 +1,29 @@
+model:
+  type: distilbert
+  name: distilbert-base-uncased
+  num_labels: 151
+  dropout: 0.1
+
+training:
+  epochs: 5
+  batch_size: 32
+  learning_rate: 0.00002
+  warmup_steps: 500
+  weight_decay: 0.01
+  random_state: 42
+
+data:
+  dataset: clinc_oos
+  subset: plus
+  max_length: 128
+  test_size: 0.1
+  val_size: 0.1
+  random_state: 42
+
+mlflow:
+  experiment_name: intent-classifier
+  run_name: distilbert-finetuned
+
+s3:
+  bucket: intent-classifier145
+  prefix: transformer

requirements.txt

@@ -0,0 +1,19 @@
+torch==2.6.0
+transformers==4.51.3
+accelerate==1.5.2
+datasets==3.3.2
+numpy==2.2.3
+pandas==2.2.3
+mlflow==2.21.3
+fastapi==0.115.12
+uvicorn==0.34.0
+streamlit==1.43.2
+pyyaml==6.0.2
+python-dotenv==1.0.1
+boto3==1.37.38
+evidently==0.7.5
+httpx==0.28.1
+pydantic-settings==2.8.1
+matplotlib==3.10.1
+seaborn==0.13.2
+scikit-learn==1.6.1

src/data/loader.py

@@ -0,0 +1,45 @@
+from pathlib import Path
+
+import pandas as pd
+from datasets import load_dataset
+
+
+def load_clinc150(subset: str = "plus") -> dict[str, pd.DataFrame]:
+    dataset = load_dataset("clinc/clinc_oos", subset)
+
+    intent_feature = dataset["train"].features["intent"]
+
+    splits = {}
+    for split_name in ["train", "validation", "test"]:
+        records = [
+            {
+                "text": item["text"],
+                "label": intent_feature.int2str(item["intent"]),
+            }
+            for item in dataset[split_name]
+        ]
+        splits[split_name] = pd.DataFrame(records)
+
+    return splits
+
+
+def save_splits(splits: dict[str, pd.DataFrame], save_dir: str = "data/raw") -> None:
+    path = Path(save_dir)
+    path.mkdir(parents=True, exist_ok=True)
+
+    for split_name, df in splits.items():
+        df.to_csv(path / f"{split_name}.csv", index=False)
+
+
+def load_splits(load_dir: str = "data/raw") -> dict[str, pd.DataFrame]:
+    path = Path(load_dir)
+    split_names = ["train", "validation", "test"]
+
+    splits = {}
+    for split_name in split_names:
+        file_path = path / f"{split_name}.csv"
+        if not file_path.exists():
+            raise FileNotFoundError(f"Split not found: {file_path}")
+        splits[split_name] = pd.read_csv(file_path)
+
+    return splits

src/data/preprocessor.py

@@ -0,0 +1,62 @@
+import json
+import re
+from pathlib import Path
+
+import pandas as pd
+
+
+def clean_text(text: str) -> str:
+    text = text.lower().strip()
+    text = re.sub(r"\s+", " ", text)
+    return text
+
+
+def build_label_map(train_df: pd.DataFrame, label_col: str = "label") -> dict[str, int]:
+    labels = sorted(train_df[label_col].unique().tolist())
+    return {label: idx for idx, label in enumerate(labels)}
+
+
+def encode_labels(
+    df: pd.DataFrame,
+    label_map: dict[str, int],
+    label_col: str = "label",
+) -> pd.DataFrame:
+    df = df.copy()
+    df["label_id"] = df[label_col].map(label_map)
+    return df
+
+
+def save_label_map(label_map: dict[str, int], save_path: str = "artifacts/label_map.json") -> None:
+    path = Path(save_path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w") as f:
+        json.dump(label_map, f, indent=2)
+
+
+def load_label_map(load_path: str = "artifacts/label_map.json") -> dict[str, int]:
+    path = Path(load_path)
+    if not path.exists():
+        raise FileNotFoundError(f"Label map not found: {path}")
+    with open(path) as f:
+        return json.load(f)
+
+
+def get_id_to_label(label_map: dict[str, int]) -> dict[int, str]:
+    return {idx: label for label, idx in label_map.items()}
+
+
+def preprocess(
+    splits: dict[str, pd.DataFrame],
+) -> tuple[dict[str, pd.DataFrame], dict[str, int]]:
+    label_map = build_label_map(splits["train"])
+
+    processed = {}
+    for split_name, df in splits.items():
+        df = df.copy()
+        df["text"] = df["text"].apply(clean_text)
+        df = encode_labels(df, label_map)
+        processed[split_name] = df
+
+    save_label_map(label_map)
+
+    return processed, label_map

src/evaluation/evaluate_all.py

@@ -0,0 +1,417 @@
+import time
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import mlflow
+import numpy as np
+import pandas as pd
+import torch
+from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix
+from torch.utils.data import DataLoader
+
+from src.data.loader import load_splits
+from src.data.preprocessor import preprocess
+from src.features.tfidf import load_vectorizer, transform
+from src.models.classical import LogisticRegressionModel, SVMModel
+from src.models.neural import IntentDatasetNN, LSTMModel, TextCNN, Vocabulary
+from src.models.transformer import IntentDatasetHF, TransformerModel
+from src.utils.config import load_config
+from src.utils.mlflow_utils import get_or_create_experiment
+from src.utils.settings import settings
+
+VECTORIZER_PATH = "artifacts/vectorizers/tfidf.pkl"
+VOCAB_PATH = "artifacts/models/vocab.pkl"
+LOGREG_PATH = "artifacts/models/logreg.pkl"
+SVM_PATH = "artifacts/models/svm.pkl"
+TEXTCNN_PATH = "artifacts/models/textcnn.pt"
+RNN_PATH = "artifacts/models/rnn.pt"
+LSTM_PATH = "artifacts/models/lstm.pt"
+DISTILBERT_DIR = "artifacts/models/distilbert"
+MAX_LENGTH_NN = 32
+MAX_LENGTH_HF = 128
+REPORT_DIR = Path("artifacts/evaluation")
+
+
+def load_data():
+    splits = load_splits("data/raw")
+    processed, label_map = preprocess(splits)
+    id_to_label = {v: k for k, v in label_map.items()}
+    label_names = [id_to_label[i] for i in range(len(label_map))]
+    return processed, label_map, label_names
+
+
+def compute_metrics(y_true: np.ndarray, y_pred: np.ndarray) -> dict:
+    from sklearn.metrics import accuracy_score, f1_score
+
+    return {
+        "accuracy": round(accuracy_score(y_true, y_pred), 4),
+        "macro_f1": round(f1_score(y_true, y_pred, average="macro", zero_division=0), 4),
+        "weighted_f1": round(f1_score(y_true, y_pred, average="weighted", zero_division=0), 4),
+    }
+
+
+def measure_latency(predict_fn, input_data, n_runs: int = 50) -> dict:
+    latencies = []
+    for _ in range(n_runs):
+        start = time.perf_counter()
+        predict_fn(input_data)
+        latencies.append((time.perf_counter() - start) * 1000)
+    latencies = np.array(latencies)
+    return {
+        "latency_mean_ms": round(float(np.mean(latencies)), 3),
+        "latency_p50_ms": round(float(np.percentile(latencies, 50)), 3),
+        "latency_p95_ms": round(float(np.percentile(latencies, 95)), 3),
+        "latency_p99_ms": round(float(np.percentile(latencies, 99)), 3),
+    }
+
+
+def eval_logreg(processed: dict, label_names: list[str]) -> tuple[dict, np.ndarray]:
+    print("  evaluating logistic regression...")
+    vectorizer = load_vectorizer(VECTORIZER_PATH)
+    X_test = transform(vectorizer, processed["test"]["text"].tolist())
+    y_test = processed["test"]["label_id"].values
+
+    model = LogisticRegressionModel()
+    model.load(LOGREG_PATH)
+
+    y_pred = model.predict(X_test)
+    metrics = compute_metrics(y_test, y_pred)
+    latency = measure_latency(model.predict, X_test[:100])
+    return {**metrics, **latency}, y_pred
+
+
+def eval_svm(processed: dict, label_names: list[str]) -> tuple[dict, np.ndarray]:
+    print("  evaluating svm...")
+    vectorizer = load_vectorizer(VECTORIZER_PATH)
+    X_test = transform(vectorizer, processed["test"]["text"].tolist())
+    y_test = processed["test"]["label_id"].values
+
+    model = SVMModel()
+    model.load(SVM_PATH)
+
+    y_pred = model.predict(X_test)
+    metrics = compute_metrics(y_test, y_pred)
+    latency = measure_latency(model.predict, X_test[:100])
+    return {**metrics, **latency}, y_pred
+
+
+def eval_neural(
+    model_type: str,
+    model_path: str,
+    processed: dict,
+    label_map: dict,
+) -> tuple[dict, np.ndarray]:
+    print(f"  evaluating {model_type}...")
+    vocab = Vocabulary.load(VOCAB_PATH)
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    num_classes = len(label_map)
+
+    neural_config = load_config("neural")
+    model_cfg = neural_config["model"][model_type]
+
+    if model_type == "textcnn":
+        model = TextCNN(
+            vocab_size=len(vocab),
+            embedding_dim=model_cfg["embedding_dim"],
+            num_filters=model_cfg["num_filters"],
+            kernel_sizes=model_cfg["kernel_sizes"],
+            num_classes=num_classes,
+            dropout=model_cfg["dropout"],
+        )
+    elif model_type == "rnn":
+        from src.models.neural import RNNModel
+
+        model = RNNModel(
+            vocab_size=len(vocab),
+            embedding_dim=model_cfg["embedding_dim"],
+            hidden_dim=model_cfg["hidden_dim"],
+            num_layers=model_cfg["num_layers"],
+            num_classes=num_classes,
+            dropout=model_cfg["dropout"],
+        )
+    elif model_type == "lstm":
+        model = LSTMModel(
+            vocab_size=len(vocab),
+            embedding_dim=model_cfg["embedding_dim"],
+            hidden_dim=model_cfg["hidden_dim"],
+            num_layers=model_cfg["num_layers"],
+            num_classes=num_classes,
+            dropout=model_cfg["dropout"],
+        )
+
+    model.load(model_path)
+    model.to(device)
+    model.eval()
+
+    dataset = IntentDatasetNN(
+        processed["test"]["text"].tolist(),
+        processed["test"]["label_id"].tolist(),
+        vocab,
+        MAX_LENGTH_NN,
+    )
+    loader = DataLoader(dataset, batch_size=64, shuffle=False)
+
+    all_preds = []
+    y_test = processed["test"]["label_id"].values
+
+    with torch.no_grad():
+        for inputs, labels in loader:
+            inputs = inputs.to(device)
+            logits = model(inputs)
+            preds = logits.argmax(dim=1).cpu().numpy()
+            all_preds.extend(preds)
+
+    y_pred = np.array(all_preds)
+    metrics = compute_metrics(y_test, y_pred)
+
+    def predict_fn(loader):
+        with torch.no_grad():
+            for inputs, _ in loader:
+                model(inputs.to(device))
+
+    latency = measure_latency(predict_fn, loader, n_runs=20)
+    return {**metrics, **latency}, y_pred
+
+
+def eval_distilbert(processed: dict, label_map: dict) -> tuple[dict, np.ndarray]:
+    print("  evaluating distilbert...")
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    transformer = TransformerModel(
+        model_name=DISTILBERT_DIR,
+        num_labels=len(label_map),
+    )
+    transformer.model.to(device)
+    transformer.model.eval()
+
+    dataset = IntentDatasetHF(
+        processed["test"]["text"].tolist(),
+        processed["test"]["label_id"].tolist(),
+        transformer.tokenizer,
+        MAX_LENGTH_HF,
+    )
+    loader = DataLoader(dataset, batch_size=32, shuffle=False)
+
+    all_preds = []
+    y_test = processed["test"]["label_id"].values
+
+    with torch.no_grad():
+        for batch in loader:
+            input_ids = batch["input_ids"].to(device)
+            attention_mask = batch["attention_mask"].to(device)
+            outputs = transformer.model(input_ids=input_ids, attention_mask=attention_mask)
+            preds = outputs.logits.argmax(dim=1).cpu().numpy()
+            all_preds.extend(preds)
+
+    y_pred = np.array(all_preds)
+    metrics = compute_metrics(y_test, y_pred)
+
+    single_text = processed["test"]["text"].iloc[:1].tolist()
+    single_dataset = IntentDatasetHF(
+        single_text,
+        [0],
+        transformer.tokenizer,
+        MAX_LENGTH_HF,
+    )
+    single_loader = DataLoader(single_dataset, batch_size=1)
+
+    def predict_fn(loader):
+        with torch.no_grad():
+            for batch in loader:
+                transformer.model(
+                    input_ids=batch["input_ids"].to(device),
+                    attention_mask=batch["attention_mask"].to(device),
+                )
+
+    latency = measure_latency(predict_fn, single_loader, n_runs=50)
+    return {**metrics, **latency}, y_pred
+
+
+def plot_comparison(results: dict, save_path: str) -> None:
+    df = pd.DataFrame(results).T.reset_index()
+    df.columns = ["model"] + list(df.columns[1:])
+
+    fig, axes = plt.subplots(1, 3, figsize=(18, 6))
+
+    metrics = ["accuracy", "macro_f1", "weighted_f1"]
+    titles = ["Accuracy", "Macro F1", "Weighted F1"]
+    colors = ["#4C72B0", "#DD8452", "#55A868", "#C44E52", "#8172B2", "#937860"]
+
+    for ax, metric, title in zip(axes, metrics, titles):
+        bars = ax.bar(df["model"], df[metric].astype(float), color=colors)
+        ax.set_title(title, fontsize=14)
+        ax.set_ylim(0, 1.1)
+        ax.set_xticks(range(len(df["model"])))
+        ax.set_xticklabels(df["model"], rotation=30, ha="right")
+        for bar, val in zip(bars, df[metric].astype(float)):
+            ax.text(
+                bar.get_x() + bar.get_width() / 2,
+                bar.get_height() + 0.01,
+                f"{val:.3f}",
+                ha="center",
+                va="bottom",
+                fontsize=9,
+            )
+
+    plt.suptitle("Model Comparison — CLINC150", fontsize=16, y=1.02)
+    plt.tight_layout()
+    plt.savefig(save_path, dpi=100, bbox_inches="tight")
+    plt.close()
+    print(f"  saved: {save_path}")
+
+
+def plot_latency(results: dict, save_path: str) -> None:
+    models = list(results.keys())
+    p50 = [results[m]["latency_p50_ms"] for m in models]
+    p95 = [results[m]["latency_p95_ms"] for m in models]
+
+    x = np.arange(len(models))
+    width = 0.35
+
+    fig, ax = plt.subplots(figsize=(12, 6))
+    ax.bar(x - width / 2, p50, width, label="P50", color="#4C72B0")
+    ax.bar(x + width / 2, p95, width, label="P95", color="#DD8452")
+    ax.set_yscale("log")
+    ax.set_ylabel("Latency (ms) — log scale")
+    ax.set_title("Inference Latency Comparison")
+    ax.set_xticks(x)
+    ax.set_xticklabels(models, rotation=30, ha="right")
+    ax.legend()
+    plt.tight_layout()
+    plt.savefig(save_path, dpi=100, bbox_inches="tight")
+    plt.close()
+    print(f"  saved: {save_path}")
+
+
+def get_top_confused_pairs(
+    y_true: np.ndarray,
+    y_pred: np.ndarray,
+    label_names: list[str],
+    top_n: int = 20,
+) -> pd.DataFrame:
+    cm = confusion_matrix(y_true, y_pred)
+    np.fill_diagonal(cm, 0)
+
+    pairs = []
+    for i in range(cm.shape[0]):
+        for j in range(cm.shape[1]):
+            if cm[i, j] > 0:
+                pairs.append(
+                    {
+                        "true_label": label_names[i],
+                        "predicted_label": label_names[j],
+                        "count": int(cm[i, j]),
+                    }
+                )
+
+    df = pd.DataFrame(pairs).sort_values("count", ascending=False).head(top_n)
+    return df.reset_index(drop=True)
+
+
+def plot_top_confused_pairs(df: pd.DataFrame, model_name: str, save_path: str) -> None:
+    fig, ax = plt.subplots(figsize=(10, max(6, len(df) * 0.35)))
+    labels = [f"{row.true_label} -> {row.predicted_label}" for row in df.itertuples()]
+    ax.barh(labels, df["count"], color="#C44E52")
+    ax.invert_yaxis()
+    ax.set_xlabel("Misclassification Count")
+    ax.set_title(f"Top Confused Pairs — {model_name}")
+    plt.tight_layout()
+    plt.savefig(save_path, dpi=100, bbox_inches="tight")
+    plt.close()
+    print(f"  saved: {save_path}")
+
+
+def plot_oos_binary_confusion(
+    y_true: np.ndarray,
+    y_pred: np.ndarray,
+    label_map: dict,
+    model_name: str,
+    save_path: str,
+) -> None:
+    oos_id = label_map.get("oos")
+    y_true_binary = (y_true == oos_id).astype(int)
+    y_pred_binary = (y_pred == oos_id).astype(int)
+
+    cm = confusion_matrix(y_true_binary, y_pred_binary)
+    fig, ax = plt.subplots(figsize=(5, 5))
+    disp = ConfusionMatrixDisplay(
+        confusion_matrix=cm,
+        display_labels=["in-scope", "oos"],
+    )
+    disp.plot(ax=ax, colorbar=False, cmap="Blues")
+    ax.set_title(f"OOS Detection — {model_name}")
+    plt.tight_layout()
+    plt.savefig(save_path, dpi=100, bbox_inches="tight")
+    plt.close()
+    print(f"  saved: {save_path}")
+
+
+def main():
+    REPORT_DIR.mkdir(parents=True, exist_ok=True)
+
+    print("loading data...")
+    processed, label_map, label_names = load_data()
+    y_test = processed["test"]["label_id"].values
+
+    print("\nevaluating all models...")
+    results = {}
+    predictions = {}
+
+    results["logreg"], predictions["logreg"] = eval_logreg(processed, label_names)
+    results["svm"], predictions["svm"] = eval_svm(processed, label_names)
+    results["textcnn"], predictions["textcnn"] = eval_neural("textcnn", TEXTCNN_PATH, processed, label_map)
+    results["rnn"], predictions["rnn"] = eval_neural("rnn", RNN_PATH, processed, label_map)
+    results["lstm"], predictions["lstm"] = eval_neural("lstm", LSTM_PATH, processed, label_map)
+    results["distilbert"], predictions["distilbert"] = eval_distilbert(processed, label_map)
+
+    print("\ngenerating plots...")
+    plot_comparison(results, str(REPORT_DIR / "model_comparison.png"))
+    plot_latency(results, str(REPORT_DIR / "latency_comparison.png"))
+
+    for model_name, y_pred in predictions.items():
+        confused_df = get_top_confused_pairs(y_test, y_pred, label_names, top_n=20)
+        confused_csv_path = REPORT_DIR / f"{model_name}_top_confused_pairs.csv"
+        confused_df.to_csv(confused_csv_path, index=False)
+        print(f"  saved: {confused_csv_path}")
+
+        plot_top_confused_pairs(
+            confused_df,
+            model_name,
+            str(REPORT_DIR / f"{model_name}_top_confused_pairs.png"),
+        )
+
+        plot_oos_binary_confusion(
+            y_test,
+            y_pred,
+            label_map,
+            model_name,
+            str(REPORT_DIR / f"{model_name}_oos_binary.png"),
+        )
+
+    print("\nlogging to MLflow...")
+    mlflow.set_tracking_uri(settings.mlflow_tracking_uri)
+    experiment_id = get_or_create_experiment("intent-classifier")
+
+    with mlflow.start_run(experiment_id=experiment_id, run_name="unified-evaluation"):
+        for model_name, metrics in results.items():
+            for metric_name, value in metrics.items():
+                mlflow.log_metric(f"{model_name}.{metric_name}", value)
+
+        mlflow.log_artifact(str(REPORT_DIR / "model_comparison.png"))
+        mlflow.log_artifact(str(REPORT_DIR / "latency_comparison.png"))
+
+    print("\nfinal results:")
+    print(f"{'model':<14} {'accuracy':<12} {'macro_f1':<12} {'weighted_f1':<14} {'p50_ms':<12} {'p95_ms'}")
+    print("-" * 78)
+    for model_name, metrics in results.items():
+        print(
+            f"{model_name:<14} "
+            f"{metrics['accuracy']:<12} "
+            f"{metrics['macro_f1']:<12} "
+            f"{metrics['weighted_f1']:<14} "
+            f"{metrics['latency_p50_ms']:<12} "
+            f"{metrics['latency_p95_ms']}"
+        )
+
+
+if __name__ == "__main__":
+    main()

src/evaluation/metrics.py

@@ -0,0 +1,47 @@
+import time
+
+import numpy as np
+import pandas as pd
+from sklearn.metrics import accuracy_score, classification_report, f1_score
+
+
+def compute_classification_metrics(
+    y_true: np.ndarray,
+    y_pred: np.ndarray,
+) -> dict:
+    return {
+        "accuracy": round(accuracy_score(y_true, y_pred), 4),
+        "macro_f1": round(f1_score(y_true, y_pred, average="macro", zero_division=0), 4),
+        "weighted_f1": round(f1_score(y_true, y_pred, average="weighted", zero_division=0), 4),
+    }
+
+
+def compute_latency(predict_fn, inputs, n_runs: int = 100) -> dict:
+    latencies = []
+    for _ in range(n_runs):
+        start = time.perf_counter()
+        predict_fn(inputs)
+        latencies.append((time.perf_counter() - start) * 1000)
+
+    latencies = np.array(latencies)
+    return {
+        "latency_mean_ms": round(float(np.mean(latencies)), 3),
+        "latency_p50_ms": round(float(np.percentile(latencies, 50)), 3),
+        "latency_p95_ms": round(float(np.percentile(latencies, 95)), 3),
+        "latency_p99_ms": round(float(np.percentile(latencies, 99)), 3),
+    }
+
+
+def get_classification_report(
+    y_true: np.ndarray,
+    y_pred: np.ndarray,
+    label_names: list[str],
+) -> str:
+    return classification_report(y_true, y_pred, target_names=label_names, zero_division=0)
+
+
+def compare_models(results: dict) -> pd.DataFrame:
+    rows = []
+    for model_name, metrics in results.items():
+        rows.append({"model": model_name, **metrics})
+    return pd.DataFrame(rows).set_index("model")

src/features/tfidf.py

@@ -0,0 +1,50 @@
+import pickle
+from pathlib import Path
+
+import scipy.sparse as sp
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+
+def _get_tfidf_value(tfidf_config: dict, key: str):
+    kebab_key = key.replace("_", "-")
+    if key in tfidf_config:
+        return tfidf_config[key]
+    if kebab_key in tfidf_config:
+        return tfidf_config[kebab_key]
+    raise KeyError(key)
+
+
+def fit_tfidf(train_texts: list[str], config: dict) -> TfidfVectorizer:
+    tfidf_config = config["tfidf"]
+    vectorizer = TfidfVectorizer(
+        max_features=_get_tfidf_value(tfidf_config, "max_features"),
+        ngram_range=tuple(_get_tfidf_value(tfidf_config, "ngram_range")),
+        min_df=_get_tfidf_value(tfidf_config, "min_df"),
+    )
+    vectorizer.fit(train_texts)
+    return vectorizer
+
+
+def transform(vectorizer: TfidfVectorizer, texts: list[str]) -> sp.csr_matrix:
+    return vectorizer.transform(texts)
+
+
+def save_vectorizer(vectorizer: TfidfVectorizer, save_path: str = "artifacts/vectorizers/tfidf.pkl") -> None:
+    path = Path(save_path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "wb") as f:
+        pickle.dump(vectorizer, f)
+
+
+def load_vectorizer(
+    load_path: str = "artifacts/vectorizers/tfidf.pkl",
+) -> TfidfVectorizer:
+    path = Path(load_path)
+    if not path.exists():
+        raise FileNotFoundError(f"Vectorizer not found: {path}")
+    with open(path, "rb") as f:
+        return pickle.load(f)
+
+
+def get_feature_names(vectorizer: TfidfVectorizer) -> list[str]:
+    return vectorizer.get_feature_names_out().tolist()