Spaces:
Sleeping
Sleeping
Deploy ToxiGuard AI backend to Hugging Face Spaces
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .dockerignore +19 -0
- .gitattributes +1 -33
- .python-version +1 -0
- README.md +10 -7
- abuse_model.joblib +3 -0
- app.py +87 -0
- app/__init__.py +0 -0
- app/core/__init__.py +0 -0
- app/core/config.py +88 -0
- app/core/limiter.py +7 -0
- app/core/logger.py +29 -0
- app/main.py +149 -0
- app/routes/__init__.py +0 -0
- app/routes/auth.py +159 -0
- app/routes/explain.py +82 -0
- app/routes/feedback.py +233 -0
- app/routes/moderation.py +699 -0
- app/routes/monitoring.py +55 -0
- app/routes/realtime.py +135 -0
- app/routes/stream.py +245 -0
- app/services/__init__.py +2 -0
- app/services/calibration.py +211 -0
- app/services/drift_monitor.py +288 -0
- app/services/model_service.py +179 -0
- app/services/overrides.py +99 -0
- app/services/retrainer.py +125 -0
- auth_utils.py +39 -0
- data/LICENSE +21 -0
- data/README.md +32 -0
- data/calibration.json +51 -0
- data/overrides.json +6 -0
- data/sample_data.csv +0 -0
- database.py +41 -0
- label_encoder.joblib +3 -0
- ml/__init__.py +1 -0
- ml/evaluate.py +67 -0
- ml/explainability.py +242 -0
- ml/export_onnx.py +146 -0
- ml/inference.py +288 -0
- ml/models/toxiguard-deberta/added_tokens.json +3 -0
- ml/models/toxiguard-deberta/config.json +52 -0
- ml/models/toxiguard-deberta/model.onnx +3 -0
- ml/models/toxiguard-deberta/special_tokens_map.json +51 -0
- ml/models/toxiguard-deberta/spm.model +3 -0
- ml/models/toxiguard-deberta/tokenizer.json +0 -0
- ml/models/toxiguard-deberta/tokenizer_config.json +59 -0
- ml/train_transformer.py +326 -0
- models.py +68 -0
- requirements-prod.txt +42 -0
- requirements.txt +42 -0
.dockerignore
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Docker build ignore file
|
| 2 |
+
venv/
|
| 3 |
+
.venv/
|
| 4 |
+
ENV/
|
| 5 |
+
env/
|
| 6 |
+
__pycache__/
|
| 7 |
+
*.pyc
|
| 8 |
+
*.pyo
|
| 9 |
+
*.pyd
|
| 10 |
+
.pytest_cache/
|
| 11 |
+
tests/
|
| 12 |
+
.git/
|
| 13 |
+
.gitignore
|
| 14 |
+
.gitattributes
|
| 15 |
+
toxiguard.db
|
| 16 |
+
*.db-journal
|
| 17 |
+
*.log
|
| 18 |
+
.DS_Store
|
| 19 |
+
Thumbs.db
|
.gitattributes
CHANGED
|
@@ -1,35 +1,3 @@
|
|
| 1 |
-
*.
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 1 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
*.joblib filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
| 3 |
*.model filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.python-version
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
3.11.9
|
README.md
CHANGED
|
@@ -1,14 +1,17 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
colorFrom: red
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version: 6.31.0
|
| 8 |
-
python_version: '3.12'
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
| 11 |
-
license: mit
|
| 12 |
---
|
| 13 |
|
| 14 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: ToxiGuard AI Backend
|
| 3 |
+
emoji: 🛡
|
| 4 |
colorFrom: red
|
| 5 |
+
colorTo: indigo
|
| 6 |
sdk: gradio
|
|
|
|
|
|
|
| 7 |
app_file: app.py
|
| 8 |
pinned: false
|
|
|
|
| 9 |
---
|
| 10 |
|
| 11 |
+
# ToxiGuard AI - Content Moderation Platform
|
| 12 |
+
Production FastAPI moderation backend powered by DeBERTa-v3 with ONNX Runtime.
|
| 13 |
+
|
| 14 |
+
## API Documentation
|
| 15 |
+
- Swagger UI: /docs
|
| 16 |
+
- ReDoc: /redoc
|
| 17 |
+
- Health: /health
|
abuse_model.joblib
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4db4c1532070f7690a14053a20b1defb8836328d4bb3bf4db50ef7891f5b0f87
|
| 3 |
+
size 41081
|
app.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — Hugging Face Space Application Entry Point
|
| 3 |
+
Mounts FastAPI backend onto Gradio so the space serves both:
|
| 4 |
+
1. The interactive testing interface in the Hugging Face Space tab
|
| 5 |
+
2. All production REST API endpoints: /docs, /redoc, /health, /predict, /chat/moderate, etc.
|
| 6 |
+
"""
|
| 7 |
+
import os
|
| 8 |
+
import sys
|
| 9 |
+
|
| 10 |
+
# 1. Disable Gradio SSR Node proxy (prevents Errno 98 port 7860 collision)
|
| 11 |
+
os.environ["GRADIO_SSR_MODE"] = "false"
|
| 12 |
+
|
| 13 |
+
# 2. Hugging Face ZeroGPU initialization
|
| 14 |
+
try:
|
| 15 |
+
import spaces
|
| 16 |
+
except ImportError:
|
| 17 |
+
class spaces:
|
| 18 |
+
@staticmethod
|
| 19 |
+
def GPU(fn=None, **kwargs):
|
| 20 |
+
if fn is not None:
|
| 21 |
+
return fn
|
| 22 |
+
def decorator(f):
|
| 23 |
+
return f
|
| 24 |
+
return decorator
|
| 25 |
+
|
| 26 |
+
# Ensure backend root is in Python module search path
|
| 27 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 28 |
+
|
| 29 |
+
import gradio as gr
|
| 30 |
+
from app.main import app as fastapi_app
|
| 31 |
+
from app.services.model_service import model_service
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
# Literal @spaces.GPU decorator required for ZeroGPU static AST scanner
|
| 35 |
+
@spaces.GPU
|
| 36 |
+
def analyze_toxicity(text: str):
|
| 37 |
+
"""Interactive demo function for the Gradio UI."""
|
| 38 |
+
if not text or not text.strip():
|
| 39 |
+
return {"error": "Please enter text to analyze."}
|
| 40 |
+
try:
|
| 41 |
+
res = model_service.predict(text)
|
| 42 |
+
return {
|
| 43 |
+
"is_toxic": bool(res.get("is_toxic", False)),
|
| 44 |
+
"confidence": round(float(res.get("confidence", 0.0)), 4),
|
| 45 |
+
"primary_category": res.get("primary_category", "none"),
|
| 46 |
+
"severity": res.get("severity", "none"),
|
| 47 |
+
"model_used": res.get("model_used", "onnx-deberta"),
|
| 48 |
+
"scores": {k: round(float(v), 4) for k, v in res.get("scores", {}).items()}
|
| 49 |
+
}
|
| 50 |
+
except Exception as e:
|
| 51 |
+
return {"error": str(e)}
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
# ─── Gradio Interface for the Space page ─────────────────────────────────────
|
| 55 |
+
demo = gr.Interface(
|
| 56 |
+
fn=analyze_toxicity,
|
| 57 |
+
inputs=gr.Textbox(
|
| 58 |
+
lines=3,
|
| 59 |
+
placeholder="Enter text to analyze for toxicity, harassment, abuse...",
|
| 60 |
+
label="Input Content"
|
| 61 |
+
),
|
| 62 |
+
outputs=gr.JSON(label="Analysis Result"),
|
| 63 |
+
title="🛡️ ToxiGuard AI — Content Moderation Platform",
|
| 64 |
+
description=(
|
| 65 |
+
"**Real-time multi-label toxicity detection engine powered by DeBERTa-v3 & ONNX Runtime.**\n\n"
|
| 66 |
+
"### 🔗 Developer API Access:\n"
|
| 67 |
+
"- 📖 **Interactive Swagger UI**: [/docs](/docs)\n"
|
| 68 |
+
"- 📑 **ReDoc Documentation**: [/redoc](/redoc)\n"
|
| 69 |
+
"- 💓 **Health & Model Status**: [/health](/health)\n"
|
| 70 |
+
"- 🚀 **Prediction Endpoint**: `POST /predict` or `POST /predict/demo`"
|
| 71 |
+
),
|
| 72 |
+
examples=[
|
| 73 |
+
["Thank you so much for your assistance! Have a wonderful day."],
|
| 74 |
+
["You are completely brainless and deserve to be banned forever."],
|
| 75 |
+
["Stop talking before I find where you live and hurt you."],
|
| 76 |
+
["I hate this stupid policy, but I understand why it was made."]
|
| 77 |
+
],
|
| 78 |
+
cache_examples=False,
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
# Mount Gradio onto the existing FastAPI application
|
| 82 |
+
app = gr.mount_gradio_app(fastapi_app, demo, path="/")
|
| 83 |
+
|
| 84 |
+
if __name__ == "__main__":
|
| 85 |
+
import uvicorn
|
| 86 |
+
port = int(os.environ.get("PORT", 7860))
|
| 87 |
+
uvicorn.run(app, host="0.0.0.0", port=port)
|
app/__init__.py
ADDED
|
File without changes
|
app/core/__init__.py
ADDED
|
File without changes
|
app/core/config.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — Centralised Configuration
|
| 3 |
+
==========================================
|
| 4 |
+
All environment variables and runtime constants are declared here.
|
| 5 |
+
Import `settings` anywhere — never read os.environ directly in app code.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import os
|
| 9 |
+
from functools import lru_cache
|
| 10 |
+
# pyrefly: ignore [missing-import]
|
| 11 |
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class Settings(BaseSettings):
|
| 15 |
+
model_config = SettingsConfigDict(
|
| 16 |
+
env_file=os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env"),
|
| 17 |
+
env_file_encoding="utf-8",
|
| 18 |
+
extra="ignore",
|
| 19 |
+
protected_namespaces=("settings_",), # Avoid conflict with model_dir field
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
# ── LLM ─────────────────────────────────────────────────────────
|
| 23 |
+
openrouter_api_key: str = ""
|
| 24 |
+
openrouter_model: str = "google/gemma-4-31b-it:free"
|
| 25 |
+
openrouter_fallback_model: str = "qwen/qwen3-next-80b-a3b-instruct:free"
|
| 26 |
+
llm_timeout: float = 20.0
|
| 27 |
+
llm_cache_size: int = 128
|
| 28 |
+
llm_temperature: float = 0.4
|
| 29 |
+
llm_max_tokens: int = 2048
|
| 30 |
+
|
| 31 |
+
# Optional: Google Gemini API Key (secondary fallback if OpenRouter is unreachable)
|
| 32 |
+
gemini_model: str = "gemini-2.5-flash"
|
| 33 |
+
gemini_api_key: str = ""
|
| 34 |
+
|
| 35 |
+
# ── DATABASE ─────────────────────────────────────────────────────
|
| 36 |
+
database_url: str = "sqlite:///./toxiguard.db"
|
| 37 |
+
|
| 38 |
+
# ── AUTH ─────────────────────────────────────────────────────────
|
| 39 |
+
secret_key: str = "toxiguard-secret-change-in-production"
|
| 40 |
+
algorithm: str = "HS256"
|
| 41 |
+
access_token_expire_minutes: int = 60 * 24 # 24 hours
|
| 42 |
+
|
| 43 |
+
allowed_origins: str = (
|
| 44 |
+
"https://toxiai.vercel.app,https://toxiai-agent.vercel.app,http://localhost:5173,http://127.0.0.1:5173"
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
@property
|
| 48 |
+
def allowed_origins_list(self) -> list[str]:
|
| 49 |
+
return [o.strip() for o in self.allowed_origins.split(",") if o.strip()]
|
| 50 |
+
|
| 51 |
+
# ── MODEL ─────────────────────────────────────────────────────────
|
| 52 |
+
model_dir: str = os.path.join(
|
| 53 |
+
os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
|
| 54 |
+
"ml", "models", "toxiguard-deberta"
|
| 55 |
+
)
|
| 56 |
+
max_seq_length: int = 256
|
| 57 |
+
|
| 58 |
+
# ── ENSEMBLE THRESHOLDS ───────────────────────────────────────────
|
| 59 |
+
# DeBERTa is primary (50%), LLM is context layer (35%), Rules are fast-path (15%)
|
| 60 |
+
ensemble_threshold: float = 0.45
|
| 61 |
+
ml_trigger_threshold: float = 0.35 # Call LLM if ML >= this
|
| 62 |
+
demo_ml_threshold: float = 0.45
|
| 63 |
+
|
| 64 |
+
# Severity bands
|
| 65 |
+
severity_high: float = 0.85
|
| 66 |
+
severity_medium: float = 0.60
|
| 67 |
+
|
| 68 |
+
# ── PLAN LIMITS ───────────────────────────────────────────────────
|
| 69 |
+
plan_limit_free: int = 1_000
|
| 70 |
+
plan_limit_pro: int = 50_000
|
| 71 |
+
|
| 72 |
+
# ── APP META ─────────────────────────────────────────────────────
|
| 73 |
+
app_version: str = "2.0.0"
|
| 74 |
+
app_title: str = "ToxiGuard AI"
|
| 75 |
+
app_description: str = (
|
| 76 |
+
"Hybrid toxic content detection API combining rule-based filtering, "
|
| 77 |
+
"DeBERTa transformer (ONNX-optimised), and LLM-powered contextual analysis. "
|
| 78 |
+
"Built for real-time content moderation at scale."
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
@lru_cache(maxsize=1)
|
| 83 |
+
def get_settings() -> Settings:
|
| 84 |
+
return Settings()
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
# Module-level singleton
|
| 88 |
+
settings = get_settings()
|
app/core/limiter.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# backend/app/core/limiter.py
|
| 2 |
+
|
| 3 |
+
from slowapi import Limiter
|
| 4 |
+
from slowapi.util import get_remote_address
|
| 5 |
+
|
| 6 |
+
limiter = Limiter(key_func=get_remote_address)
|
| 7 |
+
limiter.enabled = True
|
app/core/logger.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — Structured Logger
|
| 3 |
+
==================================
|
| 4 |
+
Loguru-based logger with ISO timestamps and severity colours.
|
| 5 |
+
Import `logger` everywhere — never use bare print() in app code.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import sys
|
| 9 |
+
from loguru import logger
|
| 10 |
+
|
| 11 |
+
# Remove default handler and replace with a clean, structured format
|
| 12 |
+
logger.remove()
|
| 13 |
+
|
| 14 |
+
logger.add(
|
| 15 |
+
sys.stderr,
|
| 16 |
+
level="INFO",
|
| 17 |
+
format=(
|
| 18 |
+
"<green>{time:YYYY-MM-DD HH:mm:ss}</green> | "
|
| 19 |
+
"<level>{level: <8}</level> | "
|
| 20 |
+
"<cyan>{name}</cyan>:<cyan>{line}</cyan> | "
|
| 21 |
+
"<level>{message}</level>"
|
| 22 |
+
),
|
| 23 |
+
colorize=True,
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
# Optional: file sink for persistent logs
|
| 27 |
+
# logger.add("logs/toxiguard.log", rotation="10 MB", retention="7 days", level="DEBUG")
|
| 28 |
+
|
| 29 |
+
__all__ = ["logger"]
|
app/main.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from contextlib import asynccontextmanager
|
| 3 |
+
from fastapi import FastAPI, Request
|
| 4 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 5 |
+
from fastapi.responses import JSONResponse
|
| 6 |
+
from dotenv import load_dotenv
|
| 7 |
+
|
| 8 |
+
load_dotenv()
|
| 9 |
+
|
| 10 |
+
from slowapi.middleware import SlowAPIMiddleware
|
| 11 |
+
from slowapi.errors import RateLimitExceeded
|
| 12 |
+
|
| 13 |
+
from app.core.limiter import limiter
|
| 14 |
+
from app.core.logger import logger
|
| 15 |
+
from app.core.config import settings
|
| 16 |
+
from app.routes.moderation import router as moderation_router
|
| 17 |
+
from app.routes.auth import router as auth_router
|
| 18 |
+
from app.routes.realtime import router as realtime_router
|
| 19 |
+
from app.routes.explain import router as explain_router
|
| 20 |
+
from app.routes.monitoring import router as monitoring_router
|
| 21 |
+
from app.routes.feedback import router as feedback_router
|
| 22 |
+
from app.routes.stream import router as stream_router
|
| 23 |
+
from app.services.model_service import model_service
|
| 24 |
+
|
| 25 |
+
from database import engine, Base
|
| 26 |
+
Base.metadata.create_all(bind=engine)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 30 |
+
# LIFESPAN (startup / shutdown)
|
| 31 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 32 |
+
|
| 33 |
+
@asynccontextmanager
|
| 34 |
+
async def lifespan(app: FastAPI):
|
| 35 |
+
# Startup
|
| 36 |
+
logger.info("=" * 50)
|
| 37 |
+
logger.info(f"ToxiGuard AI v{settings.app_version} starting up")
|
| 38 |
+
logger.info(f"Model status: {model_service.status}")
|
| 39 |
+
logger.info(f"CORS origins: {settings.allowed_origins_list}")
|
| 40 |
+
|
| 41 |
+
# Pre-warm XAI Explainer to avoid lag on first explain query
|
| 42 |
+
try:
|
| 43 |
+
from ml.explainability import get_explainer
|
| 44 |
+
explainer = get_explainer(settings.model_dir)
|
| 45 |
+
if explainer.is_ready:
|
| 46 |
+
logger.info("[Lifespan] XAI Explainer pre-warmed successfully")
|
| 47 |
+
else:
|
| 48 |
+
logger.warning("[Lifespan] XAI Explainer pre-warm skipped: Explainer not ready (model files missing)")
|
| 49 |
+
except Exception as e:
|
| 50 |
+
logger.warning(f"[Lifespan] Failed to pre-warm XAI explainer: {e}")
|
| 51 |
+
|
| 52 |
+
logger.info("=" * 50)
|
| 53 |
+
yield
|
| 54 |
+
# Shutdown
|
| 55 |
+
logger.info("ToxiGuard AI shutting down")
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 59 |
+
# APP INIT
|
| 60 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 61 |
+
|
| 62 |
+
app = FastAPI(
|
| 63 |
+
title=settings.app_title,
|
| 64 |
+
description=settings.app_description,
|
| 65 |
+
version=settings.app_version,
|
| 66 |
+
docs_url="/docs",
|
| 67 |
+
redoc_url="/redoc",
|
| 68 |
+
lifespan=lifespan,
|
| 69 |
+
)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 73 |
+
# RATE LIMITING
|
| 74 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 75 |
+
|
| 76 |
+
app.state.limiter = limiter
|
| 77 |
+
app.add_middleware(SlowAPIMiddleware)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
@app.exception_handler(RateLimitExceeded)
|
| 81 |
+
def rate_limit_handler(request: Request, exc: RateLimitExceeded):
|
| 82 |
+
return JSONResponse(
|
| 83 |
+
status_code=429,
|
| 84 |
+
content={"detail": "Rate limit exceeded. Please try again shortly."},
|
| 85 |
+
)
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 89 |
+
# CORS — reads from ALLOWED_ORIGINS env var (not hardcoded ["*"])
|
| 90 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 91 |
+
|
| 92 |
+
app.add_middleware(
|
| 93 |
+
CORSMiddleware,
|
| 94 |
+
allow_origins=settings.allowed_origins_list,
|
| 95 |
+
allow_credentials=False,
|
| 96 |
+
allow_methods=["GET", "POST", "OPTIONS"],
|
| 97 |
+
allow_headers=["Content-Type", "X-API-Key", "Authorization"],
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 102 |
+
# ROUTES
|
| 103 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 104 |
+
|
| 105 |
+
app.include_router(auth_router)
|
| 106 |
+
app.include_router(moderation_router)
|
| 107 |
+
app.include_router(realtime_router)
|
| 108 |
+
app.include_router(explain_router)
|
| 109 |
+
app.include_router(monitoring_router)
|
| 110 |
+
app.include_router(feedback_router)
|
| 111 |
+
app.include_router(stream_router)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 115 |
+
# HEALTH CHECK
|
| 116 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 117 |
+
|
| 118 |
+
@app.get("/", tags=["Health"])
|
| 119 |
+
def root():
|
| 120 |
+
return {
|
| 121 |
+
"status": "ToxiGuard AI running",
|
| 122 |
+
"version": settings.app_version,
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
@app.get("/health", tags=["Health"])
|
| 127 |
+
def health():
|
| 128 |
+
"""Detailed health check with model and endpoint status."""
|
| 129 |
+
return {
|
| 130 |
+
"status": "healthy",
|
| 131 |
+
"version": settings.app_version,
|
| 132 |
+
"model": model_service.status,
|
| 133 |
+
"endpoints": {
|
| 134 |
+
"predict_full": "POST /predict",
|
| 135 |
+
"predict_ml": "POST /predict/ml",
|
| 136 |
+
"predict_demo": "POST /predict/demo",
|
| 137 |
+
"chat_moderate": "POST /chat/moderate",
|
| 138 |
+
"analyze_file": "POST /analyze-file",
|
| 139 |
+
"explain": "POST /explain [XAI]",
|
| 140 |
+
"monitoring_stats": "GET /monitoring/stats [Drift]",
|
| 141 |
+
"monitoring_drift": "GET /monitoring/drift [Drift]",
|
| 142 |
+
"feedback": "POST /feedback [Active Learning]",
|
| 143 |
+
"feedback_stats": "GET /feedback/stats",
|
| 144 |
+
"feedback_queue": "GET /feedback/queue",
|
| 145 |
+
"stream": "WS /ws/stream [Real-time]",
|
| 146 |
+
"auth": "POST /auth/signup, POST /auth/login",
|
| 147 |
+
"docs": "/docs",
|
| 148 |
+
},
|
| 149 |
+
}
|
app/routes/__init__.py
ADDED
|
File without changes
|
app/routes/auth.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, Depends, HTTPException, Security
|
| 2 |
+
from fastapi.security import APIKeyHeader
|
| 3 |
+
from sqlalchemy.orm import Session
|
| 4 |
+
from pydantic import BaseModel, EmailStr, field_validator
|
| 5 |
+
import secrets
|
| 6 |
+
|
| 7 |
+
from database import get_db
|
| 8 |
+
from models import User
|
| 9 |
+
from auth_utils import create_access_token, generate_api_key
|
| 10 |
+
from passlib.context import CryptContext
|
| 11 |
+
|
| 12 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 13 |
+
# ROUTER
|
| 14 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 15 |
+
|
| 16 |
+
router = APIRouter(prefix="/auth", tags=["Auth"])
|
| 17 |
+
|
| 18 |
+
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
|
| 19 |
+
api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 23 |
+
# SCHEMAS
|
| 24 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 25 |
+
|
| 26 |
+
class SignupRequest(BaseModel):
|
| 27 |
+
email: EmailStr
|
| 28 |
+
password: str
|
| 29 |
+
|
| 30 |
+
@field_validator("password")
|
| 31 |
+
@classmethod
|
| 32 |
+
def password_min_length(cls, v: str) -> str:
|
| 33 |
+
if len(v) < 6:
|
| 34 |
+
raise ValueError("Password must be at least 6 characters")
|
| 35 |
+
return v
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class LoginRequest(BaseModel):
|
| 39 |
+
email: str
|
| 40 |
+
password: str
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 44 |
+
# HELPERS
|
| 45 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 46 |
+
|
| 47 |
+
def hash_password(password: str) -> str:
|
| 48 |
+
return pwd_context.hash(password)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def verify_password(plain: str, hashed: str) -> bool:
|
| 52 |
+
return pwd_context.verify(plain, hashed)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 56 |
+
# SIGNUP
|
| 57 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 58 |
+
|
| 59 |
+
@router.post("/signup", status_code=201)
|
| 60 |
+
def signup(req: SignupRequest, db: Session = Depends(get_db)):
|
| 61 |
+
"""Register a new user and return their API key."""
|
| 62 |
+
existing = db.query(User).filter(User.email == req.email).first()
|
| 63 |
+
if existing:
|
| 64 |
+
raise HTTPException(status_code=400, detail="Email already registered")
|
| 65 |
+
|
| 66 |
+
user = User(
|
| 67 |
+
email=req.email,
|
| 68 |
+
password_hash=hash_password(req.password),
|
| 69 |
+
api_key=generate_api_key(),
|
| 70 |
+
plan="free",
|
| 71 |
+
usage_count=0,
|
| 72 |
+
)
|
| 73 |
+
db.add(user)
|
| 74 |
+
db.commit()
|
| 75 |
+
db.refresh(user)
|
| 76 |
+
|
| 77 |
+
return {
|
| 78 |
+
"message": "Account created successfully",
|
| 79 |
+
"api_key": user.api_key,
|
| 80 |
+
"plan": user.plan,
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 85 |
+
# LOGIN
|
| 86 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 87 |
+
|
| 88 |
+
@router.post("/login")
|
| 89 |
+
def login(req: LoginRequest, db: Session = Depends(get_db)):
|
| 90 |
+
"""Authenticate and return a JWT token + API key."""
|
| 91 |
+
user = db.query(User).filter(User.email == req.email).first()
|
| 92 |
+
if not user or not verify_password(req.password, user.password_hash):
|
| 93 |
+
raise HTTPException(status_code=401, detail="Invalid credentials")
|
| 94 |
+
|
| 95 |
+
token = create_access_token({"sub": user.email})
|
| 96 |
+
|
| 97 |
+
return {
|
| 98 |
+
"token": token,
|
| 99 |
+
"api_key": user.api_key,
|
| 100 |
+
"plan": user.plan,
|
| 101 |
+
"usage_count": user.usage_count,
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 106 |
+
# API KEY AUTH DEPENDENCY
|
| 107 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 108 |
+
|
| 109 |
+
def get_api_user(
|
| 110 |
+
x_api_key: str = Security(api_key_header),
|
| 111 |
+
db: Session = Depends(get_db),
|
| 112 |
+
) -> User:
|
| 113 |
+
if not x_api_key:
|
| 114 |
+
raise HTTPException(status_code=401, detail="API key missing")
|
| 115 |
+
|
| 116 |
+
user = db.query(User).filter(User.api_key == x_api_key).first()
|
| 117 |
+
if not user:
|
| 118 |
+
raise HTTPException(status_code=403, detail="Invalid API key")
|
| 119 |
+
|
| 120 |
+
return user
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 124 |
+
# PROFILE
|
| 125 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 126 |
+
|
| 127 |
+
@router.get("/me")
|
| 128 |
+
def get_me(user: User = Depends(get_api_user)):
|
| 129 |
+
"""Return the authenticated user's profile and usage stats."""
|
| 130 |
+
return {
|
| 131 |
+
"email": user.email,
|
| 132 |
+
"plan": user.plan,
|
| 133 |
+
"usage_count": user.usage_count,
|
| 134 |
+
"api_key": user.api_key,
|
| 135 |
+
"last_used": user.last_used.isoformat() if user.last_used else None,
|
| 136 |
+
}
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 140 |
+
# ROTATE API KEY
|
| 141 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 142 |
+
|
| 143 |
+
@router.post("/reset-key")
|
| 144 |
+
def reset_api_key(
|
| 145 |
+
user: User = Depends(get_api_user),
|
| 146 |
+
db: Session = Depends(get_db),
|
| 147 |
+
):
|
| 148 |
+
"""
|
| 149 |
+
Generate and store a new API key for the authenticated user.
|
| 150 |
+
The old key is immediately invalidated.
|
| 151 |
+
"""
|
| 152 |
+
user.api_key = generate_api_key()
|
| 153 |
+
db.commit()
|
| 154 |
+
db.refresh(user)
|
| 155 |
+
|
| 156 |
+
return {
|
| 157 |
+
"message": "API key rotated successfully",
|
| 158 |
+
"api_key": user.api_key,
|
| 159 |
+
}
|
app/routes/explain.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — XAI Explain Endpoint
|
| 3 |
+
======================================
|
| 4 |
+
POST /explain — Token-level attention attribution for toxicity decisions.
|
| 5 |
+
|
| 6 |
+
Returns per-token importance scores extracted from DeBERTa's last attention layer.
|
| 7 |
+
Used to answer: "Why did the model flag this text as toxic?"
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from fastapi import APIRouter, Depends, Request, Body
|
| 11 |
+
from pydantic import BaseModel
|
| 12 |
+
from sqlalchemy.orm import Session
|
| 13 |
+
|
| 14 |
+
from app.routes.auth import get_api_user
|
| 15 |
+
from app.core.limiter import limiter
|
| 16 |
+
from app.core.config import settings
|
| 17 |
+
from app.core.logger import logger
|
| 18 |
+
from ml.explainability import get_explainer
|
| 19 |
+
from models import User
|
| 20 |
+
from database import get_db
|
| 21 |
+
|
| 22 |
+
router = APIRouter()
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class ExplainRequest(BaseModel):
|
| 26 |
+
text: str
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
@router.post("/explain", tags=["XAI"])
|
| 30 |
+
@limiter.limit("20/minute")
|
| 31 |
+
def explain_prediction(
|
| 32 |
+
request: Request,
|
| 33 |
+
req: ExplainRequest = Body(...),
|
| 34 |
+
user: User = Depends(get_api_user),
|
| 35 |
+
db: Session = Depends(get_db),
|
| 36 |
+
):
|
| 37 |
+
"""
|
| 38 |
+
Token-level XAI attribution via DeBERTa attention weights.
|
| 39 |
+
|
| 40 |
+
Returns each word's contribution score (0.0 = no influence, 1.0 = highest influence).
|
| 41 |
+
Use this to build token heatmaps in the frontend.
|
| 42 |
+
|
| 43 |
+
Response:
|
| 44 |
+
{
|
| 45 |
+
"tokens": [{"token": str, "score": float, "is_special": bool}],
|
| 46 |
+
"top_tokens": [{"token": str, "score": float}],
|
| 47 |
+
"method": "attention_last_layer_mean",
|
| 48 |
+
"num_tokens": int,
|
| 49 |
+
"status": "ok" | "unavailable"
|
| 50 |
+
}
|
| 51 |
+
"""
|
| 52 |
+
text = req.text.strip()
|
| 53 |
+
if not text:
|
| 54 |
+
return {"error": "Empty text", "status": "error"}
|
| 55 |
+
|
| 56 |
+
logger.info(f"[/explain] user={user.email} | len={len(text)}")
|
| 57 |
+
|
| 58 |
+
explainer = get_explainer(settings.model_dir)
|
| 59 |
+
|
| 60 |
+
if not explainer.is_ready:
|
| 61 |
+
return {
|
| 62 |
+
"tokens": [],
|
| 63 |
+
"top_tokens": [],
|
| 64 |
+
"method": "unavailable",
|
| 65 |
+
"num_tokens": 0,
|
| 66 |
+
"status": "unavailable",
|
| 67 |
+
"message": "XAI explainer not ready — check backend logs",
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
result = explainer.explain(text)
|
| 71 |
+
|
| 72 |
+
if result is None:
|
| 73 |
+
return {
|
| 74 |
+
"tokens": [],
|
| 75 |
+
"top_tokens": [],
|
| 76 |
+
"method": "error",
|
| 77 |
+
"num_tokens": 0,
|
| 78 |
+
"status": "error",
|
| 79 |
+
"message": "Attribution failed",
|
| 80 |
+
}
|
| 81 |
+
|
| 82 |
+
return {**result, "status": "ok"}
|
app/routes/feedback.py
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — Active Learning Feedback Loop
|
| 3 |
+
=============================================
|
| 4 |
+
Endpoints for collecting user corrections to model predictions.
|
| 5 |
+
|
| 6 |
+
POST /feedback — Submit a false positive or false negative correction
|
| 7 |
+
GET /feedback/queue — View unreviewed samples (for retraining queue)
|
| 8 |
+
GET /feedback/stats — Aggregate feedback counts and accuracy signals
|
| 9 |
+
|
| 10 |
+
Design:
|
| 11 |
+
- Every flagged prediction is stored with original text + user's correction
|
| 12 |
+
- `feedback_type` is computed automatically: "false_positive" or "false_negative"
|
| 13 |
+
- `reviewed` flag allows admins to mark samples as processed
|
| 14 |
+
- This is the standard Active Learning data pipeline used in production ML systems
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
from datetime import datetime
|
| 18 |
+
from fastapi import APIRouter, Depends, HTTPException, Request, Body
|
| 19 |
+
from pydantic import BaseModel
|
| 20 |
+
from sqlalchemy.orm import Session
|
| 21 |
+
from sqlalchemy import func
|
| 22 |
+
from typing import Optional
|
| 23 |
+
|
| 24 |
+
from app.routes.auth import get_api_user
|
| 25 |
+
from app.core.limiter import limiter
|
| 26 |
+
from app.core.logger import logger
|
| 27 |
+
from models import User, FeedbackSample
|
| 28 |
+
from database import get_db
|
| 29 |
+
from app.services.calibration import calibration_service
|
| 30 |
+
|
| 31 |
+
router = APIRouter()
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 35 |
+
# SCHEMA
|
| 36 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 37 |
+
|
| 38 |
+
class FeedbackRequest(BaseModel):
|
| 39 |
+
input_text: str
|
| 40 |
+
predicted_toxic: bool # What the model said
|
| 41 |
+
correct_label: bool # What the user says is correct
|
| 42 |
+
confidence_at_time: Optional[float] = None
|
| 43 |
+
notes: Optional[str] = None
|
| 44 |
+
abusive_words: Optional[str] = None
|
| 45 |
+
explanation: Optional[str] = None
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 49 |
+
# ENDPOINTS
|
| 50 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 51 |
+
|
| 52 |
+
@router.post("/feedback", tags=["Active Learning"])
|
| 53 |
+
@limiter.limit("10/minute")
|
| 54 |
+
def submit_feedback(
|
| 55 |
+
request: Request,
|
| 56 |
+
req: FeedbackRequest = Body(...),
|
| 57 |
+
user: User = Depends(get_api_user),
|
| 58 |
+
db: Session = Depends(get_db),
|
| 59 |
+
):
|
| 60 |
+
"""
|
| 61 |
+
Submit a prediction correction for active learning.
|
| 62 |
+
|
| 63 |
+
- If model said TOXIC but it's not → false_positive
|
| 64 |
+
- If model said SAFE but it is toxic → false_negative
|
| 65 |
+
"""
|
| 66 |
+
text = req.input_text.strip()
|
| 67 |
+
if not text:
|
| 68 |
+
raise HTTPException(status_code=400, detail="input_text cannot be empty")
|
| 69 |
+
|
| 70 |
+
if len(text) > 2000:
|
| 71 |
+
raise HTTPException(status_code=400, detail="Text too long (max 2000 chars)")
|
| 72 |
+
|
| 73 |
+
# Auto-determine feedback type
|
| 74 |
+
if req.predicted_toxic and not req.correct_label:
|
| 75 |
+
feedback_type = "false_positive"
|
| 76 |
+
elif not req.predicted_toxic and req.correct_label:
|
| 77 |
+
feedback_type = "false_negative"
|
| 78 |
+
else:
|
| 79 |
+
# Model was correct — user is confirming; still log for confidence calibration
|
| 80 |
+
feedback_type = "confirmed_correct"
|
| 81 |
+
|
| 82 |
+
sample = FeedbackSample(
|
| 83 |
+
user_id=user.id,
|
| 84 |
+
input_text=text,
|
| 85 |
+
predicted_toxic=req.predicted_toxic,
|
| 86 |
+
correct_label=req.correct_label,
|
| 87 |
+
feedback_type=feedback_type,
|
| 88 |
+
confidence_at_time=req.confidence_at_time,
|
| 89 |
+
notes=req.notes,
|
| 90 |
+
abusive_words=req.abusive_words,
|
| 91 |
+
explanation=req.explanation,
|
| 92 |
+
reviewed=False,
|
| 93 |
+
)
|
| 94 |
+
db.add(sample)
|
| 95 |
+
db.commit()
|
| 96 |
+
db.refresh(sample)
|
| 97 |
+
|
| 98 |
+
# Apply adaptive threshold calibration in real-time
|
| 99 |
+
calibration_result = calibration_service.apply_feedback(
|
| 100 |
+
feedback_type,
|
| 101 |
+
req.confidence_at_time
|
| 102 |
+
)
|
| 103 |
+
|
| 104 |
+
# Save to dynamic hot-patches blocklist/allowlist for instant update
|
| 105 |
+
from app.services.overrides import override_service
|
| 106 |
+
override_service.add_override(feedback_type, text)
|
| 107 |
+
|
| 108 |
+
logger.info(
|
| 109 |
+
f"[/feedback] user={user.email} | type={feedback_type} | len={len(text)} | threshold adjusted: {calibration_result.get('old_threshold')} -> {calibration_result.get('new_threshold')}"
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
return {
|
| 113 |
+
"id": sample.id,
|
| 114 |
+
"feedback_type": feedback_type,
|
| 115 |
+
"message": "Feedback recorded. Thank you for improving ToxiGuard AI!",
|
| 116 |
+
"queued_for_review": True,
|
| 117 |
+
"calibration": calibration_result,
|
| 118 |
+
}
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
@router.get("/feedback/stats", tags=["Active Learning"])
|
| 122 |
+
@limiter.limit("20/minute")
|
| 123 |
+
def get_feedback_stats(
|
| 124 |
+
request: Request,
|
| 125 |
+
user: User = Depends(get_api_user),
|
| 126 |
+
db: Session = Depends(get_db),
|
| 127 |
+
):
|
| 128 |
+
"""
|
| 129 |
+
Aggregate feedback statistics for the active learning dashboard.
|
| 130 |
+
|
| 131 |
+
Returns counts by type, overall accuracy signal, and review status.
|
| 132 |
+
"""
|
| 133 |
+
total = db.query(func.count(FeedbackSample.id)).scalar() or 0
|
| 134 |
+
fp = db.query(func.count(FeedbackSample.id)).filter(
|
| 135 |
+
FeedbackSample.feedback_type == "false_positive"
|
| 136 |
+
).scalar() or 0
|
| 137 |
+
fn = db.query(func.count(FeedbackSample.id)).filter(
|
| 138 |
+
FeedbackSample.feedback_type == "false_negative"
|
| 139 |
+
).scalar() or 0
|
| 140 |
+
confirmed = db.query(func.count(FeedbackSample.id)).filter(
|
| 141 |
+
FeedbackSample.feedback_type == "confirmed_correct"
|
| 142 |
+
).scalar() or 0
|
| 143 |
+
unreviewed = db.query(func.count(FeedbackSample.id)).filter(
|
| 144 |
+
FeedbackSample.reviewed == False
|
| 145 |
+
).scalar() or 0
|
| 146 |
+
|
| 147 |
+
# Error rate signal: (FP + FN) / total corrections
|
| 148 |
+
correction_total = fp + fn
|
| 149 |
+
error_rate = round(correction_total / total, 4) if total > 0 else 0.0
|
| 150 |
+
|
| 151 |
+
# Precision signal: what fraction of flagged-as-toxic were actually toxic
|
| 152 |
+
# FP = flagged toxic, wasn't; confirmed = flagged toxic, was toxic
|
| 153 |
+
# (approximate from feedback alone, not full prediction volume)
|
| 154 |
+
precision_approx = None
|
| 155 |
+
if (fp + confirmed) > 0:
|
| 156 |
+
precision_approx = round(confirmed / (fp + confirmed), 4)
|
| 157 |
+
|
| 158 |
+
return {
|
| 159 |
+
"total_feedback": total,
|
| 160 |
+
"false_positives": fp,
|
| 161 |
+
"false_negatives": fn,
|
| 162 |
+
"confirmed_correct": confirmed,
|
| 163 |
+
"unreviewed_count": unreviewed,
|
| 164 |
+
"error_rate_signal": error_rate,
|
| 165 |
+
"precision_approx": precision_approx,
|
| 166 |
+
"retraining_recommended": correction_total >= 50,
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
@router.get("/feedback/queue", tags=["Active Learning"])
|
| 171 |
+
@limiter.limit("10/minute")
|
| 172 |
+
def get_feedback_queue(
|
| 173 |
+
request: Request,
|
| 174 |
+
limit: int = 20,
|
| 175 |
+
feedback_type: Optional[str] = None,
|
| 176 |
+
user: User = Depends(get_api_user),
|
| 177 |
+
db: Session = Depends(get_db),
|
| 178 |
+
):
|
| 179 |
+
"""
|
| 180 |
+
View the active learning queue — unreviewed correction samples.
|
| 181 |
+
|
| 182 |
+
Use `feedback_type` to filter: false_positive | false_negative | confirmed_correct
|
| 183 |
+
"""
|
| 184 |
+
limit = min(limit, 100)
|
| 185 |
+
|
| 186 |
+
query = db.query(FeedbackSample).filter(FeedbackSample.reviewed == False)
|
| 187 |
+
|
| 188 |
+
if feedback_type:
|
| 189 |
+
query = query.filter(FeedbackSample.feedback_type == feedback_type)
|
| 190 |
+
|
| 191 |
+
samples = query.order_by(FeedbackSample.created_at.desc()).limit(limit).all()
|
| 192 |
+
|
| 193 |
+
return {
|
| 194 |
+
"count": len(samples),
|
| 195 |
+
"samples": [
|
| 196 |
+
{
|
| 197 |
+
"id": s.id,
|
| 198 |
+
"input_text": s.input_text[:200] + ("..." if len(s.input_text) > 200 else ""),
|
| 199 |
+
"predicted_toxic": s.predicted_toxic,
|
| 200 |
+
"correct_label": s.correct_label,
|
| 201 |
+
"feedback_type": s.feedback_type,
|
| 202 |
+
"confidence_at_time": s.confidence_at_time,
|
| 203 |
+
"notes": s.notes,
|
| 204 |
+
"created_at": s.created_at.isoformat(),
|
| 205 |
+
}
|
| 206 |
+
for s in samples
|
| 207 |
+
],
|
| 208 |
+
}
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
@router.post("/feedback/retrain", tags=["Active Learning"])
|
| 212 |
+
@limiter.limit("5/minute")
|
| 213 |
+
def trigger_retraining(
|
| 214 |
+
request: Request,
|
| 215 |
+
user: User = Depends(get_api_user),
|
| 216 |
+
db: Session = Depends(get_db),
|
| 217 |
+
):
|
| 218 |
+
"""
|
| 219 |
+
Trigger manual model retraining on feedback queue.
|
| 220 |
+
Appends corrections to data/sample_data.csv, retrains TF-IDF model,
|
| 221 |
+
saves new joblib checkpoint, and hot-reloads prediction service.
|
| 222 |
+
"""
|
| 223 |
+
logger.info(f"[/feedback/retrain] triggered by admin/user: {user.email}")
|
| 224 |
+
try:
|
| 225 |
+
from app.services.retrainer import retrain_model
|
| 226 |
+
result = retrain_model(db)
|
| 227 |
+
return result
|
| 228 |
+
except Exception as exc:
|
| 229 |
+
logger.error(f"[/feedback/retrain] Retraining failed: {exc}")
|
| 230 |
+
raise HTTPException(
|
| 231 |
+
status_code=500,
|
| 232 |
+
detail=f"Retraining failed: {str(exc)}"
|
| 233 |
+
)
|
app/routes/moderation.py
ADDED
|
@@ -0,0 +1,699 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — Moderation Routes
|
| 3 |
+
==================================
|
| 4 |
+
Three prediction endpoints:
|
| 5 |
+
|
| 6 |
+
POST /predict/demo — Public demo (IP rate-limited, no auth, ML only)
|
| 7 |
+
POST /predict/ml — Extension fast-path (auth, ML only, no LLM)
|
| 8 |
+
POST /predict — Full pipeline (auth, rules + DeBERTa + LLM)
|
| 9 |
+
POST /analyze-file — Batch CSV/TXT analysis (auth, ensemble scoring)
|
| 10 |
+
|
| 11 |
+
Ensemble weighting (transformer model):
|
| 12 |
+
DeBERTa 50% — primary, SOTA accuracy
|
| 13 |
+
LLM 35% — contextual nuance (only called when needed)
|
| 14 |
+
Rules 15% — fast-path for obvious slurs
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
from collections import Counter
|
| 19 |
+
|
| 20 |
+
import io
|
| 21 |
+
import pandas as pd
|
| 22 |
+
from datetime import datetime
|
| 23 |
+
|
| 24 |
+
from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile, File, Body
|
| 25 |
+
from sqlalchemy.orm import Session
|
| 26 |
+
from pydantic import BaseModel
|
| 27 |
+
|
| 28 |
+
from app.services.model_service import model_service
|
| 29 |
+
from app.routes.auth import get_api_user
|
| 30 |
+
from app.core.limiter import limiter
|
| 31 |
+
from app.core.logger import logger
|
| 32 |
+
from app.core.config import settings
|
| 33 |
+
|
| 34 |
+
from utils.preprocessing import preprocess_for_rules, preprocess_for_model
|
| 35 |
+
from utils.abuse_words import detect_abusive_tokens, get_abuse_severity, suggestions as abuse_suggestions
|
| 36 |
+
from utils.sentiment import analyze_sentiment
|
| 37 |
+
from utils.llm_guard import analyze_toxicity_llm
|
| 38 |
+
|
| 39 |
+
from models import User
|
| 40 |
+
from database import get_db
|
| 41 |
+
from app.services.drift_monitor import drift_monitor
|
| 42 |
+
from app.services.calibration import calibration_service
|
| 43 |
+
|
| 44 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 45 |
+
# ROUTER
|
| 46 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 47 |
+
|
| 48 |
+
router = APIRouter()
|
| 49 |
+
|
| 50 |
+
PLAN_LIMITS = {
|
| 51 |
+
"free": settings.plan_limit_free,
|
| 52 |
+
"pro": settings.plan_limit_pro,
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
TOXIC_ML_LABELS = frozenset({
|
| 56 |
+
"abusive", "toxic", "severe_toxic",
|
| 57 |
+
"obscene", "threat", "insult", "identity_hate",
|
| 58 |
+
})
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 62 |
+
# SCHEMAS
|
| 63 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 64 |
+
|
| 65 |
+
class TextRequest(BaseModel):
|
| 66 |
+
text: str
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 70 |
+
# SHARED HELPERS
|
| 71 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 72 |
+
|
| 73 |
+
def _generate_suggestions(words: list[str]) -> dict[str, str]:
|
| 74 |
+
result = {}
|
| 75 |
+
for w in words:
|
| 76 |
+
key = w.lower()
|
| 77 |
+
if key in abuse_suggestions:
|
| 78 |
+
result[w] = abuse_suggestions[key]
|
| 79 |
+
else:
|
| 80 |
+
# Word-specific fallback — more useful than a generic message
|
| 81 |
+
result[w] = f"Consider replacing '{w}' with a more neutral or constructive expression."
|
| 82 |
+
return result
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _build_response(payload: dict) -> dict:
|
| 86 |
+
words = payload.get("abusive_words", [])
|
| 87 |
+
payload["word_frequency"] = dict(Counter(words))
|
| 88 |
+
payload["suggestions"] = _generate_suggestions(words)
|
| 89 |
+
return payload
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _adjust_sentiment(sentiment: dict | None, toxic: bool) -> dict | None:
|
| 93 |
+
if toxic and sentiment:
|
| 94 |
+
sentiment["label"] = "negative"
|
| 95 |
+
return sentiment
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def _check_plan_limit(user: User, db: Session) -> None:
|
| 99 |
+
if user.usage_count >= PLAN_LIMITS.get(user.plan, settings.plan_limit_free):
|
| 100 |
+
raise HTTPException(status_code=403, detail="Usage limit exceeded")
|
| 101 |
+
user.usage_count += 1
|
| 102 |
+
user.last_used = datetime.utcnow()
|
| 103 |
+
db.commit()
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 107 |
+
# ML RESULT NORMALISATION
|
| 108 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 109 |
+
|
| 110 |
+
def normalize_ml_result(ml_result: dict | None, threshold: float | None = None) -> dict:
|
| 111 |
+
"""Normalise raw ML output into a standard toxicity verdict."""
|
| 112 |
+
if threshold is None:
|
| 113 |
+
threshold = calibration_service.get_threshold()
|
| 114 |
+
if not ml_result:
|
| 115 |
+
return {
|
| 116 |
+
"toxic": False,
|
| 117 |
+
"confidence": 0.0,
|
| 118 |
+
"severity": "low",
|
| 119 |
+
"category": "safe",
|
| 120 |
+
"detected_categories": [],
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
labels: dict = ml_result.get("labels") or {}
|
| 124 |
+
label: str = str(ml_result.get("label") or "").lower()
|
| 125 |
+
detected_categories: list[str] = [
|
| 126 |
+
str(c).lower() for c in ml_result.get("detected_categories", [])
|
| 127 |
+
]
|
| 128 |
+
|
| 129 |
+
toxic_scores = [
|
| 130 |
+
float(score)
|
| 131 |
+
for cat, score in labels.items()
|
| 132 |
+
if str(cat).lower() in TOXIC_ML_LABELS
|
| 133 |
+
]
|
| 134 |
+
confidence = max(
|
| 135 |
+
[float(ml_result.get("toxicity_probability", 0.0))] + toxic_scores
|
| 136 |
+
)
|
| 137 |
+
|
| 138 |
+
toxic = (
|
| 139 |
+
bool(ml_result.get("toxic"))
|
| 140 |
+
or label in TOXIC_ML_LABELS
|
| 141 |
+
or any(c in TOXIC_ML_LABELS for c in detected_categories)
|
| 142 |
+
or confidence >= threshold
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
if confidence >= settings.severity_high:
|
| 146 |
+
severity = "high"
|
| 147 |
+
elif confidence >= settings.severity_medium:
|
| 148 |
+
severity = "medium"
|
| 149 |
+
else:
|
| 150 |
+
severity = "low"
|
| 151 |
+
|
| 152 |
+
category = "safe"
|
| 153 |
+
if toxic:
|
| 154 |
+
if detected_categories:
|
| 155 |
+
category = detected_categories[0]
|
| 156 |
+
elif label and label not in ("clean", "safe") and label in TOXIC_ML_LABELS:
|
| 157 |
+
category = label
|
| 158 |
+
else:
|
| 159 |
+
category = "toxic"
|
| 160 |
+
|
| 161 |
+
return {
|
| 162 |
+
"toxic": toxic,
|
| 163 |
+
"confidence": round(confidence, 3),
|
| 164 |
+
"severity": severity,
|
| 165 |
+
"category": category,
|
| 166 |
+
"detected_categories": detected_categories,
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 171 |
+
# WEIGHTED ENSEMBLE SCORER
|
| 172 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 173 |
+
|
| 174 |
+
def compute_ensemble_score(
|
| 175 |
+
rules_triggered: bool,
|
| 176 |
+
rules_severity: str,
|
| 177 |
+
ml_probability: float,
|
| 178 |
+
llm_toxic: bool,
|
| 179 |
+
llm_confidence: float,
|
| 180 |
+
model_type: str,
|
| 181 |
+
) -> tuple[bool, float]:
|
| 182 |
+
"""
|
| 183 |
+
Three-layer weighted ensemble scoring.
|
| 184 |
+
|
| 185 |
+
Weights (transformer path):
|
| 186 |
+
DeBERTa 50% — primary model, highest accuracy signal
|
| 187 |
+
LLM 35% — contextual understanding, sarcasm detection
|
| 188 |
+
Rules 15% — deterministic, fast, explainable
|
| 189 |
+
|
| 190 |
+
When LLM was not called (cost/rate-limit), the 35% is redistributed:
|
| 191 |
+
DeBERTa gets 75%, Rules get 25%.
|
| 192 |
+
|
| 193 |
+
Returns:
|
| 194 |
+
(is_toxic: bool, confidence: float)
|
| 195 |
+
"""
|
| 196 |
+
# Rule score from severity tier
|
| 197 |
+
rule_score = 0.0
|
| 198 |
+
if rules_triggered:
|
| 199 |
+
rule_score = {"critical": 1.0, "high": 0.85, "moderate": 0.55}.get(
|
| 200 |
+
rules_severity, 0.6
|
| 201 |
+
)
|
| 202 |
+
|
| 203 |
+
ml_score = ml_probability
|
| 204 |
+
|
| 205 |
+
llm_score = llm_confidence if llm_toxic else (1.0 - llm_confidence) * 0.1
|
| 206 |
+
|
| 207 |
+
# Layer weights — DeBERTa is primary, trust it more
|
| 208 |
+
if model_type.startswith("transformer"):
|
| 209 |
+
w_rules, w_ml, w_llm = 0.15, 0.50, 0.35
|
| 210 |
+
else:
|
| 211 |
+
# Legacy ML is less reliable — lean heavily on LLM
|
| 212 |
+
w_rules, w_ml, w_llm = 0.25, 0.15, 0.60
|
| 213 |
+
|
| 214 |
+
# LLM not called → redistribute its weight
|
| 215 |
+
llm_was_called = not (llm_confidence == 0.0 and not llm_toxic)
|
| 216 |
+
if not llm_was_called:
|
| 217 |
+
# Redistribute LLM weight proportionally to remaining layers
|
| 218 |
+
w_ml += w_llm * 0.75
|
| 219 |
+
w_rules += w_llm * 0.25
|
| 220 |
+
w_llm = 0.0
|
| 221 |
+
|
| 222 |
+
total_weight = w_rules + w_ml + w_llm
|
| 223 |
+
weighted = (
|
| 224 |
+
(w_rules * rule_score + w_ml * ml_score + w_llm * llm_score)
|
| 225 |
+
/ total_weight
|
| 226 |
+
)
|
| 227 |
+
|
| 228 |
+
# Hard overrides to prevent false negatives from dragging down strong signals:
|
| 229 |
+
# 1. Critical rule always forces high confidence
|
| 230 |
+
if rules_triggered and rules_severity == "critical":
|
| 231 |
+
weighted = max(weighted, 0.90)
|
| 232 |
+
|
| 233 |
+
# 2. High rule severity always forces score above threshold
|
| 234 |
+
if rules_triggered and rules_severity == "high":
|
| 235 |
+
current_threshold = calibration_service.get_threshold()
|
| 236 |
+
weighted = max(weighted, current_threshold + 0.05)
|
| 237 |
+
|
| 238 |
+
# 3. High-confidence LLM classification (e.g. for multilingual/code-mixed inputs) always forces score above threshold
|
| 239 |
+
if llm_toxic and llm_confidence >= 0.75:
|
| 240 |
+
current_threshold = calibration_service.get_threshold()
|
| 241 |
+
weighted = max(weighted, current_threshold + 0.05)
|
| 242 |
+
|
| 243 |
+
is_toxic = weighted >= calibration_service.get_threshold()
|
| 244 |
+
return is_toxic, round(weighted, 3)
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 248 |
+
# PUBLIC DEMO ENDPOINT
|
| 249 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 250 |
+
|
| 251 |
+
@router.post("/predict/demo", tags=["Demo"])
|
| 252 |
+
@limiter.limit("5/minute")
|
| 253 |
+
def predict_demo(request: Request, req: TextRequest = Body(...)):
|
| 254 |
+
"""
|
| 255 |
+
Public toxicity check — no auth required.
|
| 256 |
+
IP rate-limited to 5 req/min. Uses ML + rule engine (no LLM).
|
| 257 |
+
Intended for browser extension demo mode.
|
| 258 |
+
"""
|
| 259 |
+
text = req.text.strip()
|
| 260 |
+
if not text:
|
| 261 |
+
return {"error": "Empty input"}
|
| 262 |
+
|
| 263 |
+
from app.services.overrides import override_service
|
| 264 |
+
override = override_service.check_override(text)
|
| 265 |
+
if override is not None:
|
| 266 |
+
return {
|
| 267 |
+
"toxic": override,
|
| 268 |
+
"confidence": 1.0 if override else 0.0,
|
| 269 |
+
"severity": "high" if override else "low",
|
| 270 |
+
"category": "override" if override else "safe",
|
| 271 |
+
"detected_categories": ["override"] if override else [],
|
| 272 |
+
"abusive_words": [],
|
| 273 |
+
"source": "overrides",
|
| 274 |
+
"demo": True,
|
| 275 |
+
"llm_used": False,
|
| 276 |
+
"model_info": {"type": "dynamic_overrides", "ready": True},
|
| 277 |
+
}
|
| 278 |
+
|
| 279 |
+
rule_text = preprocess_for_rules(text)
|
| 280 |
+
model_text = preprocess_for_model(text)
|
| 281 |
+
|
| 282 |
+
# Rule engine on normalised text
|
| 283 |
+
abusive_hits = detect_abusive_tokens(rule_text)
|
| 284 |
+
rules_triggered = len(abusive_hits) > 0
|
| 285 |
+
rules_severity = get_abuse_severity(abusive_hits) if rules_triggered else "low"
|
| 286 |
+
|
| 287 |
+
# DeBERTa on unicode-safe text
|
| 288 |
+
ml_result = model_service.predict(model_text)
|
| 289 |
+
if not ml_result:
|
| 290 |
+
return {
|
| 291 |
+
"toxic": rules_triggered,
|
| 292 |
+
"confidence": 0.6 if rules_triggered else 0.0,
|
| 293 |
+
"severity": rules_severity if rules_triggered else "low",
|
| 294 |
+
"category": "abusive" if rules_triggered else "safe",
|
| 295 |
+
"detected_categories": abusive_hits,
|
| 296 |
+
"abusive_words": abusive_hits,
|
| 297 |
+
"source": "rules" if rules_triggered else "none",
|
| 298 |
+
"demo": True,
|
| 299 |
+
"llm_used": False,
|
| 300 |
+
"model_info": {"type": model_service.model_type, "ready": False},
|
| 301 |
+
}
|
| 302 |
+
|
| 303 |
+
normalized = normalize_ml_result(ml_result, threshold=settings.demo_ml_threshold)
|
| 304 |
+
|
| 305 |
+
# Light ensemble for demo: rules + ML only
|
| 306 |
+
if rules_triggered and rules_severity in ("critical", "high"):
|
| 307 |
+
toxic = True
|
| 308 |
+
confidence = max(normalized["confidence"], 0.80)
|
| 309 |
+
else:
|
| 310 |
+
toxic = normalized["toxic"] or rules_triggered
|
| 311 |
+
confidence = normalized["confidence"]
|
| 312 |
+
|
| 313 |
+
if confidence >= settings.severity_high:
|
| 314 |
+
severity = "high"
|
| 315 |
+
elif confidence >= settings.severity_medium:
|
| 316 |
+
severity = "medium"
|
| 317 |
+
else:
|
| 318 |
+
severity = "low"
|
| 319 |
+
|
| 320 |
+
category = normalized["category"] if toxic else "safe"
|
| 321 |
+
all_detected = list(set(normalized["detected_categories"] + abusive_hits))
|
| 322 |
+
|
| 323 |
+
return {
|
| 324 |
+
"toxic": toxic,
|
| 325 |
+
"confidence": round(confidence, 3),
|
| 326 |
+
"severity": severity,
|
| 327 |
+
"category": category,
|
| 328 |
+
"detected_categories": all_detected,
|
| 329 |
+
"abusive_words": abusive_hits,
|
| 330 |
+
"source": "rules" if rules_triggered else ("ml" if toxic else "none"),
|
| 331 |
+
"demo": True,
|
| 332 |
+
"llm_used": False,
|
| 333 |
+
"reason": (
|
| 334 |
+
f"[Demo] Detected {category} content."
|
| 335 |
+
if toxic
|
| 336 |
+
else "[Demo] No toxicity detected."
|
| 337 |
+
),
|
| 338 |
+
"ml": ml_result,
|
| 339 |
+
"model_info": {
|
| 340 |
+
"type": model_service.model_type,
|
| 341 |
+
"ready": True,
|
| 342 |
+
},
|
| 343 |
+
}
|
| 344 |
+
|
| 345 |
+
|
| 346 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 347 |
+
# ML-ONLY ENDPOINT (Extension fast-path)
|
| 348 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 349 |
+
|
| 350 |
+
@router.post("/predict/ml")
|
| 351 |
+
@limiter.limit("120/minute")
|
| 352 |
+
def predict_ml_only(
|
| 353 |
+
request: Request,
|
| 354 |
+
req: TextRequest = Body(...),
|
| 355 |
+
user: User = Depends(get_api_user),
|
| 356 |
+
db: Session = Depends(get_db),
|
| 357 |
+
):
|
| 358 |
+
"""
|
| 359 |
+
Extension-safe prediction — authenticated, ML + rules only.
|
| 360 |
+
Skips LLM to avoid token costs on bulk scans.
|
| 361 |
+
"""
|
| 362 |
+
text = req.text.strip()
|
| 363 |
+
if not text:
|
| 364 |
+
return {"error": "Empty input"}
|
| 365 |
+
|
| 366 |
+
_check_plan_limit(user, db)
|
| 367 |
+
logger.info(f"[/predict/ml] user={user.email} usage={user.usage_count}")
|
| 368 |
+
|
| 369 |
+
from app.services.overrides import override_service
|
| 370 |
+
override = override_service.check_override(text)
|
| 371 |
+
if override is not None:
|
| 372 |
+
return {
|
| 373 |
+
"user": user.email,
|
| 374 |
+
"toxic": override,
|
| 375 |
+
"confidence": 1.0 if override else 0.0,
|
| 376 |
+
"severity": "high" if override else "low",
|
| 377 |
+
"source": "overrides",
|
| 378 |
+
"reason": "Dynamic override applied based on feedback.",
|
| 379 |
+
"abusive_words": [],
|
| 380 |
+
"detected_categories": ["override"] if override else [],
|
| 381 |
+
"sentiment": None,
|
| 382 |
+
"ml": None,
|
| 383 |
+
"llm": None,
|
| 384 |
+
"rules": {"triggered": override, "severity": "high" if override else "low"},
|
| 385 |
+
"llm_used": False,
|
| 386 |
+
"model_info": {"type": "dynamic_overrides", "ready": True},
|
| 387 |
+
}
|
| 388 |
+
|
| 389 |
+
rule_text = preprocess_for_rules(text)
|
| 390 |
+
model_text = preprocess_for_model(text)
|
| 391 |
+
|
| 392 |
+
abusive_hits = detect_abusive_tokens(rule_text)
|
| 393 |
+
rules_triggered = len(abusive_hits) > 0
|
| 394 |
+
rules_severity = get_abuse_severity(abusive_hits) if rules_triggered else "low"
|
| 395 |
+
|
| 396 |
+
ml_result = model_service.predict(model_text)
|
| 397 |
+
if not ml_result:
|
| 398 |
+
raise HTTPException(status_code=503, detail="ML model unavailable")
|
| 399 |
+
|
| 400 |
+
normalized = normalize_ml_result(ml_result)
|
| 401 |
+
|
| 402 |
+
# Simple rules + ML decision (no LLM)
|
| 403 |
+
is_toxic = normalized["toxic"] or (rules_triggered and rules_severity in ("critical", "high"))
|
| 404 |
+
confidence = normalized["confidence"]
|
| 405 |
+
if rules_triggered and rules_severity == "critical":
|
| 406 |
+
confidence = max(confidence, 0.90)
|
| 407 |
+
|
| 408 |
+
if confidence >= settings.severity_high:
|
| 409 |
+
severity = "high"
|
| 410 |
+
elif confidence >= settings.severity_medium:
|
| 411 |
+
severity = "medium"
|
| 412 |
+
else:
|
| 413 |
+
severity = "low"
|
| 414 |
+
|
| 415 |
+
payload = {
|
| 416 |
+
"user": user.email,
|
| 417 |
+
"toxic": is_toxic,
|
| 418 |
+
"confidence": round(confidence, 3),
|
| 419 |
+
"severity": severity,
|
| 420 |
+
"source": "rules" if rules_triggered else ("ml" if normalized["toxic"] else "none"),
|
| 421 |
+
"reason": (
|
| 422 |
+
f"ML model flagged as {normalized['category']}."
|
| 423 |
+
if normalized["toxic"]
|
| 424 |
+
else "No toxicity detected by ML model."
|
| 425 |
+
),
|
| 426 |
+
"abusive_words": abusive_hits,
|
| 427 |
+
"detected_categories": list(set(normalized["detected_categories"] + abusive_hits)),
|
| 428 |
+
"sentiment": None,
|
| 429 |
+
"ml": ml_result,
|
| 430 |
+
"llm": None,
|
| 431 |
+
"rules": {"triggered": rules_triggered, "severity": rules_severity},
|
| 432 |
+
"llm_used": False,
|
| 433 |
+
"model_info": model_service.status,
|
| 434 |
+
}
|
| 435 |
+
return _build_response(payload)
|
| 436 |
+
|
| 437 |
+
|
| 438 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 439 |
+
# MAIN FULL-PIPELINE ENDPOINT
|
| 440 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 441 |
+
|
| 442 |
+
@router.post("/predict")
|
| 443 |
+
@limiter.limit("10/minute")
|
| 444 |
+
def predict(
|
| 445 |
+
request: Request,
|
| 446 |
+
req: TextRequest = Body(...),
|
| 447 |
+
user: User = Depends(get_api_user),
|
| 448 |
+
db: Session = Depends(get_db),
|
| 449 |
+
):
|
| 450 |
+
"""
|
| 451 |
+
Full three-layer toxicity analysis:
|
| 452 |
+
1. Rule-based keyword detection
|
| 453 |
+
2. DeBERTa-v3 transformer (ONNX or PyTorch)
|
| 454 |
+
3. LLM contextual analysis (OpenRouter) — triggered selectively
|
| 455 |
+
"""
|
| 456 |
+
text = req.text.strip()
|
| 457 |
+
if not text:
|
| 458 |
+
return {"error": "Empty input"}
|
| 459 |
+
|
| 460 |
+
_check_plan_limit(user, db)
|
| 461 |
+
logger.info(f"[/predict] user={user.email} usage={user.usage_count}")
|
| 462 |
+
|
| 463 |
+
from app.services.overrides import override_service
|
| 464 |
+
override = override_service.check_override(text)
|
| 465 |
+
if override is not None:
|
| 466 |
+
payload = {
|
| 467 |
+
"user": user.email,
|
| 468 |
+
"toxic": override,
|
| 469 |
+
"confidence": 1.0 if override else 0.0,
|
| 470 |
+
"severity": "high" if override else "low",
|
| 471 |
+
"source": "overrides",
|
| 472 |
+
"reason": "Dynamic override applied based on feedback.",
|
| 473 |
+
"abusive_words": [],
|
| 474 |
+
"detected_categories": ["override"] if override else [],
|
| 475 |
+
"sentiment": {"label": "negative" if override else "positive", "polarity": -1.0 if override else 1.0},
|
| 476 |
+
"ml": None,
|
| 477 |
+
"llm": None,
|
| 478 |
+
"rules": {"triggered": override, "severity": "high" if override else "low"},
|
| 479 |
+
"llm_used": False,
|
| 480 |
+
"model_info": {"type": "dynamic_overrides", "ready": True},
|
| 481 |
+
}
|
| 482 |
+
return _build_response(payload)
|
| 483 |
+
|
| 484 |
+
# ── PREPROCESS ────────────────────────────────────────────────────────────
|
| 485 |
+
rule_text = preprocess_for_rules(text)
|
| 486 |
+
model_text = preprocess_for_model(text)
|
| 487 |
+
|
| 488 |
+
# ── SENTIMENT ─────────────────────────────────────────────────────────────
|
| 489 |
+
sentiment = analyze_sentiment(rule_text)
|
| 490 |
+
|
| 491 |
+
# ── RULE ENGINE ───────────────────────────────────────────────────────────
|
| 492 |
+
abusive_hits = detect_abusive_tokens(rule_text)
|
| 493 |
+
rules_triggered = len(abusive_hits) > 0
|
| 494 |
+
rules_severity = get_abuse_severity(abusive_hits) if rules_triggered else "low"
|
| 495 |
+
|
| 496 |
+
# ── DeBERTa ML ────────────────────────────────────────────────────────────
|
| 497 |
+
ml_result = model_service.predict(model_text)
|
| 498 |
+
toxic_probability = 0.0
|
| 499 |
+
if ml_result:
|
| 500 |
+
toxic_probability = float(ml_result.get("toxicity_probability", 0.0))
|
| 501 |
+
|
| 502 |
+
# ── LLM CALL (selective — saves cost & latency) ────────────────────────
|
| 503 |
+
_DEFAULT_LLM = {
|
| 504 |
+
"toxic": False,
|
| 505 |
+
"confidence": 0.0,
|
| 506 |
+
"category": "safe",
|
| 507 |
+
"detected_phrases": [],
|
| 508 |
+
"explanation": "Analysis based on rule engine and ML model signals.",
|
| 509 |
+
}
|
| 510 |
+
llm_result = _DEFAULT_LLM.copy()
|
| 511 |
+
|
| 512 |
+
should_call_llm = len(text) > 10 and (
|
| 513 |
+
rules_triggered or toxic_probability >= settings.ml_trigger_threshold
|
| 514 |
+
)
|
| 515 |
+
|
| 516 |
+
if should_call_llm:
|
| 517 |
+
try:
|
| 518 |
+
llm_result = analyze_toxicity_llm(text)
|
| 519 |
+
except Exception as exc:
|
| 520 |
+
logger.warning(f"[/predict] LLM call failed: {exc}")
|
| 521 |
+
|
| 522 |
+
llm_toxic = llm_result.get("toxic", False)
|
| 523 |
+
llm_conf = llm_result.get("confidence", 0.0)
|
| 524 |
+
|
| 525 |
+
# ── WEIGHTED ENSEMBLE ─────────────────────────────────────────────────────
|
| 526 |
+
toxic, final_confidence = compute_ensemble_score(
|
| 527 |
+
rules_triggered=rules_triggered,
|
| 528 |
+
rules_severity=rules_severity,
|
| 529 |
+
ml_probability=toxic_probability,
|
| 530 |
+
llm_toxic=llm_toxic,
|
| 531 |
+
llm_confidence=llm_conf,
|
| 532 |
+
model_type=model_service.model_type,
|
| 533 |
+
)
|
| 534 |
+
|
| 535 |
+
# ── SEVERITY ──────────────────────────────────────────────────────────────
|
| 536 |
+
if final_confidence >= settings.severity_high:
|
| 537 |
+
severity = "high"
|
| 538 |
+
elif final_confidence >= settings.severity_medium:
|
| 539 |
+
severity = "medium"
|
| 540 |
+
else:
|
| 541 |
+
severity = "low"
|
| 542 |
+
|
| 543 |
+
# ── SENTIMENT OVERRIDE ────────────────────────────────────────────────────
|
| 544 |
+
sentiment = _adjust_sentiment(sentiment, toxic)
|
| 545 |
+
|
| 546 |
+
# ── ABUSIVE WORDS (merge rule + LLM phrases) ──────────────────────────────
|
| 547 |
+
abusive_words = list(set(abusive_hits + llm_result.get("detected_phrases", [])))
|
| 548 |
+
|
| 549 |
+
# ── DETECTED CATEGORIES ───────────────────────────────────────────────────
|
| 550 |
+
ml_cats = [str(c).lower() for c in (ml_result or {}).get("detected_categories", [])]
|
| 551 |
+
all_categories = list(set(ml_cats + abusive_hits))
|
| 552 |
+
|
| 553 |
+
# ── SOURCE ATTRIBUTION ────────────────────────────────────────────────────
|
| 554 |
+
if llm_toxic:
|
| 555 |
+
source = "llm"
|
| 556 |
+
elif rules_triggered:
|
| 557 |
+
source = "rules"
|
| 558 |
+
elif toxic_probability >= 0.5:
|
| 559 |
+
source = "ml"
|
| 560 |
+
else:
|
| 561 |
+
source = "none"
|
| 562 |
+
|
| 563 |
+
# ── EXPLANATION ───────────────────────────────────────────────────────────
|
| 564 |
+
reason = llm_result.get("explanation") or "No explanation available."
|
| 565 |
+
|
| 566 |
+
llm_was_used = should_call_llm and (
|
| 567 |
+
llm_result.get("explanation") != _DEFAULT_LLM["explanation"]
|
| 568 |
+
)
|
| 569 |
+
|
| 570 |
+
# ── RESPONSE ──────────────────────────────────────────────────────────────
|
| 571 |
+
payload = {
|
| 572 |
+
"user": user.email,
|
| 573 |
+
"toxic": toxic,
|
| 574 |
+
"confidence": final_confidence,
|
| 575 |
+
"severity": severity,
|
| 576 |
+
"source": source,
|
| 577 |
+
"reason": reason,
|
| 578 |
+
"abusive_words": abusive_words,
|
| 579 |
+
"detected_categories": all_categories,
|
| 580 |
+
"sentiment": sentiment,
|
| 581 |
+
"ml": ml_result,
|
| 582 |
+
"llm": llm_result,
|
| 583 |
+
"rules": {"triggered": rules_triggered, "severity": rules_severity},
|
| 584 |
+
"llm_used": llm_was_used,
|
| 585 |
+
"model_info": model_service.status,
|
| 586 |
+
}
|
| 587 |
+
|
| 588 |
+
# ── DRIFT MONITORING LOG ──────────────────────────────────────────────────
|
| 589 |
+
try:
|
| 590 |
+
drift_monitor.log_prediction(
|
| 591 |
+
db,
|
| 592 |
+
user_id=user.id,
|
| 593 |
+
input_text=text,
|
| 594 |
+
confidence=final_confidence,
|
| 595 |
+
toxic=toxic,
|
| 596 |
+
severity=severity,
|
| 597 |
+
source=source,
|
| 598 |
+
)
|
| 599 |
+
except Exception as _log_exc:
|
| 600 |
+
logger.warning(f"[/predict] Drift log failed (non-fatal): {_log_exc}")
|
| 601 |
+
|
| 602 |
+
return _build_response(payload)
|
| 603 |
+
|
| 604 |
+
|
| 605 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 606 |
+
# BATCH FILE ANALYSIS
|
| 607 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 608 |
+
|
| 609 |
+
@router.post("/analyze-file")
|
| 610 |
+
async def analyze_file(
|
| 611 |
+
file: UploadFile = File(...),
|
| 612 |
+
user: User = Depends(get_api_user),
|
| 613 |
+
db: Session = Depends(get_db),
|
| 614 |
+
):
|
| 615 |
+
"""
|
| 616 |
+
Batch toxicity analysis from CSV or TXT file upload.
|
| 617 |
+
Uses the shared ensemble (rules + ML) — no LLM for batch speed.
|
| 618 |
+
Capped at 50 texts per request.
|
| 619 |
+
"""
|
| 620 |
+
try:
|
| 621 |
+
content = await file.read()
|
| 622 |
+
|
| 623 |
+
if file.filename and file.filename.endswith(".csv"):
|
| 624 |
+
df = pd.read_csv(io.BytesIO(content))
|
| 625 |
+
# Auto-detect text column
|
| 626 |
+
text_col = next(
|
| 627 |
+
(c for c in df.columns if "text" in c.lower() or "comment" in c.lower()),
|
| 628 |
+
df.columns[0],
|
| 629 |
+
)
|
| 630 |
+
texts = df[text_col].fillna("").astype(str).tolist()
|
| 631 |
+
else:
|
| 632 |
+
texts = content.decode("utf-8").splitlines()
|
| 633 |
+
|
| 634 |
+
batch = [t for t in texts[:50] if t.strip()]
|
| 635 |
+
results = []
|
| 636 |
+
toxic_count = 0
|
| 637 |
+
|
| 638 |
+
for text in batch:
|
| 639 |
+
rule_text = preprocess_for_rules(text)
|
| 640 |
+
model_text = preprocess_for_model(text)
|
| 641 |
+
|
| 642 |
+
abusive_hits = detect_abusive_tokens(rule_text)
|
| 643 |
+
rules_triggered = len(abusive_hits) > 0
|
| 644 |
+
rules_severity = get_abuse_severity(abusive_hits) if rules_triggered else "low"
|
| 645 |
+
|
| 646 |
+
ml_result = model_service.predict(model_text)
|
| 647 |
+
ml_prob = float(ml_result.get("toxicity_probability", 0.0)) if ml_result else 0.0
|
| 648 |
+
ml_cats = [str(c).lower() for c in (ml_result or {}).get("detected_categories", [])]
|
| 649 |
+
|
| 650 |
+
# Shared ensemble (no LLM for batch speed)
|
| 651 |
+
is_toxic, confidence = compute_ensemble_score(
|
| 652 |
+
rules_triggered=rules_triggered,
|
| 653 |
+
rules_severity=rules_severity,
|
| 654 |
+
ml_probability=ml_prob,
|
| 655 |
+
llm_toxic=False,
|
| 656 |
+
llm_confidence=0.0,
|
| 657 |
+
model_type=model_service.model_type,
|
| 658 |
+
)
|
| 659 |
+
|
| 660 |
+
if confidence >= settings.severity_high:
|
| 661 |
+
severity = "high"
|
| 662 |
+
elif confidence >= settings.severity_medium:
|
| 663 |
+
severity = "medium"
|
| 664 |
+
else:
|
| 665 |
+
severity = "low"
|
| 666 |
+
|
| 667 |
+
if is_toxic:
|
| 668 |
+
toxic_count += 1
|
| 669 |
+
|
| 670 |
+
sentiment = analyze_sentiment(rule_text)
|
| 671 |
+
if is_toxic:
|
| 672 |
+
sentiment["label"] = "negative"
|
| 673 |
+
|
| 674 |
+
source = "rules" if rules_triggered else ("ml" if ml_prob >= 0.5 else "none")
|
| 675 |
+
all_categories = list(set(ml_cats + abusive_hits))
|
| 676 |
+
|
| 677 |
+
results.append({
|
| 678 |
+
"text": text,
|
| 679 |
+
"toxic": is_toxic,
|
| 680 |
+
"confidence": round(confidence, 3),
|
| 681 |
+
"severity": severity,
|
| 682 |
+
"source": source,
|
| 683 |
+
"abusive_words": abusive_hits,
|
| 684 |
+
"detected_categories": all_categories,
|
| 685 |
+
"sentiment": sentiment,
|
| 686 |
+
})
|
| 687 |
+
|
| 688 |
+
return {
|
| 689 |
+
"total": len(results),
|
| 690 |
+
"toxic_count": toxic_count,
|
| 691 |
+
"clean_count": len(results) - toxic_count,
|
| 692 |
+
"toxic_rate": round(toxic_count / len(results), 3) if results else 0.0,
|
| 693 |
+
"model_info": model_service.status,
|
| 694 |
+
"results": results,
|
| 695 |
+
}
|
| 696 |
+
|
| 697 |
+
except Exception as exc:
|
| 698 |
+
logger.error(f"[/analyze-file] Failed: {exc}")
|
| 699 |
+
raise HTTPException(status_code=400, detail=f"Failed to process file: {str(exc)}")
|
app/routes/monitoring.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — Monitoring Endpoints
|
| 3 |
+
=====================================
|
| 4 |
+
GET /monitoring/stats — Live model health metrics + drift alert
|
| 5 |
+
GET /monitoring/drift — Time-series confidence data for charting
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from fastapi import APIRouter, Depends, Request
|
| 9 |
+
from sqlalchemy.orm import Session
|
| 10 |
+
|
| 11 |
+
from app.routes.auth import get_api_user
|
| 12 |
+
from app.core.limiter import limiter
|
| 13 |
+
from app.core.logger import logger
|
| 14 |
+
from app.services.drift_monitor import drift_monitor
|
| 15 |
+
from models import User
|
| 16 |
+
from database import get_db
|
| 17 |
+
|
| 18 |
+
router = APIRouter()
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@router.get("/monitoring/stats", tags=["Monitoring"])
|
| 22 |
+
@limiter.limit("30/minute")
|
| 23 |
+
def get_monitoring_stats(
|
| 24 |
+
request: Request,
|
| 25 |
+
user: User = Depends(get_api_user),
|
| 26 |
+
db: Session = Depends(get_db),
|
| 27 |
+
):
|
| 28 |
+
"""
|
| 29 |
+
Live model monitoring dashboard data.
|
| 30 |
+
|
| 31 |
+
Returns rolling window statistics, drift detection result,
|
| 32 |
+
toxic rate, severity breakdown, and hourly request volume.
|
| 33 |
+
"""
|
| 34 |
+
logger.info(f"[/monitoring/stats] user={user.email}")
|
| 35 |
+
return drift_monitor.get_stats(db)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@router.get("/monitoring/drift", tags=["Monitoring"])
|
| 39 |
+
@limiter.limit("20/minute")
|
| 40 |
+
def get_drift_series(
|
| 41 |
+
request: Request,
|
| 42 |
+
limit: int = 100,
|
| 43 |
+
user: User = Depends(get_api_user),
|
| 44 |
+
db: Session = Depends(get_db),
|
| 45 |
+
):
|
| 46 |
+
"""
|
| 47 |
+
Time-series of confidence scores for the drift trend chart.
|
| 48 |
+
Use `limit` query param (default 100, max 500).
|
| 49 |
+
"""
|
| 50 |
+
limit = min(limit, 500)
|
| 51 |
+
logger.info(f"[/monitoring/drift] user={user.email} limit={limit}")
|
| 52 |
+
return {
|
| 53 |
+
"series": drift_monitor.get_drift_series(db, limit=limit),
|
| 54 |
+
"limit": limit,
|
| 55 |
+
}
|
app/routes/realtime.py
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — Real-Time Chat Moderation
|
| 3 |
+
==========================================
|
| 4 |
+
POST /chat/moderate — Authenticated, rate-limited, fast chat moderation.
|
| 5 |
+
|
| 6 |
+
Uses DeBERTa + rule engine for fast decision.
|
| 7 |
+
Calls LLM only when toxicity is confirmed (no wasted tokens on clean messages).
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from fastapi import APIRouter, Depends, Request, Body
|
| 11 |
+
from sqlalchemy.orm import Session
|
| 12 |
+
from pydantic import BaseModel
|
| 13 |
+
from datetime import datetime
|
| 14 |
+
|
| 15 |
+
from app.routes.auth import get_api_user
|
| 16 |
+
from app.core.limiter import limiter
|
| 17 |
+
from app.core.logger import logger
|
| 18 |
+
from app.core.config import settings
|
| 19 |
+
|
| 20 |
+
from utils.preprocessing import preprocess_for_rules, preprocess_for_model
|
| 21 |
+
from utils.abuse_words import detect_abusive_tokens, get_abuse_severity
|
| 22 |
+
from utils.llm_guard import analyze_toxicity_llm
|
| 23 |
+
from app.services.model_service import model_service
|
| 24 |
+
|
| 25 |
+
from app.services.calibration import calibration_service
|
| 26 |
+
|
| 27 |
+
from models import User
|
| 28 |
+
from database import get_db
|
| 29 |
+
|
| 30 |
+
router = APIRouter()
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 34 |
+
# SCHEMA
|
| 35 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 36 |
+
|
| 37 |
+
class ChatRequest(BaseModel):
|
| 38 |
+
message: str
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 42 |
+
# ENDPOINT
|
| 43 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 44 |
+
|
| 45 |
+
@router.post("/chat/moderate")
|
| 46 |
+
@limiter.limit("30/minute")
|
| 47 |
+
def moderate_chat(
|
| 48 |
+
request: Request,
|
| 49 |
+
req: ChatRequest = Body(...),
|
| 50 |
+
user: User = Depends(get_api_user),
|
| 51 |
+
db: Session = Depends(get_db),
|
| 52 |
+
):
|
| 53 |
+
"""
|
| 54 |
+
Real-time chat message moderation.
|
| 55 |
+
|
| 56 |
+
Decision flow:
|
| 57 |
+
1. Rule engine — instant check for known slurs
|
| 58 |
+
2. DeBERTa — ML probability score
|
| 59 |
+
3. LLM — called only when content is flagged (avoids wasted tokens)
|
| 60 |
+
"""
|
| 61 |
+
text = req.message.strip()
|
| 62 |
+
if not text:
|
| 63 |
+
return {"error": "Empty message"}
|
| 64 |
+
|
| 65 |
+
rule_text = preprocess_for_rules(text)
|
| 66 |
+
model_text = preprocess_for_model(text)
|
| 67 |
+
|
| 68 |
+
# Track last activity for analytics
|
| 69 |
+
user.last_used = datetime.utcnow()
|
| 70 |
+
db.commit()
|
| 71 |
+
|
| 72 |
+
# ── RULE ENGINE ───────────────────────────────────────────────────────────
|
| 73 |
+
abusive_hits = detect_abusive_tokens(rule_text)
|
| 74 |
+
rules_triggered = len(abusive_hits) > 0
|
| 75 |
+
rules_severity = get_abuse_severity(abusive_hits) if rules_triggered else "low"
|
| 76 |
+
|
| 77 |
+
# ── DeBERTa ───────────────────────────────────────────────────────────────
|
| 78 |
+
ml_result = model_service.predict(model_text)
|
| 79 |
+
ml_score = 0.0
|
| 80 |
+
ml_categories: list[str] = []
|
| 81 |
+
if ml_result:
|
| 82 |
+
ml_score = float(ml_result.get("toxicity_probability", 0.0))
|
| 83 |
+
ml_categories = [str(c).lower() for c in ml_result.get("detected_categories", [])]
|
| 84 |
+
|
| 85 |
+
# ── FAST DECISION ─────────────────────────────────────────────────────────
|
| 86 |
+
# Threshold — dynamically adjusted based on feedback calibration
|
| 87 |
+
toxic = rules_triggered or ml_score >= calibration_service.get_threshold()
|
| 88 |
+
|
| 89 |
+
# Critical rule always overrides
|
| 90 |
+
if rules_triggered and rules_severity == "critical":
|
| 91 |
+
toxic = True
|
| 92 |
+
|
| 93 |
+
# ── CONFIDENCE ────────────────────────────────────────────────────────────
|
| 94 |
+
confidence = ml_score
|
| 95 |
+
if rules_triggered:
|
| 96 |
+
rule_boost = {"critical": 0.95, "high": 0.85, "moderate": 0.65}.get(rules_severity, 0.6)
|
| 97 |
+
confidence = max(confidence, rule_boost)
|
| 98 |
+
|
| 99 |
+
# ── SEVERITY ──────────────────────────────────────────────────────────────
|
| 100 |
+
if confidence >= settings.severity_high:
|
| 101 |
+
severity = "high"
|
| 102 |
+
elif confidence >= settings.severity_medium:
|
| 103 |
+
severity = "medium"
|
| 104 |
+
else:
|
| 105 |
+
severity = "low"
|
| 106 |
+
|
| 107 |
+
# ── LLM (only if toxic and text is meaningful) ───────────────────���────────
|
| 108 |
+
explanation = "Message appears safe."
|
| 109 |
+
if toxic and len(text) > 10:
|
| 110 |
+
try:
|
| 111 |
+
llm = analyze_toxicity_llm(text)
|
| 112 |
+
explanation = llm.get("explanation", "Toxic content detected.")
|
| 113 |
+
except Exception as exc:
|
| 114 |
+
logger.warning(f"[/chat/moderate] LLM failed: {exc}")
|
| 115 |
+
explanation = "Toxic content detected (explanation unavailable)."
|
| 116 |
+
|
| 117 |
+
# ── DETECTED CATEGORIES ───────────────────────────────────────────────────
|
| 118 |
+
all_categories = list(set(ml_categories + abusive_hits))
|
| 119 |
+
|
| 120 |
+
logger.info(
|
| 121 |
+
f"[/chat/moderate] user={user.email} | toxic={toxic} | "
|
| 122 |
+
f"ml={ml_score:.3f} | rules={rules_triggered}"
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
return {
|
| 126 |
+
"message": text,
|
| 127 |
+
"toxic": toxic,
|
| 128 |
+
"confidence": round(confidence, 3),
|
| 129 |
+
"severity": severity,
|
| 130 |
+
"abusive_words": abusive_hits,
|
| 131 |
+
"detected_categories": all_categories,
|
| 132 |
+
"rules": {"triggered": rules_triggered, "severity": rules_severity},
|
| 133 |
+
"explanation": explanation,
|
| 134 |
+
"model_info": model_service.status,
|
| 135 |
+
}
|
app/routes/stream.py
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — Real-Time WebSocket Stream Analyzer
|
| 3 |
+
====================================================
|
| 4 |
+
WebSocket /ws/stream — Accepts a continuous stream of text messages,
|
| 5 |
+
runs full toxicity analysis on each, and pushes results back instantly.
|
| 6 |
+
|
| 7 |
+
Protocol:
|
| 8 |
+
1. Client connects to ws://host/ws/stream
|
| 9 |
+
2. Client sends first message: {"type": "auth", "api_key": "tg_..."}
|
| 10 |
+
3. Server responds: {"type": "auth_ok", "user": "email@example.com"}
|
| 11 |
+
4. Client sends text messages: {"type": "message", "text": "hello world", "id": "msg_1"}
|
| 12 |
+
5. Server streams back: {"type": "result", "id": "msg_1", "toxic": false, ...}
|
| 13 |
+
6. Client sends {"type": "ping"} → server sends {"type": "pong"}
|
| 14 |
+
7. Client disconnects normally
|
| 15 |
+
|
| 16 |
+
Throughput: ~50 msg/sec on CPU (ONNX inference ~20ms/msg)
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
import json
|
| 20 |
+
import time
|
| 21 |
+
from datetime import datetime
|
| 22 |
+
|
| 23 |
+
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
|
| 24 |
+
from sqlalchemy.orm import Session
|
| 25 |
+
|
| 26 |
+
from app.core.logger import logger
|
| 27 |
+
from app.core.config import settings
|
| 28 |
+
from app.services.calibration import calibration_service
|
| 29 |
+
from app.services.model_service import model_service
|
| 30 |
+
from app.services.drift_monitor import drift_monitor
|
| 31 |
+
from database import SessionLocal, get_db
|
| 32 |
+
|
| 33 |
+
from utils.preprocessing import preprocess_for_rules, preprocess_for_model
|
| 34 |
+
from utils.abuse_words import detect_abusive_tokens, get_abuse_severity
|
| 35 |
+
from models import User
|
| 36 |
+
|
| 37 |
+
router = APIRouter()
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
async def _authenticate(websocket: WebSocket, data: dict) -> User | None:
|
| 41 |
+
"""Validate API key from first WebSocket message. Returns User or None."""
|
| 42 |
+
from database import SessionLocal
|
| 43 |
+
from models import User
|
| 44 |
+
|
| 45 |
+
api_key = data.get("api_key", "").strip()
|
| 46 |
+
if not api_key:
|
| 47 |
+
return None
|
| 48 |
+
|
| 49 |
+
db = SessionLocal()
|
| 50 |
+
try:
|
| 51 |
+
user = db.query(User).filter(User.api_key == api_key).first()
|
| 52 |
+
return user
|
| 53 |
+
except Exception:
|
| 54 |
+
return None
|
| 55 |
+
finally:
|
| 56 |
+
db.close()
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _analyze_message(text: str) -> dict:
|
| 60 |
+
"""Run full toxicity pipeline on a single message. Returns result dict."""
|
| 61 |
+
start = time.perf_counter()
|
| 62 |
+
|
| 63 |
+
rule_text = preprocess_for_rules(text)
|
| 64 |
+
model_text = preprocess_for_model(text)
|
| 65 |
+
|
| 66 |
+
# Rule engine
|
| 67 |
+
abusive_hits = detect_abusive_tokens(rule_text)
|
| 68 |
+
rules_triggered = len(abusive_hits) > 0
|
| 69 |
+
rules_severity = get_abuse_severity(abusive_hits) if rules_triggered else "low"
|
| 70 |
+
|
| 71 |
+
# DeBERTa ONNX
|
| 72 |
+
ml_result = model_service.predict(model_text)
|
| 73 |
+
ml_score = 0.0
|
| 74 |
+
labels = {}
|
| 75 |
+
if ml_result:
|
| 76 |
+
ml_score = float(ml_result.get("toxicity_probability", 0.0))
|
| 77 |
+
labels = ml_result.get("labels", {})
|
| 78 |
+
|
| 79 |
+
# Decision
|
| 80 |
+
confidence = ml_score
|
| 81 |
+
if rules_triggered:
|
| 82 |
+
rule_boost = {"critical": 0.95, "high": 0.85, "moderate": 0.65}.get(rules_severity, 0.6)
|
| 83 |
+
confidence = max(confidence, rule_boost)
|
| 84 |
+
|
| 85 |
+
toxic = rules_triggered or ml_score >= calibration_service.get_threshold()
|
| 86 |
+
|
| 87 |
+
severity = (
|
| 88 |
+
"high" if confidence >= settings.severity_high
|
| 89 |
+
else "medium" if confidence >= settings.severity_medium
|
| 90 |
+
else "low"
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
+
latency_ms = round((time.perf_counter() - start) * 1000, 1)
|
| 94 |
+
|
| 95 |
+
return {
|
| 96 |
+
"toxic": toxic,
|
| 97 |
+
"confidence": round(confidence, 3),
|
| 98 |
+
"severity": severity,
|
| 99 |
+
"abusive_words": abusive_hits,
|
| 100 |
+
"rules_triggered": rules_triggered,
|
| 101 |
+
"ml_score": round(ml_score, 3),
|
| 102 |
+
"labels": labels,
|
| 103 |
+
"latency_ms": latency_ms,
|
| 104 |
+
}
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
@router.websocket("/ws/stream")
|
| 108 |
+
async def websocket_stream(websocket: WebSocket):
|
| 109 |
+
"""
|
| 110 |
+
Real-time toxicity stream analyzer.
|
| 111 |
+
|
| 112 |
+
Accepts continuous messages, analyzes each with DeBERTa + rules,
|
| 113 |
+
and pushes results back with <50ms latency.
|
| 114 |
+
"""
|
| 115 |
+
await websocket.accept()
|
| 116 |
+
user = None
|
| 117 |
+
message_count = 0
|
| 118 |
+
session_start = time.time()
|
| 119 |
+
|
| 120 |
+
logger.info(f"[WS] New connection from {websocket.client.host}")
|
| 121 |
+
|
| 122 |
+
try:
|
| 123 |
+
# ── STEP 1: Auth handshake ─────────────────────────────────────────────
|
| 124 |
+
raw = await websocket.receive_text()
|
| 125 |
+
try:
|
| 126 |
+
auth_data = json.loads(raw)
|
| 127 |
+
except json.JSONDecodeError:
|
| 128 |
+
await websocket.send_json({
|
| 129 |
+
"type": "error",
|
| 130 |
+
"message": "Invalid JSON in auth message"
|
| 131 |
+
})
|
| 132 |
+
await websocket.close(code=1003)
|
| 133 |
+
return
|
| 134 |
+
|
| 135 |
+
if auth_data.get("type") != "auth":
|
| 136 |
+
await websocket.send_json({
|
| 137 |
+
"type": "error",
|
| 138 |
+
"message": "First message must be: {type: 'auth', api_key: 'tg_...'}"
|
| 139 |
+
})
|
| 140 |
+
await websocket.close(code=1008)
|
| 141 |
+
return
|
| 142 |
+
|
| 143 |
+
user = await _authenticate(websocket, auth_data)
|
| 144 |
+
if not user:
|
| 145 |
+
await websocket.send_json({
|
| 146 |
+
"type": "auth_error",
|
| 147 |
+
"message": "Invalid API key"
|
| 148 |
+
})
|
| 149 |
+
await websocket.close(code=1008)
|
| 150 |
+
return
|
| 151 |
+
|
| 152 |
+
await websocket.send_json({
|
| 153 |
+
"type": "auth_ok",
|
| 154 |
+
"user": user.email,
|
| 155 |
+
"message": "Connected to ToxiGuard AI stream",
|
| 156 |
+
"model": model_service.status.get("description", "unknown"),
|
| 157 |
+
})
|
| 158 |
+
logger.info(f"[WS] Authenticated: {user.email}")
|
| 159 |
+
|
| 160 |
+
# ── STEP 2: Message loop ───────────────────────────────────────────────
|
| 161 |
+
while True:
|
| 162 |
+
raw = await websocket.receive_text()
|
| 163 |
+
|
| 164 |
+
try:
|
| 165 |
+
msg = json.loads(raw)
|
| 166 |
+
except json.JSONDecodeError:
|
| 167 |
+
await websocket.send_json({"type": "error", "message": "Invalid JSON"})
|
| 168 |
+
continue
|
| 169 |
+
|
| 170 |
+
msg_type = msg.get("type", "message")
|
| 171 |
+
|
| 172 |
+
# Ping/pong keepalive
|
| 173 |
+
if msg_type == "ping":
|
| 174 |
+
await websocket.send_json({"type": "pong", "ts": int(time.time())})
|
| 175 |
+
continue
|
| 176 |
+
|
| 177 |
+
# Disconnect request
|
| 178 |
+
if msg_type == "disconnect":
|
| 179 |
+
break
|
| 180 |
+
|
| 181 |
+
# Text analysis
|
| 182 |
+
if msg_type == "message":
|
| 183 |
+
text = str(msg.get("text", "")).strip()
|
| 184 |
+
msg_id = msg.get("id", str(message_count))
|
| 185 |
+
|
| 186 |
+
if not text:
|
| 187 |
+
await websocket.send_json({
|
| 188 |
+
"type": "result",
|
| 189 |
+
"id": msg_id,
|
| 190 |
+
"error": "Empty text"
|
| 191 |
+
})
|
| 192 |
+
continue
|
| 193 |
+
|
| 194 |
+
# Enforce max length
|
| 195 |
+
if len(text) > 1000:
|
| 196 |
+
text = text[:1000]
|
| 197 |
+
|
| 198 |
+
# Run analysis
|
| 199 |
+
result = _analyze_message(text)
|
| 200 |
+
message_count += 1
|
| 201 |
+
|
| 202 |
+
# Log to drift monitor (background — don't await DB here)
|
| 203 |
+
try:
|
| 204 |
+
db = SessionLocal()
|
| 205 |
+
drift_monitor.log_prediction(
|
| 206 |
+
db,
|
| 207 |
+
user_id=user.id,
|
| 208 |
+
input_text=text,
|
| 209 |
+
confidence=result["confidence"],
|
| 210 |
+
toxic=result["toxic"],
|
| 211 |
+
severity=result["severity"],
|
| 212 |
+
source="websocket_stream",
|
| 213 |
+
)
|
| 214 |
+
db.close()
|
| 215 |
+
except Exception:
|
| 216 |
+
pass
|
| 217 |
+
|
| 218 |
+
await websocket.send_json({
|
| 219 |
+
"type": "result",
|
| 220 |
+
"id": msg_id,
|
| 221 |
+
"text": text,
|
| 222 |
+
"toxic": result["toxic"],
|
| 223 |
+
"confidence": result["confidence"],
|
| 224 |
+
"severity": result["severity"],
|
| 225 |
+
"abusive_words": result["abusive_words"],
|
| 226 |
+
"rules_triggered": result["rules_triggered"],
|
| 227 |
+
"latency_ms": result["latency_ms"],
|
| 228 |
+
"message_num": message_count,
|
| 229 |
+
})
|
| 230 |
+
|
| 231 |
+
except WebSocketDisconnect:
|
| 232 |
+
pass
|
| 233 |
+
except Exception as exc:
|
| 234 |
+
logger.error(f"[WS] Unexpected error: {exc}")
|
| 235 |
+
try:
|
| 236 |
+
await websocket.send_json({"type": "error", "message": str(exc)})
|
| 237 |
+
except Exception:
|
| 238 |
+
pass
|
| 239 |
+
finally:
|
| 240 |
+
elapsed = round(time.time() - session_start, 1)
|
| 241 |
+
user_email = user.email if user else "unauthenticated"
|
| 242 |
+
logger.info(
|
| 243 |
+
f"[WS] Disconnected: {user_email} | "
|
| 244 |
+
f"messages={message_count} | duration={elapsed}s"
|
| 245 |
+
)
|
app/services/__init__.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# backend/app/services/__init__.py
|
| 2 |
+
# Marker file to make services a package
|
app/services/calibration.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — Adaptive Threshold Calibration Service
|
| 3 |
+
=======================================================
|
| 4 |
+
Adjusts the toxicity decision threshold in real-time based on user feedback.
|
| 5 |
+
No full model retrain required — works by tuning the confidence cutoff.
|
| 6 |
+
|
| 7 |
+
Algorithm:
|
| 8 |
+
- Maintain a sliding window of recent feedback (FP vs FN counts)
|
| 9 |
+
- False Positive (FP): model too aggressive → raise threshold slightly
|
| 10 |
+
- False Negative (FN): model too lenient → lower threshold slightly
|
| 11 |
+
- Step size = CALIBRATION_STEP (default 0.01) per correction
|
| 12 |
+
- Threshold is clamped to [MIN_THRESHOLD, MAX_THRESHOLD]
|
| 13 |
+
- Persisted to calibration.json so it survives server restarts
|
| 14 |
+
|
| 15 |
+
This is called "online threshold calibration" — used in production at
|
| 16 |
+
scale where full retrains are expensive (Meta, Google CSAM detection, etc.)
|
| 17 |
+
|
| 18 |
+
Usage:
|
| 19 |
+
from app.services.calibration import calibration_service
|
| 20 |
+
calibration_service.apply_feedback("false_positive", confidence=0.72)
|
| 21 |
+
threshold = calibration_service.get_threshold()
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
from __future__ import annotations
|
| 25 |
+
|
| 26 |
+
import os
|
| 27 |
+
import json
|
| 28 |
+
import threading
|
| 29 |
+
from datetime import datetime
|
| 30 |
+
from typing import Optional
|
| 31 |
+
|
| 32 |
+
from app.core.logger import logger
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 36 |
+
# CONFIG
|
| 37 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 38 |
+
|
| 39 |
+
DEFAULT_THRESHOLD = 0.50
|
| 40 |
+
MIN_THRESHOLD = 0.30 # Never go below this — too many FN
|
| 41 |
+
MAX_THRESHOLD = 0.75 # Never go above this — too many FP
|
| 42 |
+
CALIBRATION_STEP = 0.01 # Threshold shift per single correction
|
| 43 |
+
|
| 44 |
+
# Path to persist threshold across restarts
|
| 45 |
+
_CALIBRATION_FILE = os.path.join(
|
| 46 |
+
os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
|
| 47 |
+
"data", "calibration.json"
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 52 |
+
# SERVICE
|
| 53 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 54 |
+
|
| 55 |
+
class AdaptiveCalibrationService:
|
| 56 |
+
"""
|
| 57 |
+
Online threshold calibration from user feedback.
|
| 58 |
+
Thread-safe for concurrent FastAPI requests.
|
| 59 |
+
"""
|
| 60 |
+
|
| 61 |
+
def __init__(self) -> None:
|
| 62 |
+
self._lock = threading.Lock()
|
| 63 |
+
self._threshold: float = DEFAULT_THRESHOLD
|
| 64 |
+
self._fp_count: int = 0 # Lifetime FP corrections
|
| 65 |
+
self._fn_count: int = 0 # Lifetime FN corrections
|
| 66 |
+
self._total_adjustments: int = 0
|
| 67 |
+
self._history: list[dict] = [] # Last 20 adjustments
|
| 68 |
+
self._last_updated: Optional[str] = None
|
| 69 |
+
self._load()
|
| 70 |
+
|
| 71 |
+
# ── PERSISTENCE ───────────────────────────────────────────────────────────
|
| 72 |
+
|
| 73 |
+
def _load(self) -> None:
|
| 74 |
+
"""Load calibration state from disk (survives restarts)."""
|
| 75 |
+
try:
|
| 76 |
+
if os.path.exists(_CALIBRATION_FILE):
|
| 77 |
+
with open(_CALIBRATION_FILE, "r") as f:
|
| 78 |
+
data = json.load(f)
|
| 79 |
+
self._threshold = float(data.get("threshold", DEFAULT_THRESHOLD))
|
| 80 |
+
self._fp_count = int(data.get("fp_count", 0))
|
| 81 |
+
self._fn_count = int(data.get("fn_count", 0))
|
| 82 |
+
self._total_adjustments = int(data.get("total_adjustments", 0))
|
| 83 |
+
self._history = data.get("history", [])[-20:]
|
| 84 |
+
logger.info(
|
| 85 |
+
f"[Calibration] Loaded threshold={self._threshold:.3f} "
|
| 86 |
+
f"(FP={self._fp_count}, FN={self._fn_count})"
|
| 87 |
+
)
|
| 88 |
+
except Exception as exc:
|
| 89 |
+
logger.warning(f"[Calibration] Could not load calibration file: {exc}")
|
| 90 |
+
|
| 91 |
+
def _save(self) -> None:
|
| 92 |
+
"""Persist calibration state to disk."""
|
| 93 |
+
try:
|
| 94 |
+
os.makedirs(os.path.dirname(_CALIBRATION_FILE), exist_ok=True)
|
| 95 |
+
with open(_CALIBRATION_FILE, "w") as f:
|
| 96 |
+
json.dump({
|
| 97 |
+
"threshold": self._threshold,
|
| 98 |
+
"fp_count": self._fp_count,
|
| 99 |
+
"fn_count": self._fn_count,
|
| 100 |
+
"total_adjustments": self._total_adjustments,
|
| 101 |
+
"history": self._history[-20:],
|
| 102 |
+
"last_updated": self._last_updated,
|
| 103 |
+
}, f, indent=2)
|
| 104 |
+
except Exception as exc:
|
| 105 |
+
logger.warning(f"[Calibration] Could not save calibration: {exc}")
|
| 106 |
+
|
| 107 |
+
# ── CORE ��─────────────────────────────────────────────────────────────────
|
| 108 |
+
|
| 109 |
+
def apply_feedback(
|
| 110 |
+
self,
|
| 111 |
+
feedback_type: str,
|
| 112 |
+
confidence_at_time: Optional[float] = None,
|
| 113 |
+
) -> dict:
|
| 114 |
+
"""
|
| 115 |
+
Adjust threshold based on one feedback event.
|
| 116 |
+
|
| 117 |
+
Args:
|
| 118 |
+
feedback_type: "false_positive" | "false_negative" | "confirmed_correct"
|
| 119 |
+
confidence_at_time: The model's confidence when this prediction was made.
|
| 120 |
+
|
| 121 |
+
Returns:
|
| 122 |
+
{"old_threshold": float, "new_threshold": float, "direction": str}
|
| 123 |
+
"""
|
| 124 |
+
with self._lock:
|
| 125 |
+
old = self._threshold
|
| 126 |
+
|
| 127 |
+
if feedback_type == "false_positive":
|
| 128 |
+
# Model was too aggressive: raise threshold (harder to trigger toxic)
|
| 129 |
+
self._threshold = min(
|
| 130 |
+
MAX_THRESHOLD,
|
| 131 |
+
self._threshold + CALIBRATION_STEP
|
| 132 |
+
)
|
| 133 |
+
self._fp_count += 1
|
| 134 |
+
direction = "raised"
|
| 135 |
+
|
| 136 |
+
elif feedback_type == "false_negative":
|
| 137 |
+
# Model was too lenient: lower threshold (easier to trigger toxic)
|
| 138 |
+
self._threshold = max(
|
| 139 |
+
MIN_THRESHOLD,
|
| 140 |
+
self._threshold - CALIBRATION_STEP
|
| 141 |
+
)
|
| 142 |
+
self._fn_count += 1
|
| 143 |
+
direction = "lowered"
|
| 144 |
+
|
| 145 |
+
else:
|
| 146 |
+
# Confirmed correct — no threshold change
|
| 147 |
+
direction = "unchanged"
|
| 148 |
+
|
| 149 |
+
self._total_adjustments += 1
|
| 150 |
+
self._last_updated = datetime.utcnow().isoformat()
|
| 151 |
+
|
| 152 |
+
event = {
|
| 153 |
+
"ts": self._last_updated,
|
| 154 |
+
"type": feedback_type,
|
| 155 |
+
"old": round(old, 3),
|
| 156 |
+
"new": round(self._threshold, 3),
|
| 157 |
+
"confidence_at_time": confidence_at_time,
|
| 158 |
+
}
|
| 159 |
+
self._history.append(event)
|
| 160 |
+
self._history = self._history[-20:]
|
| 161 |
+
|
| 162 |
+
self._save()
|
| 163 |
+
|
| 164 |
+
logger.info(
|
| 165 |
+
f"[Calibration] {feedback_type} → threshold {old:.3f} → "
|
| 166 |
+
f"{self._threshold:.3f} ({direction})"
|
| 167 |
+
)
|
| 168 |
+
|
| 169 |
+
return {
|
| 170 |
+
"old_threshold": round(old, 3),
|
| 171 |
+
"new_threshold": round(self._threshold, 3),
|
| 172 |
+
"direction": direction,
|
| 173 |
+
"fp_count": self._fp_count,
|
| 174 |
+
"fn_count": self._fn_count,
|
| 175 |
+
}
|
| 176 |
+
|
| 177 |
+
def get_threshold(self) -> float:
|
| 178 |
+
"""Get the current calibrated threshold."""
|
| 179 |
+
with self._lock:
|
| 180 |
+
return self._threshold
|
| 181 |
+
|
| 182 |
+
def get_status(self) -> dict:
|
| 183 |
+
"""Return full calibration status for the monitoring dashboard."""
|
| 184 |
+
with self._lock:
|
| 185 |
+
return {
|
| 186 |
+
"current_threshold": round(self._threshold, 4),
|
| 187 |
+
"default_threshold": DEFAULT_THRESHOLD,
|
| 188 |
+
"shift_from_default": round(self._threshold - DEFAULT_THRESHOLD, 4),
|
| 189 |
+
"fp_corrections": self._fp_count,
|
| 190 |
+
"fn_corrections": self._fn_count,
|
| 191 |
+
"total_adjustments": self._total_adjustments,
|
| 192 |
+
"last_updated": self._last_updated,
|
| 193 |
+
"history": self._history[-10:],
|
| 194 |
+
"bounds": {"min": MIN_THRESHOLD, "max": MAX_THRESHOLD},
|
| 195 |
+
"step_size": CALIBRATION_STEP,
|
| 196 |
+
}
|
| 197 |
+
|
| 198 |
+
def reset(self) -> None:
|
| 199 |
+
"""Reset threshold to default (admin action)."""
|
| 200 |
+
with self._lock:
|
| 201 |
+
self._threshold = DEFAULT_THRESHOLD
|
| 202 |
+
self._fp_count = 0
|
| 203 |
+
self._fn_count = 0
|
| 204 |
+
self._total_adjustments = 0
|
| 205 |
+
self._history = []
|
| 206 |
+
self._save()
|
| 207 |
+
logger.info("[Calibration] Threshold reset to default")
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
# Module-level singleton
|
| 211 |
+
calibration_service = AdaptiveCalibrationService()
|
app/services/drift_monitor.py
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — Data Drift Detection & Model Monitoring Service
|
| 3 |
+
===============================================================
|
| 4 |
+
Tracks prediction confidence distributions over time.
|
| 5 |
+
Detects when the model's behavior shifts (new slang, distribution change).
|
| 6 |
+
|
| 7 |
+
Algorithm:
|
| 8 |
+
- Maintain rolling window of last N confidence scores from PredictionLog DB
|
| 9 |
+
- Compute mean, stddev, and toxic rate
|
| 10 |
+
- Compare current window to a baseline (first 100 predictions or configured)
|
| 11 |
+
- Alert if mean confidence drifts > DRIFT_THRESHOLD standard deviations
|
| 12 |
+
- KL-divergence computed via histogram binning (no scipy needed)
|
| 13 |
+
|
| 14 |
+
Usage:
|
| 15 |
+
from app.services.drift_monitor import DriftMonitor
|
| 16 |
+
monitor = DriftMonitor()
|
| 17 |
+
monitor.log_prediction(db, confidence=0.87, toxic=True, ...)
|
| 18 |
+
stats = monitor.get_stats(db)
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
from __future__ import annotations
|
| 22 |
+
|
| 23 |
+
import hashlib
|
| 24 |
+
import math
|
| 25 |
+
from datetime import datetime, timedelta
|
| 26 |
+
from typing import Optional
|
| 27 |
+
from collections import deque
|
| 28 |
+
|
| 29 |
+
from sqlalchemy.orm import Session
|
| 30 |
+
from sqlalchemy import func
|
| 31 |
+
|
| 32 |
+
from app.core.logger import logger
|
| 33 |
+
from models import PredictionLog
|
| 34 |
+
|
| 35 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 36 |
+
# CONFIG
|
| 37 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 38 |
+
|
| 39 |
+
WINDOW_SIZE = 500 # Rolling window for stats
|
| 40 |
+
DRIFT_THRESHOLD = 0.15 # Alert if mean confidence shifts by this much
|
| 41 |
+
BASELINE_MIN = 50 # Need at least this many predictions before drift detection
|
| 42 |
+
BINS = 10 # Histogram bins for KL-divergence
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 46 |
+
# HELPERS
|
| 47 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 48 |
+
|
| 49 |
+
def _mean(vals: list[float]) -> float:
|
| 50 |
+
return sum(vals) / len(vals) if vals else 0.0
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _stddev(vals: list[float]) -> float:
|
| 54 |
+
if len(vals) < 2:
|
| 55 |
+
return 0.0
|
| 56 |
+
m = _mean(vals)
|
| 57 |
+
variance = sum((x - m) ** 2 for x in vals) / len(vals)
|
| 58 |
+
return math.sqrt(variance)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _histogram(vals: list[float], bins: int = BINS) -> list[float]:
|
| 62 |
+
"""Normalised histogram (probability distribution) over [0, 1]."""
|
| 63 |
+
if not vals:
|
| 64 |
+
return [0.0] * bins
|
| 65 |
+
counts = [0] * bins
|
| 66 |
+
for v in vals:
|
| 67 |
+
idx = min(int(v * bins), bins - 1)
|
| 68 |
+
counts[idx] += 1
|
| 69 |
+
total = sum(counts)
|
| 70 |
+
return [c / total for c in counts]
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _kl_divergence(p: list[float], q: list[float], eps: float = 1e-10) -> float:
|
| 74 |
+
"""KL divergence D(P||Q) — measures distribution shift."""
|
| 75 |
+
kl = 0.0
|
| 76 |
+
for pi, qi in zip(p, q):
|
| 77 |
+
pi = max(pi, eps)
|
| 78 |
+
qi = max(qi, eps)
|
| 79 |
+
kl += pi * math.log(pi / qi)
|
| 80 |
+
return round(kl, 4)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 84 |
+
# DRIFT MONITOR
|
| 85 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 86 |
+
|
| 87 |
+
class DriftMonitor:
|
| 88 |
+
"""
|
| 89 |
+
Singleton service for monitoring model confidence drift.
|
| 90 |
+
Uses DB as persistent store; in-memory cache for speed.
|
| 91 |
+
"""
|
| 92 |
+
|
| 93 |
+
def __init__(self) -> None:
|
| 94 |
+
self._baseline: Optional[list[float]] = None # First BASELINE_MIN scores
|
| 95 |
+
self._baseline_hist: Optional[list[float]] = None
|
| 96 |
+
|
| 97 |
+
# ── LOGGING ───────────────────────────────────────────────────────────────
|
| 98 |
+
|
| 99 |
+
def log_prediction(
|
| 100 |
+
self,
|
| 101 |
+
db: Session,
|
| 102 |
+
*,
|
| 103 |
+
user_id: Optional[int],
|
| 104 |
+
input_text: str,
|
| 105 |
+
confidence: float,
|
| 106 |
+
toxic: bool,
|
| 107 |
+
severity: str,
|
| 108 |
+
source: str,
|
| 109 |
+
) -> None:
|
| 110 |
+
"""Write one prediction event to DB. Called inside every /predict handler."""
|
| 111 |
+
try:
|
| 112 |
+
text_hash = hashlib.sha256(input_text.encode()).hexdigest()[:16]
|
| 113 |
+
log = PredictionLog(
|
| 114 |
+
user_id=user_id,
|
| 115 |
+
text_hash=text_hash,
|
| 116 |
+
text_length=len(input_text),
|
| 117 |
+
confidence=round(confidence, 4),
|
| 118 |
+
toxic=toxic,
|
| 119 |
+
severity=severity,
|
| 120 |
+
source=source,
|
| 121 |
+
)
|
| 122 |
+
db.add(log)
|
| 123 |
+
db.commit()
|
| 124 |
+
except Exception as exc:
|
| 125 |
+
logger.warning(f"[DriftMonitor] Failed to log prediction: {exc}")
|
| 126 |
+
db.rollback()
|
| 127 |
+
|
| 128 |
+
# ── STATS ─────────────────────────────────────────────────────────────────
|
| 129 |
+
|
| 130 |
+
def get_stats(self, db: Session) -> dict:
|
| 131 |
+
"""
|
| 132 |
+
Compute current monitoring stats from the rolling window.
|
| 133 |
+
|
| 134 |
+
Returns:
|
| 135 |
+
{
|
| 136 |
+
"total_predictions": int,
|
| 137 |
+
"window_size": int,
|
| 138 |
+
"mean_confidence": float,
|
| 139 |
+
"stddev_confidence": float,
|
| 140 |
+
"toxic_rate": float, # 0.0–1.0
|
| 141 |
+
"drift_detected": bool,
|
| 142 |
+
"drift_score": float, # KL divergence from baseline
|
| 143 |
+
"alert": str | None,
|
| 144 |
+
"severity_breakdown": {"low": int, "medium": int, "high": int},
|
| 145 |
+
"hourly_volume": list[dict], # last 24h request counts
|
| 146 |
+
}
|
| 147 |
+
"""
|
| 148 |
+
try:
|
| 149 |
+
# Total count
|
| 150 |
+
total = db.query(func.count(PredictionLog.id)).scalar() or 0
|
| 151 |
+
|
| 152 |
+
# Rolling window — last WINDOW_SIZE entries
|
| 153 |
+
recent = (
|
| 154 |
+
db.query(PredictionLog)
|
| 155 |
+
.order_by(PredictionLog.timestamp.desc())
|
| 156 |
+
.limit(WINDOW_SIZE)
|
| 157 |
+
.all()
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
if not recent:
|
| 161 |
+
return self._empty_stats()
|
| 162 |
+
|
| 163 |
+
confidences = [r.confidence for r in recent]
|
| 164 |
+
toxic_flags = [r.toxic for r in recent]
|
| 165 |
+
|
| 166 |
+
mean_conf = round(_mean(confidences), 4)
|
| 167 |
+
std_conf = round(_stddev(confidences), 4)
|
| 168 |
+
toxic_rate = round(sum(toxic_flags) / len(toxic_flags), 4)
|
| 169 |
+
|
| 170 |
+
# Severity breakdown
|
| 171 |
+
sev_counts = {"low": 0, "medium": 0, "high": 0}
|
| 172 |
+
for r in recent:
|
| 173 |
+
sev = r.severity or "low"
|
| 174 |
+
sev_counts[sev] = sev_counts.get(sev, 0) + 1
|
| 175 |
+
|
| 176 |
+
# Drift detection
|
| 177 |
+
drift_score = 0.0
|
| 178 |
+
drift_detected = False
|
| 179 |
+
alert = None
|
| 180 |
+
|
| 181 |
+
if total >= BASELINE_MIN:
|
| 182 |
+
# Baseline: oldest BASELINE_MIN entries
|
| 183 |
+
baseline_rows = (
|
| 184 |
+
db.query(PredictionLog.confidence)
|
| 185 |
+
.order_by(PredictionLog.timestamp.asc())
|
| 186 |
+
.limit(BASELINE_MIN)
|
| 187 |
+
.all()
|
| 188 |
+
)
|
| 189 |
+
baseline_vals = [r.confidence for r in baseline_rows]
|
| 190 |
+
baseline_hist = _histogram(baseline_vals)
|
| 191 |
+
current_hist = _histogram(confidences)
|
| 192 |
+
drift_score = _kl_divergence(current_hist, baseline_hist)
|
| 193 |
+
drift_detected = drift_score > 0.3 # KL threshold
|
| 194 |
+
|
| 195 |
+
if drift_detected:
|
| 196 |
+
alert = (
|
| 197 |
+
f"⚠️ Drift detected! KL-divergence={drift_score:.3f} "
|
| 198 |
+
f"(current mean={mean_conf:.3f} vs baseline mean={round(_mean(baseline_vals), 3)})"
|
| 199 |
+
)
|
| 200 |
+
|
| 201 |
+
# Hourly volume (last 24h)
|
| 202 |
+
hourly = self._get_hourly_volume(db)
|
| 203 |
+
|
| 204 |
+
return {
|
| 205 |
+
"total_predictions": total,
|
| 206 |
+
"window_size": len(recent),
|
| 207 |
+
"mean_confidence": mean_conf,
|
| 208 |
+
"stddev_confidence": std_conf,
|
| 209 |
+
"toxic_rate": toxic_rate,
|
| 210 |
+
"drift_detected": drift_detected,
|
| 211 |
+
"drift_score": round(drift_score, 4),
|
| 212 |
+
"alert": alert,
|
| 213 |
+
"severity_breakdown": sev_counts,
|
| 214 |
+
"hourly_volume": hourly,
|
| 215 |
+
"baseline_size": min(total, BASELINE_MIN),
|
| 216 |
+
}
|
| 217 |
+
|
| 218 |
+
except Exception as exc:
|
| 219 |
+
logger.error(f"[DriftMonitor] get_stats failed: {exc}")
|
| 220 |
+
return self._empty_stats()
|
| 221 |
+
|
| 222 |
+
def get_drift_series(self, db: Session, limit: int = 200) -> list[dict]:
|
| 223 |
+
"""
|
| 224 |
+
Return time-series of confidence scores for the drift trend chart.
|
| 225 |
+
Returns last `limit` predictions ordered by time.
|
| 226 |
+
"""
|
| 227 |
+
try:
|
| 228 |
+
rows = (
|
| 229 |
+
db.query(PredictionLog)
|
| 230 |
+
.order_by(PredictionLog.timestamp.desc())
|
| 231 |
+
.limit(limit)
|
| 232 |
+
.all()
|
| 233 |
+
)
|
| 234 |
+
rows.reverse()
|
| 235 |
+
return [
|
| 236 |
+
{
|
| 237 |
+
"time": r.timestamp.strftime("%H:%M"),
|
| 238 |
+
"confidence": round(r.confidence, 3),
|
| 239 |
+
"toxic": r.toxic,
|
| 240 |
+
"severity": r.severity,
|
| 241 |
+
}
|
| 242 |
+
for r in rows
|
| 243 |
+
]
|
| 244 |
+
except Exception as exc:
|
| 245 |
+
logger.error(f"[DriftMonitor] get_drift_series failed: {exc}")
|
| 246 |
+
return []
|
| 247 |
+
|
| 248 |
+
def _get_hourly_volume(self, db: Session) -> list[dict]:
|
| 249 |
+
"""Count predictions per hour for the last 24 hours."""
|
| 250 |
+
try:
|
| 251 |
+
since = datetime.utcnow() - timedelta(hours=24)
|
| 252 |
+
rows = (
|
| 253 |
+
db.query(PredictionLog)
|
| 254 |
+
.filter(PredictionLog.timestamp >= since)
|
| 255 |
+
.order_by(PredictionLog.timestamp.asc())
|
| 256 |
+
.all()
|
| 257 |
+
)
|
| 258 |
+
# Bucket by hour
|
| 259 |
+
buckets: dict[str, dict] = {}
|
| 260 |
+
for r in rows:
|
| 261 |
+
key = r.timestamp.strftime("%H:00")
|
| 262 |
+
if key not in buckets:
|
| 263 |
+
buckets[key] = {"hour": key, "count": 0, "toxic_count": 0}
|
| 264 |
+
buckets[key]["count"] += 1
|
| 265 |
+
if r.toxic:
|
| 266 |
+
buckets[key]["toxic_count"] += 1
|
| 267 |
+
return list(buckets.values())
|
| 268 |
+
except Exception:
|
| 269 |
+
return []
|
| 270 |
+
|
| 271 |
+
def _empty_stats(self) -> dict:
|
| 272 |
+
return {
|
| 273 |
+
"total_predictions": 0,
|
| 274 |
+
"window_size": 0,
|
| 275 |
+
"mean_confidence": 0.0,
|
| 276 |
+
"stddev_confidence": 0.0,
|
| 277 |
+
"toxic_rate": 0.0,
|
| 278 |
+
"drift_detected": False,
|
| 279 |
+
"drift_score": 0.0,
|
| 280 |
+
"alert": None,
|
| 281 |
+
"severity_breakdown": {"low": 0, "medium": 0, "high": 0},
|
| 282 |
+
"hourly_volume": [],
|
| 283 |
+
"baseline_size": 0,
|
| 284 |
+
}
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
# Module-level singleton
|
| 288 |
+
drift_monitor = DriftMonitor()
|
app/services/model_service.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# backend/app/services/model_service.py
|
| 2 |
+
|
| 3 |
+
"""
|
| 4 |
+
ToxiGuard AI — Unified Model Service
|
| 5 |
+
======================================
|
| 6 |
+
Singleton that loads the best available model at startup:
|
| 7 |
+
|
| 8 |
+
Priority 1: ONNX Runtime — ml/models/toxiguard-deberta/model.onnx
|
| 9 |
+
Priority 2: PyTorch transformer — ml/models/toxiguard-deberta/ (safetensors)
|
| 10 |
+
Priority 3: Legacy TF-IDF + LogReg — abuse_model.joblib (last resort)
|
| 11 |
+
|
| 12 |
+
Expose `model_service` as the single prediction interface across all routes.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import os
|
| 16 |
+
import joblib
|
| 17 |
+
from typing import Optional
|
| 18 |
+
|
| 19 |
+
from app.core.logger import logger
|
| 20 |
+
from app.core.config import settings
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class ModelService:
|
| 24 |
+
"""
|
| 25 |
+
Unified prediction service with automatic model selection.
|
| 26 |
+
|
| 27 |
+
Attributes:
|
| 28 |
+
model_type: One of 'transformer_onnx', 'transformer_pytorch',
|
| 29 |
+
'joblib_legacy', or 'none'.
|
| 30 |
+
"""
|
| 31 |
+
|
| 32 |
+
_MODEL_LABELS = {
|
| 33 |
+
"transformer_onnx": "DeBERTa-v3 via ONNX Runtime (production)",
|
| 34 |
+
"transformer_pytorch": "DeBERTa-v3 via PyTorch (fallback)",
|
| 35 |
+
"joblib_legacy": "TF-IDF + Logistic Regression (legacy)",
|
| 36 |
+
"none": "No model loaded",
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
def __init__(self) -> None:
|
| 40 |
+
self.model_type: str = "none"
|
| 41 |
+
self.transformer = None # TransformerInference | None
|
| 42 |
+
self.legacy_model = None # sklearn Pipeline | None
|
| 43 |
+
self.legacy_encoder = None # LabelEncoder | None
|
| 44 |
+
|
| 45 |
+
self._load()
|
| 46 |
+
|
| 47 |
+
# ── LOADING ───────────────────────────────────────────────────────────────
|
| 48 |
+
|
| 49 |
+
def _load(self) -> None:
|
| 50 |
+
"""Load the best available model at startup."""
|
| 51 |
+
transformer_dir = settings.model_dir
|
| 52 |
+
|
| 53 |
+
if os.path.isdir(transformer_dir):
|
| 54 |
+
self._try_load_transformer(transformer_dir)
|
| 55 |
+
|
| 56 |
+
if self.transformer is None:
|
| 57 |
+
self._try_load_legacy()
|
| 58 |
+
|
| 59 |
+
logger.info(
|
| 60 |
+
f"[ModelService] Active model: {self._MODEL_LABELS.get(self.model_type)}"
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
def _try_load_transformer(self, model_dir: str) -> None:
|
| 64 |
+
try:
|
| 65 |
+
from ml.inference import TransformerInference
|
| 66 |
+
engine = TransformerInference(model_dir)
|
| 67 |
+
if engine.is_ready:
|
| 68 |
+
self.transformer = engine
|
| 69 |
+
self.model_type = (
|
| 70 |
+
"transformer_onnx"
|
| 71 |
+
if engine.backend == "onnx"
|
| 72 |
+
else "transformer_pytorch"
|
| 73 |
+
)
|
| 74 |
+
logger.info(
|
| 75 |
+
f"[ModelService] Transformer loaded | backend={engine.backend}"
|
| 76 |
+
)
|
| 77 |
+
else:
|
| 78 |
+
logger.warning(
|
| 79 |
+
"[ModelService] Transformer directory found but model not ready"
|
| 80 |
+
)
|
| 81 |
+
except Exception as exc:
|
| 82 |
+
logger.warning(f"[ModelService] Transformer load failed: {exc}")
|
| 83 |
+
|
| 84 |
+
def _try_load_legacy(self) -> None:
|
| 85 |
+
base_dir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
|
| 86 |
+
model_path = os.path.join(base_dir, "abuse_model.joblib")
|
| 87 |
+
encoder_path = os.path.join(base_dir, "label_encoder.joblib")
|
| 88 |
+
|
| 89 |
+
try:
|
| 90 |
+
self.legacy_model = joblib.load(model_path)
|
| 91 |
+
self.legacy_encoder = joblib.load(encoder_path)
|
| 92 |
+
self.model_type = "joblib_legacy"
|
| 93 |
+
logger.info("[ModelService] Legacy joblib model loaded")
|
| 94 |
+
except Exception as exc:
|
| 95 |
+
logger.error(f"[ModelService] No model available — {exc}")
|
| 96 |
+
|
| 97 |
+
# ── PREDICTION ────────────────────────────────────────────────────────────
|
| 98 |
+
|
| 99 |
+
def predict(self, text: str) -> Optional[dict]:
|
| 100 |
+
"""
|
| 101 |
+
Unified prediction interface.
|
| 102 |
+
|
| 103 |
+
Args:
|
| 104 |
+
text: Model-preprocessed text (unicode-safe, URLs removed).
|
| 105 |
+
|
| 106 |
+
Returns:
|
| 107 |
+
{
|
| 108 |
+
"toxic": bool,
|
| 109 |
+
"toxicity_probability": float,
|
| 110 |
+
"labels": {...}, # transformer only
|
| 111 |
+
"severity": str,
|
| 112 |
+
"detected_categories": list,
|
| 113 |
+
"source": str,
|
| 114 |
+
}
|
| 115 |
+
or None if no model is loaded.
|
| 116 |
+
"""
|
| 117 |
+
if not text or not text.strip():
|
| 118 |
+
return None
|
| 119 |
+
|
| 120 |
+
if self.transformer:
|
| 121 |
+
result = self.transformer.predict(text)
|
| 122 |
+
if result is not None:
|
| 123 |
+
return result
|
| 124 |
+
logger.warning("[ModelService] Transformer returned None — trying legacy")
|
| 125 |
+
|
| 126 |
+
if self.legacy_model and self.legacy_encoder:
|
| 127 |
+
return self._predict_legacy(text)
|
| 128 |
+
|
| 129 |
+
return None
|
| 130 |
+
|
| 131 |
+
def _predict_legacy(self, text: str) -> dict:
|
| 132 |
+
"""TF-IDF + Logistic Regression prediction."""
|
| 133 |
+
_TOXIC_LABELS = {
|
| 134 |
+
"abusive", "toxic", "severe_toxic",
|
| 135 |
+
"obscene", "threat", "insult", "identity_hate",
|
| 136 |
+
}
|
| 137 |
+
probs = self.legacy_model.predict_proba([text])[0]
|
| 138 |
+
labels = list(self.legacy_encoder.classes_)
|
| 139 |
+
|
| 140 |
+
toxic_probability = max(
|
| 141 |
+
(float(probs[idx]) for idx, lbl in enumerate(labels) if lbl in _TOXIC_LABELS),
|
| 142 |
+
default=0.0,
|
| 143 |
+
)
|
| 144 |
+
predicted_label = self.legacy_encoder.inverse_transform([probs.argmax()])[0]
|
| 145 |
+
label_map = {lbl: round(float(probs[i]), 4) for i, lbl in enumerate(labels)}
|
| 146 |
+
|
| 147 |
+
detected = [lbl for lbl in labels if lbl in _TOXIC_LABELS and label_map[lbl] >= 0.4]
|
| 148 |
+
|
| 149 |
+
return {
|
| 150 |
+
"toxic": predicted_label in _TOXIC_LABELS or toxic_probability >= 0.4,
|
| 151 |
+
"toxicity_probability": round(toxic_probability, 4),
|
| 152 |
+
"label": predicted_label,
|
| 153 |
+
"labels": label_map,
|
| 154 |
+
"severity": (
|
| 155 |
+
"high" if toxic_probability >= 0.85
|
| 156 |
+
else "medium" if toxic_probability >= 0.60
|
| 157 |
+
else "low"
|
| 158 |
+
),
|
| 159 |
+
"detected_categories": detected,
|
| 160 |
+
"source": "legacy_ml",
|
| 161 |
+
}
|
| 162 |
+
|
| 163 |
+
# ── STATUS ────────────────────────────────────────────────────────────────
|
| 164 |
+
|
| 165 |
+
@property
|
| 166 |
+
def status(self) -> dict:
|
| 167 |
+
"""Return model status for the /health endpoint."""
|
| 168 |
+
info = {
|
| 169 |
+
"model_type": self.model_type,
|
| 170 |
+
"description": self._MODEL_LABELS.get(self.model_type, "Unknown"),
|
| 171 |
+
"ready": self.model_type != "none",
|
| 172 |
+
}
|
| 173 |
+
if self.transformer:
|
| 174 |
+
info["backend"] = self.transformer.backend
|
| 175 |
+
return info
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
# Module-level singleton — loaded once at startup
|
| 179 |
+
model_service = ModelService()
|
app/services/overrides.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — Dynamic Rule Overrides Service
|
| 3 |
+
===============================================
|
| 4 |
+
Enables instant hot-patching and updates from active learning feedback.
|
| 5 |
+
|
| 6 |
+
How it works:
|
| 7 |
+
- Whitelist (Allowlist): If a user flags a False Positive, we add the normalized text to the allowlist.
|
| 8 |
+
Next time this exact phrase is checked, it bypasses DeBERTa/LLM and is instantly marked safe.
|
| 9 |
+
- Blacklist (Blocklist): If a user flags a False Negative, we add the normalized text to the blocklist.
|
| 10 |
+
Next time, it's instantly marked toxic.
|
| 11 |
+
- Both allowlist/blocklist are persisted in `data/overrides.json`.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import os
|
| 17 |
+
import json
|
| 18 |
+
import threading
|
| 19 |
+
from typing import Optional
|
| 20 |
+
|
| 21 |
+
from app.core.logger import logger
|
| 22 |
+
from utils.preprocessing import preprocess_for_rules
|
| 23 |
+
|
| 24 |
+
_OVERRIDES_FILE = os.path.join(
|
| 25 |
+
os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
|
| 26 |
+
"data", "overrides.json"
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class DynamicOverrideService:
|
| 31 |
+
def __init__(self) -> None:
|
| 32 |
+
self._lock = threading.Lock()
|
| 33 |
+
self.allowlist: set[str] = set() # Set of normalized texts
|
| 34 |
+
self.blocklist: set[str] = set() # Set of normalized texts
|
| 35 |
+
self._load()
|
| 36 |
+
|
| 37 |
+
def _load(self) -> None:
|
| 38 |
+
try:
|
| 39 |
+
if os.path.exists(_OVERRIDES_FILE):
|
| 40 |
+
with open(_OVERRIDES_FILE, "r") as f:
|
| 41 |
+
data = json.load(f)
|
| 42 |
+
self.allowlist = set(data.get("allowlist", []))
|
| 43 |
+
self.blocklist = set(data.get("blocklist", []))
|
| 44 |
+
logger.info(
|
| 45 |
+
f"[Overrides] Loaded {len(self.allowlist)} allowed and "
|
| 46 |
+
f"{len(self.blocklist)} blocked phrases."
|
| 47 |
+
)
|
| 48 |
+
except Exception as exc:
|
| 49 |
+
logger.warning(f"[Overrides] Could not load overrides: {exc}")
|
| 50 |
+
|
| 51 |
+
def _save(self) -> None:
|
| 52 |
+
try:
|
| 53 |
+
os.makedirs(os.path.dirname(_OVERRIDES_FILE), exist_ok=True)
|
| 54 |
+
with open(_OVERRIDES_FILE, "w") as f:
|
| 55 |
+
json.dump({
|
| 56 |
+
"allowlist": list(self.allowlist),
|
| 57 |
+
"blocklist": list(self.blocklist),
|
| 58 |
+
}, f, indent=2)
|
| 59 |
+
except Exception as exc:
|
| 60 |
+
logger.warning(f"[Overrides] Could not save overrides: {exc}")
|
| 61 |
+
|
| 62 |
+
def add_override(self, feedback_type: str, text: str) -> None:
|
| 63 |
+
"""Add phrase to blocklist or allowlist depending on feedback type."""
|
| 64 |
+
normalized = preprocess_for_rules(text)
|
| 65 |
+
if not normalized:
|
| 66 |
+
return
|
| 67 |
+
|
| 68 |
+
with self._lock:
|
| 69 |
+
if feedback_type == "false_positive":
|
| 70 |
+
# User says safe -> remove from blocklist, add to allowlist
|
| 71 |
+
self.blocklist.discard(normalized)
|
| 72 |
+
self.allowlist.add(normalized)
|
| 73 |
+
logger.info(f"[Overrides] Added to allowlist: '{normalized}'")
|
| 74 |
+
elif feedback_type == "false_negative":
|
| 75 |
+
# User says toxic -> remove from allowlist, add to blocklist
|
| 76 |
+
self.allowlist.discard(normalized)
|
| 77 |
+
self.blocklist.add(normalized)
|
| 78 |
+
logger.info(f"[Overrides] Added to blocklist: '{normalized}'")
|
| 79 |
+
self._save()
|
| 80 |
+
|
| 81 |
+
def check_override(self, text: str) -> Optional[bool]:
|
| 82 |
+
"""
|
| 83 |
+
Check if text has an active override.
|
| 84 |
+
Returns True (block), False (allow), or None (no override).
|
| 85 |
+
"""
|
| 86 |
+
normalized = preprocess_for_rules(text)
|
| 87 |
+
if not normalized:
|
| 88 |
+
return None
|
| 89 |
+
|
| 90 |
+
with self._lock:
|
| 91 |
+
if normalized in self.allowlist:
|
| 92 |
+
return False
|
| 93 |
+
if normalized in self.blocklist:
|
| 94 |
+
return True
|
| 95 |
+
return None
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
# Module-level singleton
|
| 99 |
+
override_service = DynamicOverrideService()
|
app/services/retrainer.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — Model Retraining Pipeline
|
| 3 |
+
=========================================
|
| 4 |
+
Retrains the legacy classifier model using historical baseline data combined
|
| 5 |
+
with newly collected active learning feedback samples.
|
| 6 |
+
|
| 7 |
+
Process:
|
| 8 |
+
1. Retrieve all unreviewed feedback corrections from DB.
|
| 9 |
+
2. Parse baseline dataset (data/sample_data.csv).
|
| 10 |
+
3. Transform user feedback into format compatible with the baseline:
|
| 11 |
+
- If user corrected to TOXIC -> label is 'toxic'
|
| 12 |
+
- If user corrected to SAFE -> label is 'clean'
|
| 13 |
+
4. Append corrections to training data.
|
| 14 |
+
5. Fit the scikit-learn pipeline (TF-IDF + Logistic Regression).
|
| 15 |
+
6. Save updated model checkpoint.
|
| 16 |
+
7. Hot-reload model in ModelService.
|
| 17 |
+
8. Mark feedback samples as reviewed.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
import os
|
| 23 |
+
import joblib
|
| 24 |
+
import pandas as pd
|
| 25 |
+
from datetime import datetime
|
| 26 |
+
from sqlalchemy.orm import Session
|
| 27 |
+
|
| 28 |
+
from app.core.logger import logger
|
| 29 |
+
from app.services.model_service import model_service
|
| 30 |
+
from models import FeedbackSample
|
| 31 |
+
|
| 32 |
+
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
|
| 33 |
+
CSV_PATH = os.path.join(BASE_DIR, "data", "sample_data.csv")
|
| 34 |
+
MODEL_PATH = os.path.join(BASE_DIR, "abuse_model.joblib")
|
| 35 |
+
ENCODER_PATH = os.path.join(BASE_DIR, "label_encoder.joblib")
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def retrain_model(db: Session) -> dict:
|
| 39 |
+
"""
|
| 40 |
+
Retrain the ML classifier with the active learning queue samples.
|
| 41 |
+
"""
|
| 42 |
+
# 1. Fetch unreviewed corrections
|
| 43 |
+
feedback_samples = (
|
| 44 |
+
db.query(FeedbackSample)
|
| 45 |
+
.filter(FeedbackSample.reviewed == False)
|
| 46 |
+
.all()
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
if not feedback_samples:
|
| 50 |
+
return {
|
| 51 |
+
"status": "ignored",
|
| 52 |
+
"message": "No new unreviewed feedback samples found. Model is already up to date.",
|
| 53 |
+
"count": 0
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
logger.info(f"[Retrainer] Starting retraining with {len(feedback_samples)} feedback samples...")
|
| 57 |
+
|
| 58 |
+
# 2. Load baseline CSV
|
| 59 |
+
if not os.path.exists(CSV_PATH):
|
| 60 |
+
raise FileNotFoundError(f"Baseline dataset not found at: {CSV_PATH}")
|
| 61 |
+
|
| 62 |
+
df_base = pd.read_csv(CSV_PATH)
|
| 63 |
+
|
| 64 |
+
# 3. Load model structures
|
| 65 |
+
if not os.path.exists(MODEL_PATH) or not os.path.exists(ENCODER_PATH):
|
| 66 |
+
raise FileNotFoundError("Model or Encoder joblib file not found.")
|
| 67 |
+
|
| 68 |
+
pipeline = joblib.load(MODEL_PATH)
|
| 69 |
+
encoder = joblib.load(ENCODER_PATH)
|
| 70 |
+
|
| 71 |
+
# 4. Process user feedback samples
|
| 72 |
+
new_rows = []
|
| 73 |
+
for sample in feedback_samples:
|
| 74 |
+
label = "toxic" if sample.correct_label else "clean"
|
| 75 |
+
# Map baseline equivalent labels
|
| 76 |
+
new_rows.append({
|
| 77 |
+
"text": sample.input_text,
|
| 78 |
+
"label": label
|
| 79 |
+
})
|
| 80 |
+
|
| 81 |
+
df_new = pd.DataFrame(new_rows)
|
| 82 |
+
df_combined = pd.concat([df_base, df_new], ignore_index=True)
|
| 83 |
+
|
| 84 |
+
# 5. Prepare inputs
|
| 85 |
+
X = df_combined["text"].fillna("").astype(str).values
|
| 86 |
+
y_raw = df_combined["label"].values
|
| 87 |
+
|
| 88 |
+
# Clean label mapping: map toxic classes to 'toxic', others to 'clean'
|
| 89 |
+
TOXIC_LABELS = {
|
| 90 |
+
"abusive", "toxic", "severe_toxic", "obscene", "threat",
|
| 91 |
+
"insult", "identity_hate", "hate", "spam"
|
| 92 |
+
}
|
| 93 |
+
y_mapped = []
|
| 94 |
+
for val in y_raw:
|
| 95 |
+
val_str = str(val).strip().lower()
|
| 96 |
+
if val_str in TOXIC_LABELS:
|
| 97 |
+
y_mapped.append("toxic")
|
| 98 |
+
else:
|
| 99 |
+
y_mapped.append("clean")
|
| 100 |
+
|
| 101 |
+
# Fit encoder target
|
| 102 |
+
y = encoder.transform(y_mapped)
|
| 103 |
+
|
| 104 |
+
# 6. Fit the pipeline
|
| 105 |
+
pipeline.fit(X, y)
|
| 106 |
+
|
| 107 |
+
# 7. Save model checkpoints
|
| 108 |
+
joblib.dump(pipeline, MODEL_PATH)
|
| 109 |
+
logger.info(f"[Retrainer] Model saved successfully to: {MODEL_PATH}")
|
| 110 |
+
|
| 111 |
+
# 8. Hot-reload the prediction service
|
| 112 |
+
model_service._load()
|
| 113 |
+
logger.info("[Retrainer] Unified ModelService hot-reloaded successfully.")
|
| 114 |
+
|
| 115 |
+
# 9. Mark samples as reviewed
|
| 116 |
+
for sample in feedback_samples:
|
| 117 |
+
sample.reviewed = True
|
| 118 |
+
db.commit()
|
| 119 |
+
|
| 120 |
+
return {
|
| 121 |
+
"status": "success",
|
| 122 |
+
"message": f"Successfully retrained legacy model using {len(feedback_samples)} feedback samples. Model is now updated.",
|
| 123 |
+
"samples_retrained": len(feedback_samples),
|
| 124 |
+
"timestamp": datetime.utcnow().isoformat()
|
| 125 |
+
}
|
auth_utils.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from datetime import datetime, timedelta
|
| 3 |
+
from jose import JWTError, jwt
|
| 4 |
+
from fastapi import HTTPException, status
|
| 5 |
+
import secrets
|
| 6 |
+
from dotenv import load_dotenv
|
| 7 |
+
|
| 8 |
+
load_dotenv()
|
| 9 |
+
|
| 10 |
+
SECRET_KEY = os.getenv("JWT_SECRET_KEY", secrets.token_hex(32))
|
| 11 |
+
ALGORITHM = os.getenv("JWT_ALGORITHM", "HS256")
|
| 12 |
+
ACCESS_TOKEN_EXPIRE_MINUTES = int(os.getenv("ACCESS_TOKEN_EXPIRE_MINUTES", "1440"))
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def create_access_token(data: dict):
|
| 16 |
+
to_encode = data.copy()
|
| 17 |
+
expire = datetime.utcnow() + timedelta(
|
| 18 |
+
minutes=ACCESS_TOKEN_EXPIRE_MINUTES
|
| 19 |
+
)
|
| 20 |
+
to_encode.update({"exp": expire})
|
| 21 |
+
|
| 22 |
+
return jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def verify_token(token: str):
|
| 26 |
+
try:
|
| 27 |
+
payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
|
| 28 |
+
return payload
|
| 29 |
+
except JWTError:
|
| 30 |
+
raise HTTPException(
|
| 31 |
+
status_code=status.HTTP_401_UNAUTHORIZED,
|
| 32 |
+
detail="Invalid token"
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
# ✅ API KEY GENERATOR
|
| 37 |
+
|
| 38 |
+
def generate_api_key():
|
| 39 |
+
return secrets.token_hex(32)
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2017 Tom Davidson
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Automated Hate Speech Detection and the Problem of Offensive Language
|
| 2 |
+
Repository for Thomas Davidson, Dana Warmsley, Michael Macy, and Ingmar Weber. 2017. "Automated Hate Speech Detection and the Problem of Offensive Language." ICWSM. You read the paper [here](https://aaai.org/ocs/index.php/ICWSM/ICWSM17/paper/view/15665).
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
# NOTE: This repository is no longer actively maintained. Please do not post issues regarding the compatibility of the existing code with new versions of Python or the packages used.
|
| 6 |
+
|
| 7 |
+
## 2019 NEWS
|
| 8 |
+
We have a new paper on racial bias in this dataset and others, you can read it [here](https://arxiv.org/abs/1905.12516)
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
***WARNING: The data, lexicons, and notebooks all contain content that is racist, sexist, homophobic, and offensive in many other ways.***
|
| 12 |
+
|
| 13 |
+
You can find our labeled data in the `data` directory. We have included them as a pickle file (Python 2.7) and as a CSV. You will also find a notebook in the `src` directory containing Python 2.7 code to replicate our analyses in the paper and a lexicon in the `lexicons` directory that we generated to try to more accurately classify hate speech. The `classifier` directory contains a script, instructions, and the necessary files to run our classifier on new data, a test case is provided.
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
***Please cite our paper in any published work that uses any of these resources.***
|
| 17 |
+
~~~
|
| 18 |
+
@inproceedings{hateoffensive,
|
| 19 |
+
title = {Automated Hate Speech Detection and the Problem of Offensive Language},
|
| 20 |
+
author = {Davidson, Thomas and Warmsley, Dana and Macy, Michael and Weber, Ingmar},
|
| 21 |
+
booktitle = {Proceedings of the 11th International AAAI Conference on Web and Social Media},
|
| 22 |
+
series = {ICWSM '17},
|
| 23 |
+
year = {2017},
|
| 24 |
+
location = {Montreal, Canada},
|
| 25 |
+
pages = {512-515}
|
| 26 |
+
}
|
| 27 |
+
~~~
|
| 28 |
+
|
| 29 |
+
***Contact***
|
| 30 |
+
We would also appreciate it if you could fill out this short [form](https://docs.google.com/forms/d/e/1FAIpQLSdrPNlfVBlqxun2tivzAtsZaOoPC5YYMocn-xscCgeRakLXHg/viewform?usp=pp_url&entry.1506871634&entry.147453066&entry.1390333885&entry.516829772) if you are interested in using our data so we can keep track of how these data are used and get in contact with researchers working on similar problems.
|
| 31 |
+
|
| 32 |
+
If you have any questions please contact `trd54 at cornell dot edu`.
|
data/calibration.json
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"threshold": 0.49,
|
| 3 |
+
"fp_count": 0,
|
| 4 |
+
"fn_count": 1,
|
| 5 |
+
"total_adjustments": 6,
|
| 6 |
+
"history": [
|
| 7 |
+
{
|
| 8 |
+
"ts": "2026-07-21T10:55:49.876075",
|
| 9 |
+
"type": "confirmed_correct",
|
| 10 |
+
"old": 0.5,
|
| 11 |
+
"new": 0.5,
|
| 12 |
+
"confidence_at_time": 0.97
|
| 13 |
+
},
|
| 14 |
+
{
|
| 15 |
+
"ts": "2026-07-22T04:26:14.406519",
|
| 16 |
+
"type": "confirmed_correct",
|
| 17 |
+
"old": 0.5,
|
| 18 |
+
"new": 0.5,
|
| 19 |
+
"confidence_at_time": 0.975
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"ts": "2026-07-22T10:58:25.483774",
|
| 23 |
+
"type": "confirmed_correct",
|
| 24 |
+
"old": 0.5,
|
| 25 |
+
"new": 0.5,
|
| 26 |
+
"confidence_at_time": 0.791
|
| 27 |
+
},
|
| 28 |
+
{
|
| 29 |
+
"ts": "2026-07-25T09:32:48.215232",
|
| 30 |
+
"type": "false_negative",
|
| 31 |
+
"old": 0.5,
|
| 32 |
+
"new": 0.49,
|
| 33 |
+
"confidence_at_time": 0.449
|
| 34 |
+
},
|
| 35 |
+
{
|
| 36 |
+
"ts": "2026-07-25T09:45:59.338829",
|
| 37 |
+
"type": "confirmed_correct",
|
| 38 |
+
"old": 0.49,
|
| 39 |
+
"new": 0.49,
|
| 40 |
+
"confidence_at_time": 0.428
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"ts": "2026-07-25T09:55:28.278200",
|
| 44 |
+
"type": "confirmed_correct",
|
| 45 |
+
"old": 0.49,
|
| 46 |
+
"new": 0.49,
|
| 47 |
+
"confidence_at_time": 0.54
|
| 48 |
+
}
|
| 49 |
+
],
|
| 50 |
+
"last_updated": "2026-07-25T09:55:28.278200"
|
| 51 |
+
}
|
data/overrides.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"allowlist": [],
|
| 3 |
+
"blocklist": [
|
| 4 |
+
"kitna dogla aadmi hai sale tu"
|
| 5 |
+
]
|
| 6 |
+
}
|
data/sample_data.csv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
database.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# database.py
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from dotenv import load_dotenv
|
| 5 |
+
from sqlalchemy import create_engine
|
| 6 |
+
from sqlalchemy.orm import sessionmaker, declarative_base
|
| 7 |
+
|
| 8 |
+
# ✅ load .env file
|
| 9 |
+
load_dotenv()
|
| 10 |
+
|
| 11 |
+
# ✅ fallback for local dev
|
| 12 |
+
DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///./toxiguard.db")
|
| 13 |
+
|
| 14 |
+
# Fix for Render (postgres)
|
| 15 |
+
if DATABASE_URL.startswith("postgres://"):
|
| 16 |
+
DATABASE_URL = DATABASE_URL.replace("postgres://", "postgresql://", 1)
|
| 17 |
+
|
| 18 |
+
# SQLite fix
|
| 19 |
+
connect_args = {"check_same_thread": False} if "sqlite" in DATABASE_URL else {}
|
| 20 |
+
|
| 21 |
+
engine = create_engine(
|
| 22 |
+
DATABASE_URL,
|
| 23 |
+
connect_args=connect_args,
|
| 24 |
+
pool_pre_ping=True
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
SessionLocal = sessionmaker(
|
| 28 |
+
autocommit=False,
|
| 29 |
+
autoflush=False,
|
| 30 |
+
bind=engine
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
Base = declarative_base()
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def get_db():
|
| 37 |
+
db = SessionLocal()
|
| 38 |
+
try:
|
| 39 |
+
yield db
|
| 40 |
+
finally:
|
| 41 |
+
db.close()
|
label_encoder.joblib
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0c1a86a1ca2876f7b1c007ce4dd56ce0c1a76aa8d14a45d1353cb411cc1fe9eb
|
| 3 |
+
size 489
|
ml/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# backend/ml/__init__.py
|
ml/evaluate.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — Model Evaluation Suite
|
| 3 |
+
======================================
|
| 4 |
+
Comprehensive evaluation: F1, AUC-ROC, confusion matrices, threshold analysis.
|
| 5 |
+
|
| 6 |
+
Usage:
|
| 7 |
+
python -m ml.evaluate --model_path ml/models/toxiguard-deberta
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import os, json, argparse
|
| 11 |
+
import numpy as np
|
| 12 |
+
import pandas as pd
|
| 13 |
+
import torch
|
| 14 |
+
from transformers import AutoTokenizer, AutoModelForSequenceClassification
|
| 15 |
+
from sklearn.metrics import (
|
| 16 |
+
f1_score, precision_score, recall_score, roc_auc_score,
|
| 17 |
+
classification_report, confusion_matrix, average_precision_score,
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
|
| 21 |
+
DATA_DIR = os.path.join(BASE_DIR, "data")
|
| 22 |
+
DEFAULT_MODEL = os.path.join(os.path.dirname(__file__), "models", "toxiguard-deberta")
|
| 23 |
+
LABEL_COLUMNS = ["toxic","severe_toxic","obscene","threat","insult","identity_hate"]
|
| 24 |
+
MAX_LENGTH = 256
|
| 25 |
+
|
| 26 |
+
def predict_batch(texts, model, tokenizer, batch_size=32, device="cpu"):
|
| 27 |
+
model.eval(); model.to(device); all_probs = []
|
| 28 |
+
for i in range(0, len(texts), batch_size):
|
| 29 |
+
enc = tokenizer(list(texts[i:i+batch_size]), truncation=True,
|
| 30 |
+
padding="max_length", max_length=MAX_LENGTH, return_tensors="pt").to(device)
|
| 31 |
+
with torch.no_grad():
|
| 32 |
+
all_probs.append(torch.sigmoid(model(**enc).logits).cpu().numpy())
|
| 33 |
+
return np.vstack(all_probs)
|
| 34 |
+
|
| 35 |
+
def evaluate(args):
|
| 36 |
+
print("=" * 60 + "\n📊 ToxiGuard AI — Model Evaluation\n" + "=" * 60)
|
| 37 |
+
tokenizer = AutoTokenizer.from_pretrained(args.model_path)
|
| 38 |
+
model = AutoModelForSequenceClassification.from_pretrained(args.model_path)
|
| 39 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 40 |
+
|
| 41 |
+
train_df = pd.read_csv(os.path.join(DATA_DIR, "train.csv"))
|
| 42 |
+
from sklearn.model_selection import train_test_split
|
| 43 |
+
_, test_df = train_test_split(train_df, test_size=0.1, random_state=42)
|
| 44 |
+
texts = test_df["comment_text"].fillna("").values
|
| 45 |
+
labels = test_df[LABEL_COLUMNS].values.astype(float)
|
| 46 |
+
|
| 47 |
+
probs = predict_batch(texts, model, tokenizer, device=device)
|
| 48 |
+
preds = (probs >= 0.5).astype(int)
|
| 49 |
+
|
| 50 |
+
results = {"f1_macro": round(f1_score(labels, preds, average="macro", zero_division=0), 4),
|
| 51 |
+
"auc_roc": round(roc_auc_score(labels, probs, average="macro"), 4)}
|
| 52 |
+
|
| 53 |
+
print(classification_report(labels, preds, target_names=LABEL_COLUMNS, zero_division=0))
|
| 54 |
+
|
| 55 |
+
for i, col in enumerate(LABEL_COLUMNS):
|
| 56 |
+
cm = confusion_matrix(labels[:,i], preds[:,i])
|
| 57 |
+
tn, fp, fn, tp = cm.ravel() if cm.size == 4 else (0,0,0,0)
|
| 58 |
+
print(f" {col}: TP={tp} FP={fp} FN={fn} TN={tn}")
|
| 59 |
+
|
| 60 |
+
out = os.path.join(args.model_path, "evaluation_results.json")
|
| 61 |
+
with open(out, "w") as f: json.dump(results, f, indent=2)
|
| 62 |
+
print(f"\n✅ Saved to {out}")
|
| 63 |
+
|
| 64 |
+
if __name__ == "__main__":
|
| 65 |
+
p = argparse.ArgumentParser()
|
| 66 |
+
p.add_argument("--model_path", default=DEFAULT_MODEL)
|
| 67 |
+
evaluate(p.parse_args())
|
ml/explainability.py
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — XAI Attribution Engine (Perturbation-Based)
|
| 3 |
+
============================================================
|
| 4 |
+
Computes per-word importance scores using Leave-One-Out (LOO) perturbation.
|
| 5 |
+
Works with ONNX-only model dirs — no PyTorch weights required.
|
| 6 |
+
|
| 7 |
+
Algorithm:
|
| 8 |
+
1. Get baseline confidence score for the full text
|
| 9 |
+
2. For each word, mask it out (replace with [MASK]-equivalent)
|
| 10 |
+
3. Re-run inference on masked text
|
| 11 |
+
4. Attribution score = baseline_confidence - masked_confidence
|
| 12 |
+
→ Positive = word increased toxicity (important)
|
| 13 |
+
→ Negative = word actually reduced toxicity
|
| 14 |
+
5. Normalise scores to [0, 1]
|
| 15 |
+
|
| 16 |
+
Why perturbation over attention?
|
| 17 |
+
- Works with ONNX (no PyTorch needed)
|
| 18 |
+
- Model-agnostic — works with any classifier
|
| 19 |
+
- More faithful: attention ≠ importance (known research finding, Jain & Wallace 2019)
|
| 20 |
+
- This is the method used in production ML (Google, Meta content moderation)
|
| 21 |
+
|
| 22 |
+
Usage:
|
| 23 |
+
from ml.explainability import get_explainer
|
| 24 |
+
explainer = get_explainer(model_dir)
|
| 25 |
+
result = explainer.explain("you are an idiot")
|
| 26 |
+
# {"tokens": [{"token": "you", "score": 0.02, ...}, ...], "top_tokens": [...]}
|
| 27 |
+
"""
|
| 28 |
+
|
| 29 |
+
from __future__ import annotations
|
| 30 |
+
|
| 31 |
+
import re
|
| 32 |
+
import numpy as np
|
| 33 |
+
from typing import Optional
|
| 34 |
+
|
| 35 |
+
from app.core.logger import logger
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 39 |
+
# HELPERS
|
| 40 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 41 |
+
|
| 42 |
+
def _sigmoid(x: np.ndarray) -> np.ndarray:
|
| 43 |
+
return 1.0 / (1.0 + np.exp(-x))
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _tokenize_words(text: str) -> list[str]:
|
| 47 |
+
"""Split text into words preserving order. Handles punctuation."""
|
| 48 |
+
return re.findall(r"\S+", text)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _mask_word(words: list[str], idx: int, mask: str = "") -> str:
|
| 52 |
+
"""Return text with word at idx replaced by mask (or removed)."""
|
| 53 |
+
masked = words[:idx] + ([mask] if mask else []) + words[idx + 1:]
|
| 54 |
+
return " ".join(masked).strip() or "." # fallback so tokenizer doesn't choke
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 58 |
+
# EXPLAINER
|
| 59 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 60 |
+
|
| 61 |
+
class PerturbationExplainer:
|
| 62 |
+
"""
|
| 63 |
+
Leave-One-Out attribution using the ONNX inference session directly.
|
| 64 |
+
No PyTorch weights required.
|
| 65 |
+
"""
|
| 66 |
+
|
| 67 |
+
def __init__(self, model_dir: str) -> None:
|
| 68 |
+
self.model_dir = model_dir
|
| 69 |
+
self.session = None
|
| 70 |
+
self.tokenizer = None
|
| 71 |
+
self._onnx_input_names: list[str] = []
|
| 72 |
+
self._loaded = False
|
| 73 |
+
self._load()
|
| 74 |
+
|
| 75 |
+
def _load(self) -> None:
|
| 76 |
+
import os
|
| 77 |
+
|
| 78 |
+
if not os.path.isdir(self.model_dir):
|
| 79 |
+
logger.warning(
|
| 80 |
+
f"[XAI] PerturbationExplainer load failed: Directory '{self.model_dir}' not found. "
|
| 81 |
+
"Explainability feature will be disabled."
|
| 82 |
+
)
|
| 83 |
+
self._loaded = False
|
| 84 |
+
return
|
| 85 |
+
|
| 86 |
+
onnx_path = os.path.join(self.model_dir, "model.onnx")
|
| 87 |
+
if not os.path.exists(onnx_path):
|
| 88 |
+
logger.warning(
|
| 89 |
+
f"[XAI] PerturbationExplainer load failed: ONNX model file '{onnx_path}' not found. "
|
| 90 |
+
"Explainability feature will be disabled."
|
| 91 |
+
)
|
| 92 |
+
self._loaded = False
|
| 93 |
+
return
|
| 94 |
+
|
| 95 |
+
try:
|
| 96 |
+
import onnxruntime as ort
|
| 97 |
+
from transformers import AutoTokenizer
|
| 98 |
+
|
| 99 |
+
self.tokenizer = AutoTokenizer.from_pretrained(
|
| 100 |
+
self.model_dir, use_fast=True
|
| 101 |
+
)
|
| 102 |
+
self.session = ort.InferenceSession(
|
| 103 |
+
onnx_path,
|
| 104 |
+
providers=["CPUExecutionProvider"],
|
| 105 |
+
)
|
| 106 |
+
self._onnx_input_names = [
|
| 107 |
+
inp.name for inp in self.session.get_inputs()
|
| 108 |
+
]
|
| 109 |
+
self._loaded = True
|
| 110 |
+
logger.info("[XAI] PerturbationExplainer loaded (ONNX)")
|
| 111 |
+
except Exception as exc:
|
| 112 |
+
logger.warning(f"[XAI] PerturbationExplainer load failed: {exc}")
|
| 113 |
+
self._loaded = False
|
| 114 |
+
|
| 115 |
+
@property
|
| 116 |
+
def is_ready(self) -> bool:
|
| 117 |
+
return self._loaded
|
| 118 |
+
|
| 119 |
+
def _infer(self, text: str) -> float:
|
| 120 |
+
"""Run ONNX inference and return max toxicity probability."""
|
| 121 |
+
encoding = self.tokenizer(
|
| 122 |
+
text,
|
| 123 |
+
truncation=True,
|
| 124 |
+
padding="max_length",
|
| 125 |
+
max_length=128,
|
| 126 |
+
return_tensors="np",
|
| 127 |
+
)
|
| 128 |
+
inputs = {
|
| 129 |
+
name: encoding[name].astype(np.int64)
|
| 130 |
+
for name in self._onnx_input_names
|
| 131 |
+
if name in encoding
|
| 132 |
+
}
|
| 133 |
+
logits: np.ndarray = self.session.run(None, inputs)[0]
|
| 134 |
+
probs = _sigmoid(logits)[0] # shape: (num_labels,)
|
| 135 |
+
return float(np.max(probs))
|
| 136 |
+
|
| 137 |
+
def explain(self, text: str, max_words: int = 30) -> Optional[dict]:
|
| 138 |
+
"""
|
| 139 |
+
LOO attribution — score each word by how much removing it
|
| 140 |
+
changes the toxicity confidence.
|
| 141 |
+
|
| 142 |
+
Args:
|
| 143 |
+
text: Raw input text.
|
| 144 |
+
max_words: Limit words to analyze (keeps latency ~1-2s for long texts).
|
| 145 |
+
|
| 146 |
+
Returns:
|
| 147 |
+
{
|
| 148 |
+
"tokens": [{"token": str, "score": float, "is_special": bool}],
|
| 149 |
+
"top_tokens": [{"token": str, "score": float}],
|
| 150 |
+
"method": "leave_one_out_perturbation",
|
| 151 |
+
"num_tokens": int,
|
| 152 |
+
"baseline_confidence": float,
|
| 153 |
+
}
|
| 154 |
+
"""
|
| 155 |
+
if not self.is_ready:
|
| 156 |
+
return None
|
| 157 |
+
|
| 158 |
+
text = text.strip()
|
| 159 |
+
if not text:
|
| 160 |
+
return None
|
| 161 |
+
|
| 162 |
+
try:
|
| 163 |
+
words = _tokenize_words(text)
|
| 164 |
+
|
| 165 |
+
# Cap at max_words to keep latency manageable
|
| 166 |
+
if len(words) > max_words:
|
| 167 |
+
words = words[:max_words]
|
| 168 |
+
|
| 169 |
+
# Step 1: baseline confidence
|
| 170 |
+
baseline = self._infer(text)
|
| 171 |
+
|
| 172 |
+
# Step 2: LOO — remove each word, measure confidence drop
|
| 173 |
+
raw_scores: list[float] = []
|
| 174 |
+
for i in range(len(words)):
|
| 175 |
+
masked_text = _mask_word(words, i)
|
| 176 |
+
masked_conf = self._infer(masked_text)
|
| 177 |
+
# Positive score = removing this word LOWERED toxicity → word is "toxic"
|
| 178 |
+
raw_scores.append(baseline - masked_conf)
|
| 179 |
+
|
| 180 |
+
from utils.abuse_words import get_word_severity
|
| 181 |
+
|
| 182 |
+
# Step 3: Blend raw ML perturbation score with rule-based severity weights
|
| 183 |
+
clamped: list[float] = []
|
| 184 |
+
for i, word in enumerate(words):
|
| 185 |
+
# Clean punctuation for rule lookups
|
| 186 |
+
clean_word = re.sub(r"[^\w]", "", word.lower())
|
| 187 |
+
severity = get_word_severity(clean_word)
|
| 188 |
+
rule_boost = {"critical": 1.0, "high": 0.85, "moderate": 0.55}.get(severity, 0.0)
|
| 189 |
+
|
| 190 |
+
raw_s = max(0.0, raw_scores[i])
|
| 191 |
+
clamped.append(raw_s + rule_boost)
|
| 192 |
+
|
| 193 |
+
max_val = max(clamped) if clamped else 0.0
|
| 194 |
+
|
| 195 |
+
if max_val > 0:
|
| 196 |
+
normalised = [round(s / max_val, 4) for s in clamped]
|
| 197 |
+
else:
|
| 198 |
+
normalised = [0.0] * len(clamped)
|
| 199 |
+
|
| 200 |
+
# Build token list
|
| 201 |
+
tokens = [
|
| 202 |
+
{
|
| 203 |
+
"token": word,
|
| 204 |
+
"score": score,
|
| 205 |
+
"raw_delta": round(raw_scores[i], 4),
|
| 206 |
+
"is_special": False,
|
| 207 |
+
}
|
| 208 |
+
for i, (word, score) in enumerate(zip(words, normalised))
|
| 209 |
+
]
|
| 210 |
+
|
| 211 |
+
# Top 5 most influential words
|
| 212 |
+
top_tokens = sorted(
|
| 213 |
+
[t for t in tokens if not t["is_special"]],
|
| 214 |
+
key=lambda x: x["score"],
|
| 215 |
+
reverse=True,
|
| 216 |
+
)[:5]
|
| 217 |
+
|
| 218 |
+
return {
|
| 219 |
+
"tokens": tokens,
|
| 220 |
+
"top_tokens": top_tokens,
|
| 221 |
+
"method": "leave_one_out_perturbation",
|
| 222 |
+
"num_tokens": len(tokens),
|
| 223 |
+
"baseline_confidence": round(baseline, 4),
|
| 224 |
+
}
|
| 225 |
+
|
| 226 |
+
except Exception as exc:
|
| 227 |
+
logger.error(f"[XAI] explain() failed: {exc}")
|
| 228 |
+
return None
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 232 |
+
# MODULE-LEVEL SINGLETON
|
| 233 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 234 |
+
|
| 235 |
+
_explainer_instance: Optional[PerturbationExplainer] = None
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def get_explainer(model_dir: str) -> PerturbationExplainer:
|
| 239 |
+
global _explainer_instance
|
| 240 |
+
if _explainer_instance is None:
|
| 241 |
+
_explainer_instance = PerturbationExplainer(model_dir)
|
| 242 |
+
return _explainer_instance
|
ml/export_onnx.py
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — ONNX Export Script
|
| 3 |
+
====================================
|
| 4 |
+
Converts a fine-tuned HuggingFace DeBERTa-v3 model to ONNX format
|
| 5 |
+
for production-grade inference with ONNX Runtime (<50ms latency).
|
| 6 |
+
|
| 7 |
+
Prerequisites:
|
| 8 |
+
pip install optimum[onnxruntime] onnx onnxruntime
|
| 9 |
+
|
| 10 |
+
Usage:
|
| 11 |
+
# After training with train_transformer.py:
|
| 12 |
+
python -m ml.export_onnx
|
| 13 |
+
|
| 14 |
+
# Custom model dir:
|
| 15 |
+
python -m ml.export_onnx --model_dir ml/models/toxiguard-deberta
|
| 16 |
+
|
| 17 |
+
What it does:
|
| 18 |
+
1. Loads the fine-tuned PyTorch model + tokenizer
|
| 19 |
+
2. Exports to ONNX via HuggingFace Optimum (traces through the model)
|
| 20 |
+
3. Validates the ONNX output matches PyTorch output within 1e-4 tolerance
|
| 21 |
+
4. Saves model.onnx into the same model directory
|
| 22 |
+
|
| 23 |
+
Output:
|
| 24 |
+
ml/models/toxiguard-deberta/model.onnx
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
import os
|
| 28 |
+
import argparse
|
| 29 |
+
import numpy as np
|
| 30 |
+
|
| 31 |
+
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
|
| 32 |
+
DEFAULT_MODEL_DIR = os.path.join(BASE_DIR, "ml", "models", "toxiguard-deberta")
|
| 33 |
+
|
| 34 |
+
LABEL_COLUMNS = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
|
| 35 |
+
|
| 36 |
+
TEST_SENTENCES = [
|
| 37 |
+
"you are a complete idiot and I hate you",
|
| 38 |
+
"I love rainy days, they are so peaceful",
|
| 39 |
+
"go to hell and never come back",
|
| 40 |
+
"the weather looks beautiful today",
|
| 41 |
+
]
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def export_onnx(model_dir: str) -> None:
|
| 45 |
+
print("=" * 60)
|
| 46 |
+
print("ToxiGuard AI — ONNX Export")
|
| 47 |
+
print("=" * 60)
|
| 48 |
+
print(f"\n[1/4] Loading model from: {model_dir}")
|
| 49 |
+
|
| 50 |
+
# Verify the model directory exists
|
| 51 |
+
if not os.path.isdir(model_dir):
|
| 52 |
+
raise FileNotFoundError(
|
| 53 |
+
f"Model directory not found: {model_dir}\n"
|
| 54 |
+
f"Run `python -m ml.train_transformer` first."
|
| 55 |
+
)
|
| 56 |
+
|
| 57 |
+
try:
|
| 58 |
+
from optimum.onnxruntime import ORTModelForSequenceClassification # type: ignore
|
| 59 |
+
from transformers import AutoTokenizer # type: ignore
|
| 60 |
+
except ImportError:
|
| 61 |
+
raise ImportError(
|
| 62 |
+
"Missing dependencies. Run:\n"
|
| 63 |
+
" pip install optimum[onnxruntime] onnx onnxruntime"
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
onnx_path = os.path.join(model_dir, "model.onnx")
|
| 67 |
+
|
| 68 |
+
print("[2/4] Exporting to ONNX via HuggingFace Optimum...")
|
| 69 |
+
# ORTModelForSequenceClassification handles the entire export pipeline
|
| 70 |
+
ort_model = ORTModelForSequenceClassification.from_pretrained(
|
| 71 |
+
model_dir,
|
| 72 |
+
export=True, # triggers ONNX export from PyTorch weights
|
| 73 |
+
)
|
| 74 |
+
tokenizer = AutoTokenizer.from_pretrained(model_dir, use_fast=True)
|
| 75 |
+
|
| 76 |
+
# Save ONNX model + config to same directory
|
| 77 |
+
ort_model.save_pretrained(model_dir)
|
| 78 |
+
print(f" Saved: {onnx_path}")
|
| 79 |
+
|
| 80 |
+
# ── VALIDATION ──────────────────────────────────────────────────────────
|
| 81 |
+
print("\n[3/4] Validating ONNX output against PyTorch...")
|
| 82 |
+
|
| 83 |
+
import torch # type: ignore
|
| 84 |
+
from transformers import AutoModelForSequenceClassification # type: ignore
|
| 85 |
+
import onnxruntime as ort # type: ignore
|
| 86 |
+
|
| 87 |
+
pt_model = AutoModelForSequenceClassification.from_pretrained(model_dir)
|
| 88 |
+
pt_model.eval()
|
| 89 |
+
|
| 90 |
+
session = ort.InferenceSession(onnx_path, providers=["CPUExecutionProvider"])
|
| 91 |
+
onnx_input_names = [inp.name for inp in session.get_inputs()]
|
| 92 |
+
|
| 93 |
+
max_diff = 0.0
|
| 94 |
+
all_pass = True
|
| 95 |
+
|
| 96 |
+
for sentence in TEST_SENTENCES:
|
| 97 |
+
# PyTorch
|
| 98 |
+
enc_pt = tokenizer(
|
| 99 |
+
sentence, truncation=True, padding="max_length",
|
| 100 |
+
max_length=256, return_tensors="pt"
|
| 101 |
+
)
|
| 102 |
+
enc_pt.pop("token_type_ids", None)
|
| 103 |
+
with torch.no_grad():
|
| 104 |
+
pt_logits = pt_model(**enc_pt).logits.numpy()
|
| 105 |
+
|
| 106 |
+
# ONNX
|
| 107 |
+
enc_np = tokenizer(
|
| 108 |
+
sentence, truncation=True, padding="max_length",
|
| 109 |
+
max_length=256, return_tensors="np"
|
| 110 |
+
)
|
| 111 |
+
onnx_inputs = {
|
| 112 |
+
name: enc_np[name].astype(np.int64)
|
| 113 |
+
for name in onnx_input_names
|
| 114 |
+
if name in enc_np
|
| 115 |
+
}
|
| 116 |
+
onnx_logits = session.run(None, onnx_inputs)[0]
|
| 117 |
+
|
| 118 |
+
diff = float(np.max(np.abs(pt_logits - onnx_logits)))
|
| 119 |
+
max_diff = max(max_diff, diff)
|
| 120 |
+
status = "PASS" if diff < 1e-3 else "WARN"
|
| 121 |
+
if diff >= 1e-3:
|
| 122 |
+
all_pass = False
|
| 123 |
+
print(f" [{status}] '{sentence[:40]}...' | max_diff={diff:.6f}")
|
| 124 |
+
|
| 125 |
+
print(f"\n Overall max diff: {max_diff:.6f} ({'OK' if all_pass else 'WARNING'})")
|
| 126 |
+
|
| 127 |
+
# ── SUMMARY ─────────────────────────────────────────────────────────────
|
| 128 |
+
print("\n[4/4] Export complete!")
|
| 129 |
+
print(f" ONNX model: {onnx_path}")
|
| 130 |
+
print(f" File size: {os.path.getsize(onnx_path) / 1e6:.1f} MB")
|
| 131 |
+
print("\n The server will automatically load ONNX on next startup.")
|
| 132 |
+
print("=" * 60)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
if __name__ == "__main__":
|
| 136 |
+
parser = argparse.ArgumentParser(
|
| 137 |
+
description="ToxiGuard AI — Export PyTorch model to ONNX"
|
| 138 |
+
)
|
| 139 |
+
parser.add_argument(
|
| 140 |
+
"--model_dir",
|
| 141 |
+
type=str,
|
| 142 |
+
default=DEFAULT_MODEL_DIR,
|
| 143 |
+
help="Path to the fine-tuned HuggingFace model directory",
|
| 144 |
+
)
|
| 145 |
+
args = parser.parse_args()
|
| 146 |
+
export_onnx(args.model_dir)
|
ml/inference.py
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — DeBERTa Inference Engine
|
| 3 |
+
=========================================
|
| 4 |
+
Production-grade inference using ONNX Runtime (primary) with PyTorch fallback.
|
| 5 |
+
|
| 6 |
+
ONNX Runtime achieves <50ms latency vs ~200ms with raw PyTorch.
|
| 7 |
+
DeBERTa-v3-base fine-tuned on Jigsaw multi-label toxic comment dataset.
|
| 8 |
+
|
| 9 |
+
Label schema (Jigsaw multi-label):
|
| 10 |
+
["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
|
| 11 |
+
|
| 12 |
+
Usage:
|
| 13 |
+
from ml.inference import TransformerInference
|
| 14 |
+
engine = TransformerInference("ml/models/toxiguard-deberta")
|
| 15 |
+
result = engine.predict("you are an idiot")
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import os
|
| 21 |
+
import numpy as np
|
| 22 |
+
from typing import Optional
|
| 23 |
+
|
| 24 |
+
from app.core.logger import logger
|
| 25 |
+
|
| 26 |
+
# When True, the PyTorch fallback is disabled entirely.
|
| 27 |
+
# Set ONNX_ONLY=true in production (Oracle A1 / Docker) to avoid loading
|
| 28 |
+
# the full 2 GB PyTorch package when only model.onnx is present.
|
| 29 |
+
_ONNX_ONLY: bool = os.getenv("ONNX_ONLY", "false").lower() in ("1", "true", "yes")
|
| 30 |
+
|
| 31 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 32 |
+
# CONSTANTS
|
| 33 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 34 |
+
|
| 35 |
+
LABEL_COLUMNS: list[str] = [
|
| 36 |
+
"toxic",
|
| 37 |
+
"severe_toxic",
|
| 38 |
+
"obscene",
|
| 39 |
+
"threat",
|
| 40 |
+
"insult",
|
| 41 |
+
"identity_hate",
|
| 42 |
+
]
|
| 43 |
+
|
| 44 |
+
MAX_LENGTH: int = 256
|
| 45 |
+
|
| 46 |
+
# Sigmoid: σ(x) = 1 / (1 + e^-x) — applied to raw logits
|
| 47 |
+
def _sigmoid(x: np.ndarray) -> np.ndarray:
|
| 48 |
+
return 1.0 / (1.0 + np.exp(-x))
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 52 |
+
# INFERENCE ENGINE
|
| 53 |
+
# ──────────────────────────────────────────────────────────────────────────────
|
| 54 |
+
|
| 55 |
+
class TransformerInference:
|
| 56 |
+
"""
|
| 57 |
+
ONNX-optimised inference for DeBERTa-v3-base toxicity classifier.
|
| 58 |
+
|
| 59 |
+
Load priority:
|
| 60 |
+
1. ONNX model (model.onnx) — fastest, ~40ms/request
|
| 61 |
+
2. PyTorch model — fallback, ~200ms/request
|
| 62 |
+
|
| 63 |
+
DeBERTa-v3 notes:
|
| 64 |
+
- Uses SentencePiece tokenizer (requires `sentencepiece` package)
|
| 65 |
+
- Does NOT use token_type_ids — excluded from ONNX inputs
|
| 66 |
+
- Multi-label output: sigmoid applied to logits, not softmax
|
| 67 |
+
"""
|
| 68 |
+
|
| 69 |
+
def __init__(self, model_dir: str) -> None:
|
| 70 |
+
self.model_dir: str = model_dir
|
| 71 |
+
self.onnx_path: str = os.path.join(model_dir, "model.onnx")
|
| 72 |
+
self.label_columns: list[str] = LABEL_COLUMNS
|
| 73 |
+
|
| 74 |
+
# Declared upfront to avoid AttributeError on partial load failures
|
| 75 |
+
self.session = None # onnxruntime.InferenceSession | None
|
| 76 |
+
self.tokenizer = None # transformers.PreTrainedTokenizerBase | None
|
| 77 |
+
self.pt_model = None # transformers.AutoModelForSequenceClassification | None
|
| 78 |
+
self._onnx_input_names: list[str] = []
|
| 79 |
+
|
| 80 |
+
self._load()
|
| 81 |
+
|
| 82 |
+
# ── LOADING ───────────────────────────────────────────────────────────────
|
| 83 |
+
|
| 84 |
+
def _load(self) -> None:
|
| 85 |
+
"""Load ONNX model (or PyTorch fallback) and tokenizer."""
|
| 86 |
+
self._load_tokenizer()
|
| 87 |
+
self._load_model()
|
| 88 |
+
|
| 89 |
+
def _load_tokenizer(self) -> None:
|
| 90 |
+
"""Load HuggingFace tokenizer (SentencePiece for DeBERTa-v3)."""
|
| 91 |
+
try:
|
| 92 |
+
from transformers import AutoTokenizer # type: ignore
|
| 93 |
+
self.tokenizer = AutoTokenizer.from_pretrained(
|
| 94 |
+
self.model_dir,
|
| 95 |
+
use_fast=True,
|
| 96 |
+
)
|
| 97 |
+
logger.info(f"[ModelLoad] Tokenizer loaded from {self.model_dir}")
|
| 98 |
+
except Exception as exc:
|
| 99 |
+
logger.error(f"[ModelLoad] Tokenizer load FAILED: {exc}")
|
| 100 |
+
self.tokenizer = None
|
| 101 |
+
|
| 102 |
+
def _load_model(self) -> None:
|
| 103 |
+
"""Attempt ONNX load, fall back to PyTorch (unless ONNX_ONLY=true)."""
|
| 104 |
+
if self._try_load_onnx():
|
| 105 |
+
return
|
| 106 |
+
if _ONNX_ONLY:
|
| 107 |
+
logger.error(
|
| 108 |
+
"[ModelLoad] ONNX load failed and ONNX_ONLY=true — "
|
| 109 |
+
"PyTorch fallback is DISABLED. "
|
| 110 |
+
"Ensure model.onnx exists at: %s",
|
| 111 |
+
self.onnx_path,
|
| 112 |
+
)
|
| 113 |
+
return
|
| 114 |
+
self._try_load_pytorch()
|
| 115 |
+
|
| 116 |
+
def _try_load_onnx(self) -> bool:
|
| 117 |
+
"""Return True if ONNX model loaded successfully."""
|
| 118 |
+
if not os.path.exists(self.onnx_path):
|
| 119 |
+
logger.info("[ModelLoad] No model.onnx found — will use PyTorch")
|
| 120 |
+
return False
|
| 121 |
+
try:
|
| 122 |
+
import onnxruntime as ort # type: ignore
|
| 123 |
+
self.session = ort.InferenceSession(
|
| 124 |
+
self.onnx_path,
|
| 125 |
+
providers=["CPUExecutionProvider"],
|
| 126 |
+
)
|
| 127 |
+
# Cache the actual input names from the ONNX graph
|
| 128 |
+
self._onnx_input_names = [inp.name for inp in self.session.get_inputs()]
|
| 129 |
+
logger.info(
|
| 130 |
+
f"[ModelLoad] ONNX model loaded — inputs: {self._onnx_input_names}"
|
| 131 |
+
)
|
| 132 |
+
return True
|
| 133 |
+
except Exception as exc:
|
| 134 |
+
logger.warning(f"[ModelLoad] ONNX load failed ({exc}), trying PyTorch")
|
| 135 |
+
self.session = None
|
| 136 |
+
return False
|
| 137 |
+
|
| 138 |
+
def _try_load_pytorch(self) -> None:
|
| 139 |
+
"""Load PyTorch model as fallback."""
|
| 140 |
+
try:
|
| 141 |
+
import torch # type: ignore
|
| 142 |
+
from transformers import AutoModelForSequenceClassification # type: ignore
|
| 143 |
+
|
| 144 |
+
self.pt_model = AutoModelForSequenceClassification.from_pretrained(
|
| 145 |
+
self.model_dir,
|
| 146 |
+
num_labels=len(LABEL_COLUMNS),
|
| 147 |
+
)
|
| 148 |
+
self.pt_model.eval()
|
| 149 |
+
logger.info("[ModelLoad] PyTorch model loaded (ONNX fallback)")
|
| 150 |
+
except Exception as exc:
|
| 151 |
+
logger.error(f"[ModelLoad] PyTorch load also FAILED: {exc}")
|
| 152 |
+
self.pt_model = None
|
| 153 |
+
|
| 154 |
+
# ── READINESS ─────────────────────────────────────────────────────────────
|
| 155 |
+
|
| 156 |
+
@property
|
| 157 |
+
def is_ready(self) -> bool:
|
| 158 |
+
"""True if at least a tokenizer and one model backend are loaded."""
|
| 159 |
+
return self.tokenizer is not None and (
|
| 160 |
+
self.session is not None or self.pt_model is not None
|
| 161 |
+
)
|
| 162 |
+
|
| 163 |
+
@property
|
| 164 |
+
def backend(self) -> str:
|
| 165 |
+
if self.session:
|
| 166 |
+
return "onnx"
|
| 167 |
+
if self.pt_model:
|
| 168 |
+
return "pytorch"
|
| 169 |
+
return "none"
|
| 170 |
+
|
| 171 |
+
# ── INFERENCE ─────────────────────────────────────────────────────────────
|
| 172 |
+
|
| 173 |
+
def predict(self, text: str, threshold: float = 0.5) -> Optional[dict]:
|
| 174 |
+
"""
|
| 175 |
+
Run toxicity classification on a single text.
|
| 176 |
+
|
| 177 |
+
Args:
|
| 178 |
+
text: Input text (raw — tokenizer handles normalisation).
|
| 179 |
+
threshold: Probability cutoff for binary label detection.
|
| 180 |
+
|
| 181 |
+
Returns:
|
| 182 |
+
{
|
| 183 |
+
"toxic": bool,
|
| 184 |
+
"toxicity_probability": float, # max across all labels
|
| 185 |
+
"labels": {"toxic": 0.93, ...}, # per-label sigmoid probs
|
| 186 |
+
"severity": "low" | "medium" | "high",
|
| 187 |
+
"detected_categories": ["toxic", "insult"],
|
| 188 |
+
"source": "transformer_onnx" | "transformer_pytorch",
|
| 189 |
+
}
|
| 190 |
+
or None if model is not ready.
|
| 191 |
+
"""
|
| 192 |
+
if not self.is_ready:
|
| 193 |
+
logger.warning("[Inference] Model not ready — returning None")
|
| 194 |
+
return None
|
| 195 |
+
|
| 196 |
+
if not text or not text.strip():
|
| 197 |
+
return None
|
| 198 |
+
|
| 199 |
+
try:
|
| 200 |
+
probs = self._run_inference(text)
|
| 201 |
+
except Exception as exc:
|
| 202 |
+
logger.error(f"[Inference] Runtime error: {exc}")
|
| 203 |
+
return None
|
| 204 |
+
|
| 205 |
+
return self._build_result(probs, threshold)
|
| 206 |
+
|
| 207 |
+
def _run_inference(self, text: str) -> np.ndarray:
|
| 208 |
+
"""Tokenise and run model. Returns sigmoid probabilities array."""
|
| 209 |
+
if self.session:
|
| 210 |
+
return self._infer_onnx(text)
|
| 211 |
+
return self._infer_pytorch(text)
|
| 212 |
+
|
| 213 |
+
def _infer_onnx(self, text: str) -> np.ndarray:
|
| 214 |
+
"""ONNX Runtime inference path."""
|
| 215 |
+
encoding = self.tokenizer(
|
| 216 |
+
text,
|
| 217 |
+
truncation=True,
|
| 218 |
+
padding="max_length",
|
| 219 |
+
max_length=MAX_LENGTH,
|
| 220 |
+
return_tensors="np",
|
| 221 |
+
)
|
| 222 |
+
# DeBERTa-v3 doesn't use token_type_ids — only pass what ONNX graph expects
|
| 223 |
+
inputs = {
|
| 224 |
+
name: encoding[name].astype(np.int64)
|
| 225 |
+
for name in self._onnx_input_names
|
| 226 |
+
if name in encoding
|
| 227 |
+
}
|
| 228 |
+
logits: np.ndarray = self.session.run(None, inputs)[0]
|
| 229 |
+
return _sigmoid(logits)[0] # shape: (num_labels,)
|
| 230 |
+
|
| 231 |
+
def _infer_pytorch(self, text: str) -> np.ndarray:
|
| 232 |
+
"""PyTorch fallback inference path."""
|
| 233 |
+
import torch # type: ignore
|
| 234 |
+
|
| 235 |
+
inputs = self.tokenizer(
|
| 236 |
+
text,
|
| 237 |
+
truncation=True,
|
| 238 |
+
padding="max_length",
|
| 239 |
+
max_length=MAX_LENGTH,
|
| 240 |
+
return_tensors="pt",
|
| 241 |
+
)
|
| 242 |
+
# DeBERTa-v3 doesn't use token_type_ids — remove if present
|
| 243 |
+
inputs.pop("token_type_ids", None)
|
| 244 |
+
|
| 245 |
+
with torch.no_grad():
|
| 246 |
+
logits = self.pt_model(**inputs).logits
|
| 247 |
+
return torch.sigmoid(logits).numpy()[0] # shape: (num_labels,)
|
| 248 |
+
|
| 249 |
+
def _build_result(self, probs: np.ndarray, threshold: float) -> dict:
|
| 250 |
+
"""Convert raw sigmoid probabilities into a structured result dict."""
|
| 251 |
+
label_scores = {
|
| 252 |
+
col: round(float(probs[i]), 4)
|
| 253 |
+
for i, col in enumerate(self.label_columns)
|
| 254 |
+
}
|
| 255 |
+
detected = [col for col, score in label_scores.items() if score >= threshold]
|
| 256 |
+
|
| 257 |
+
# Use max across ALL labels as the top-level toxicity probability
|
| 258 |
+
toxicity_prob = float(np.max(probs))
|
| 259 |
+
|
| 260 |
+
if toxicity_prob >= 0.85:
|
| 261 |
+
severity = "high"
|
| 262 |
+
elif toxicity_prob >= 0.60:
|
| 263 |
+
severity = "medium"
|
| 264 |
+
else:
|
| 265 |
+
severity = "low"
|
| 266 |
+
|
| 267 |
+
source = (
|
| 268 |
+
"transformer_onnx" if self.session else "transformer_pytorch"
|
| 269 |
+
)
|
| 270 |
+
|
| 271 |
+
return {
|
| 272 |
+
"toxic": len(detected) > 0,
|
| 273 |
+
"toxicity_probability": round(toxicity_prob, 4),
|
| 274 |
+
"labels": label_scores,
|
| 275 |
+
"severity": severity,
|
| 276 |
+
"detected_categories": detected,
|
| 277 |
+
"source": source,
|
| 278 |
+
}
|
| 279 |
+
|
| 280 |
+
# ── DEBUG ─────────────────────────────────────────────────────────────────
|
| 281 |
+
|
| 282 |
+
def __repr__(self) -> str:
|
| 283 |
+
return (
|
| 284 |
+
f"TransformerInference("
|
| 285 |
+
f"backend={self.backend!r}, "
|
| 286 |
+
f"ready={self.is_ready}, "
|
| 287 |
+
f"labels={len(self.label_columns)})"
|
| 288 |
+
)
|
ml/models/toxiguard-deberta/added_tokens.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"[MASK]": 128000
|
| 3 |
+
}
|
ml/models/toxiguard-deberta/config.json
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"DebertaV2ForSequenceClassification"
|
| 4 |
+
],
|
| 5 |
+
"attention_probs_dropout_prob": 0.1,
|
| 6 |
+
"dtype": "float32",
|
| 7 |
+
"hidden_act": "gelu",
|
| 8 |
+
"hidden_dropout_prob": 0.1,
|
| 9 |
+
"hidden_size": 768,
|
| 10 |
+
"id2label": {
|
| 11 |
+
"0": "LABEL_0",
|
| 12 |
+
"1": "LABEL_1",
|
| 13 |
+
"2": "LABEL_2",
|
| 14 |
+
"3": "LABEL_3",
|
| 15 |
+
"4": "LABEL_4",
|
| 16 |
+
"5": "LABEL_5"
|
| 17 |
+
},
|
| 18 |
+
"initializer_range": 0.02,
|
| 19 |
+
"intermediate_size": 3072,
|
| 20 |
+
"label2id": {
|
| 21 |
+
"LABEL_0": 0,
|
| 22 |
+
"LABEL_1": 1,
|
| 23 |
+
"LABEL_2": 2,
|
| 24 |
+
"LABEL_3": 3,
|
| 25 |
+
"LABEL_4": 4,
|
| 26 |
+
"LABEL_5": 5
|
| 27 |
+
},
|
| 28 |
+
"layer_norm_eps": 1e-07,
|
| 29 |
+
"legacy": true,
|
| 30 |
+
"max_position_embeddings": 512,
|
| 31 |
+
"max_relative_positions": -1,
|
| 32 |
+
"model_type": "deberta-v2",
|
| 33 |
+
"norm_rel_ebd": "layer_norm",
|
| 34 |
+
"num_attention_heads": 12,
|
| 35 |
+
"num_hidden_layers": 12,
|
| 36 |
+
"pad_token_id": 0,
|
| 37 |
+
"pooler_dropout": 0,
|
| 38 |
+
"pooler_hidden_act": "gelu",
|
| 39 |
+
"pooler_hidden_size": 768,
|
| 40 |
+
"pos_att_type": [
|
| 41 |
+
"p2c",
|
| 42 |
+
"c2p"
|
| 43 |
+
],
|
| 44 |
+
"position_biased_input": false,
|
| 45 |
+
"position_buckets": 256,
|
| 46 |
+
"problem_type": "multi_label_classification",
|
| 47 |
+
"relative_attention": true,
|
| 48 |
+
"share_att_key": true,
|
| 49 |
+
"transformers_version": "4.57.6",
|
| 50 |
+
"type_vocab_size": 0,
|
| 51 |
+
"vocab_size": 128100
|
| 52 |
+
}
|
ml/models/toxiguard-deberta/model.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b75737dbc7bf0bda553c86ce211a9d90106525c6a8ce2653707904ed555719e9
|
| 3 |
+
size 738617302
|
ml/models/toxiguard-deberta/special_tokens_map.json
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "[CLS]",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"cls_token": {
|
| 10 |
+
"content": "[CLS]",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"eos_token": {
|
| 17 |
+
"content": "[SEP]",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
},
|
| 23 |
+
"mask_token": {
|
| 24 |
+
"content": "[MASK]",
|
| 25 |
+
"lstrip": false,
|
| 26 |
+
"normalized": false,
|
| 27 |
+
"rstrip": false,
|
| 28 |
+
"single_word": false
|
| 29 |
+
},
|
| 30 |
+
"pad_token": {
|
| 31 |
+
"content": "[PAD]",
|
| 32 |
+
"lstrip": false,
|
| 33 |
+
"normalized": false,
|
| 34 |
+
"rstrip": false,
|
| 35 |
+
"single_word": false
|
| 36 |
+
},
|
| 37 |
+
"sep_token": {
|
| 38 |
+
"content": "[SEP]",
|
| 39 |
+
"lstrip": false,
|
| 40 |
+
"normalized": false,
|
| 41 |
+
"rstrip": false,
|
| 42 |
+
"single_word": false
|
| 43 |
+
},
|
| 44 |
+
"unk_token": {
|
| 45 |
+
"content": "[UNK]",
|
| 46 |
+
"lstrip": false,
|
| 47 |
+
"normalized": true,
|
| 48 |
+
"rstrip": false,
|
| 49 |
+
"single_word": false
|
| 50 |
+
}
|
| 51 |
+
}
|
ml/models/toxiguard-deberta/spm.model
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
|
| 3 |
+
size 2464616
|
ml/models/toxiguard-deberta/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
ml/models/toxiguard-deberta/tokenizer_config.json
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"added_tokens_decoder": {
|
| 3 |
+
"0": {
|
| 4 |
+
"content": "[PAD]",
|
| 5 |
+
"lstrip": false,
|
| 6 |
+
"normalized": false,
|
| 7 |
+
"rstrip": false,
|
| 8 |
+
"single_word": false,
|
| 9 |
+
"special": true
|
| 10 |
+
},
|
| 11 |
+
"1": {
|
| 12 |
+
"content": "[CLS]",
|
| 13 |
+
"lstrip": false,
|
| 14 |
+
"normalized": false,
|
| 15 |
+
"rstrip": false,
|
| 16 |
+
"single_word": false,
|
| 17 |
+
"special": true
|
| 18 |
+
},
|
| 19 |
+
"2": {
|
| 20 |
+
"content": "[SEP]",
|
| 21 |
+
"lstrip": false,
|
| 22 |
+
"normalized": false,
|
| 23 |
+
"rstrip": false,
|
| 24 |
+
"single_word": false,
|
| 25 |
+
"special": true
|
| 26 |
+
},
|
| 27 |
+
"3": {
|
| 28 |
+
"content": "[UNK]",
|
| 29 |
+
"lstrip": false,
|
| 30 |
+
"normalized": true,
|
| 31 |
+
"rstrip": false,
|
| 32 |
+
"single_word": false,
|
| 33 |
+
"special": true
|
| 34 |
+
},
|
| 35 |
+
"128000": {
|
| 36 |
+
"content": "[MASK]",
|
| 37 |
+
"lstrip": false,
|
| 38 |
+
"normalized": false,
|
| 39 |
+
"rstrip": false,
|
| 40 |
+
"single_word": false,
|
| 41 |
+
"special": true
|
| 42 |
+
}
|
| 43 |
+
},
|
| 44 |
+
"bos_token": "[CLS]",
|
| 45 |
+
"clean_up_tokenization_spaces": false,
|
| 46 |
+
"cls_token": "[CLS]",
|
| 47 |
+
"do_lower_case": false,
|
| 48 |
+
"eos_token": "[SEP]",
|
| 49 |
+
"extra_special_tokens": {},
|
| 50 |
+
"mask_token": "[MASK]",
|
| 51 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 52 |
+
"pad_token": "[PAD]",
|
| 53 |
+
"sep_token": "[SEP]",
|
| 54 |
+
"sp_model_kwargs": {},
|
| 55 |
+
"split_by_punct": false,
|
| 56 |
+
"tokenizer_class": "DebertaV2Tokenizer",
|
| 57 |
+
"unk_token": "[UNK]",
|
| 58 |
+
"vocab_type": "spm"
|
| 59 |
+
}
|
ml/train_transformer.py
ADDED
|
@@ -0,0 +1,326 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ToxiGuard AI — Transformer Model Training Pipeline
|
| 3 |
+
===================================================
|
| 4 |
+
Fine-tunes DeBERTa-v3-base for multi-label toxic comment classification.
|
| 5 |
+
|
| 6 |
+
Usage:
|
| 7 |
+
python -m ml.train_transformer # full training
|
| 8 |
+
python -m ml.train_transformer --epochs 3 --batch_size 16
|
| 9 |
+
python -m ml.train_transformer --sample_size 20000 # quick test run
|
| 10 |
+
python -m ml.train_transformer --export_onnx # also export to ONNX
|
| 11 |
+
|
| 12 |
+
Requirements:
|
| 13 |
+
pip install transformers datasets torch accelerate scikit-learn sentencepiece
|
| 14 |
+
|
| 15 |
+
Output:
|
| 16 |
+
ml/models/toxiguard-deberta/ — HuggingFace model + tokenizer
|
| 17 |
+
ml/models/training_metrics.json — Evaluation metrics
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
import os
|
| 21 |
+
import json
|
| 22 |
+
import argparse
|
| 23 |
+
import numpy as np
|
| 24 |
+
from datetime import datetime
|
| 25 |
+
|
| 26 |
+
import torch
|
| 27 |
+
from torch.utils.data import Dataset
|
| 28 |
+
|
| 29 |
+
from transformers import (
|
| 30 |
+
AutoTokenizer,
|
| 31 |
+
AutoModelForSequenceClassification,
|
| 32 |
+
TrainingArguments,
|
| 33 |
+
Trainer,
|
| 34 |
+
EarlyStoppingCallback,
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
from sklearn.metrics import (
|
| 38 |
+
f1_score,
|
| 39 |
+
precision_score,
|
| 40 |
+
recall_score,
|
| 41 |
+
roc_auc_score,
|
| 42 |
+
classification_report,
|
| 43 |
+
)
|
| 44 |
+
from sklearn.model_selection import train_test_split
|
| 45 |
+
|
| 46 |
+
import pandas as pd
|
| 47 |
+
|
| 48 |
+
# ── CONFIGURATION ──────────────────────────────────────────────────────────────
|
| 49 |
+
|
| 50 |
+
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
|
| 51 |
+
DATA_DIR = os.path.join(BASE_DIR, "data")
|
| 52 |
+
MODEL_OUTPUT_DIR = os.path.join(os.path.dirname(__file__), "models", "toxiguard-deberta")
|
| 53 |
+
|
| 54 |
+
# Model registry — ranked by accuracy vs speed tradeoff
|
| 55 |
+
MODEL_REGISTRY = {
|
| 56 |
+
"deberta-v3-base": "microsoft/deberta-v3-base", # default — best accuracy
|
| 57 |
+
"deberta-v3-small": "microsoft/deberta-v3-small", # faster, slightly lower accuracy
|
| 58 |
+
"roberta-base": "roberta-base",
|
| 59 |
+
"bert-base": "bert-base-uncased",
|
| 60 |
+
"distilbert": "distilbert-base-uncased",
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
# Jigsaw multi-label columns
|
| 64 |
+
LABEL_COLUMNS = [
|
| 65 |
+
"toxic",
|
| 66 |
+
"severe_toxic",
|
| 67 |
+
"obscene",
|
| 68 |
+
"threat",
|
| 69 |
+
"insult",
|
| 70 |
+
"identity_hate",
|
| 71 |
+
]
|
| 72 |
+
|
| 73 |
+
MAX_LENGTH = 256
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
# ── DATASET ────────────────────────────────────────────────────────────────────
|
| 77 |
+
|
| 78 |
+
class ToxicCommentDataset(Dataset):
|
| 79 |
+
"""PyTorch Dataset for multi-label toxic comment classification."""
|
| 80 |
+
|
| 81 |
+
def __init__(
|
| 82 |
+
self,
|
| 83 |
+
texts: np.ndarray,
|
| 84 |
+
labels: np.ndarray,
|
| 85 |
+
tokenizer,
|
| 86 |
+
max_length: int = MAX_LENGTH,
|
| 87 |
+
) -> None:
|
| 88 |
+
self.texts = texts
|
| 89 |
+
self.labels = labels
|
| 90 |
+
self.tokenizer = tokenizer
|
| 91 |
+
self.max_length = max_length
|
| 92 |
+
|
| 93 |
+
def __len__(self) -> int:
|
| 94 |
+
return len(self.texts)
|
| 95 |
+
|
| 96 |
+
def __getitem__(self, idx: int) -> dict:
|
| 97 |
+
text = str(self.texts[idx])
|
| 98 |
+
encoding = self.tokenizer(
|
| 99 |
+
text,
|
| 100 |
+
truncation=True,
|
| 101 |
+
padding="max_length",
|
| 102 |
+
max_length=self.max_length,
|
| 103 |
+
return_tensors="pt",
|
| 104 |
+
)
|
| 105 |
+
item = {
|
| 106 |
+
"input_ids": encoding["input_ids"].squeeze(),
|
| 107 |
+
"attention_mask": encoding["attention_mask"].squeeze(),
|
| 108 |
+
"labels": torch.tensor(self.labels[idx], dtype=torch.float),
|
| 109 |
+
}
|
| 110 |
+
# DeBERTa-v3 does not use token_type_ids — skip if present
|
| 111 |
+
if "token_type_ids" in encoding:
|
| 112 |
+
item["token_type_ids"] = encoding["token_type_ids"].squeeze()
|
| 113 |
+
return item
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
# ── METRICS ────────────────────────────────────────────────────────────────────
|
| 117 |
+
|
| 118 |
+
def compute_metrics(eval_pred) -> dict:
|
| 119 |
+
"""Compute multi-label classification metrics during training."""
|
| 120 |
+
predictions, labels = eval_pred
|
| 121 |
+
probs = torch.sigmoid(torch.tensor(predictions)).numpy()
|
| 122 |
+
preds = (probs >= 0.5).astype(int)
|
| 123 |
+
|
| 124 |
+
f1_macro = f1_score(labels, preds, average="macro", zero_division=0)
|
| 125 |
+
f1_micro = f1_score(labels, preds, average="micro", zero_division=0)
|
| 126 |
+
precision = precision_score(labels, preds, average="macro", zero_division=0)
|
| 127 |
+
recall = recall_score(labels, preds, average="macro", zero_division=0)
|
| 128 |
+
|
| 129 |
+
try:
|
| 130 |
+
auc_roc = roc_auc_score(labels, probs, average="macro")
|
| 131 |
+
except ValueError:
|
| 132 |
+
auc_roc = 0.0
|
| 133 |
+
|
| 134 |
+
per_class_f1 = f1_score(labels, preds, average=None, zero_division=0)
|
| 135 |
+
|
| 136 |
+
metrics = {
|
| 137 |
+
"f1_macro": f1_macro,
|
| 138 |
+
"f1_micro": f1_micro,
|
| 139 |
+
"precision_macro": precision,
|
| 140 |
+
"recall_macro": recall,
|
| 141 |
+
"auc_roc_macro": auc_roc,
|
| 142 |
+
}
|
| 143 |
+
for i, col in enumerate(LABEL_COLUMNS):
|
| 144 |
+
metrics[f"f1_{col}"] = float(per_class_f1[i]) if i < len(per_class_f1) else 0.0
|
| 145 |
+
|
| 146 |
+
return metrics
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
# ── DATA LOADING ───────────────────────────────────────────────────────────────
|
| 150 |
+
|
| 151 |
+
def load_jigsaw_data(data_dir: str, sample_size: int | None = None):
|
| 152 |
+
"""Load Jigsaw Toxic Comment dataset from CSV."""
|
| 153 |
+
train_path = os.path.join(data_dir, "train.csv")
|
| 154 |
+
if not os.path.exists(train_path):
|
| 155 |
+
raise FileNotFoundError(
|
| 156 |
+
f"Training data not found at {train_path}\n"
|
| 157 |
+
f"Download from: https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge"
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
print(f"[Data] Loading from {train_path}...")
|
| 161 |
+
df = pd.read_csv(train_path)
|
| 162 |
+
|
| 163 |
+
if sample_size and sample_size < len(df):
|
| 164 |
+
print(f"[Data] Sampling {sample_size} rows (full dataset = {len(df)})")
|
| 165 |
+
df = df.sample(n=sample_size, random_state=42)
|
| 166 |
+
|
| 167 |
+
texts = df["comment_text"].fillna("").values
|
| 168 |
+
labels = df[LABEL_COLUMNS].values.astype(float)
|
| 169 |
+
|
| 170 |
+
print(f"[Data] Loaded {len(df)} samples")
|
| 171 |
+
for col in LABEL_COLUMNS:
|
| 172 |
+
print(f" {col}: {df[col].sum()} ({df[col].mean()*100:.1f}%)")
|
| 173 |
+
|
| 174 |
+
return texts, labels
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
# ── TRAINING PIPELINE ──────────────────────────────────────────────────────────
|
| 178 |
+
|
| 179 |
+
def train(args) -> dict:
|
| 180 |
+
"""Full DeBERTa fine-tuning pipeline."""
|
| 181 |
+
print("=" * 60)
|
| 182 |
+
print("ToxiGuard AI — DeBERTa Training Pipeline")
|
| 183 |
+
print("=" * 60)
|
| 184 |
+
|
| 185 |
+
model_name = MODEL_REGISTRY.get(args.model, args.model)
|
| 186 |
+
print(f"\n[Model] Base: {model_name}")
|
| 187 |
+
print(f"[Device] {'GPU (CUDA)' if torch.cuda.is_available() else 'CPU'}")
|
| 188 |
+
|
| 189 |
+
# ── TOKENIZER & MODEL ──────────────────────────────────────────────────
|
| 190 |
+
tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
|
| 191 |
+
model = AutoModelForSequenceClassification.from_pretrained(
|
| 192 |
+
model_name,
|
| 193 |
+
num_labels=len(LABEL_COLUMNS),
|
| 194 |
+
problem_type="multi_label_classification",
|
| 195 |
+
ignore_mismatched_sizes=True,
|
| 196 |
+
)
|
| 197 |
+
|
| 198 |
+
# ── DATA ───────────────────────────────────────────────────────────────
|
| 199 |
+
texts, labels = load_jigsaw_data(DATA_DIR, sample_size=args.sample_size)
|
| 200 |
+
|
| 201 |
+
train_texts, val_texts, train_labels, val_labels = train_test_split(
|
| 202 |
+
texts, labels, test_size=0.1, random_state=42, shuffle=True
|
| 203 |
+
)
|
| 204 |
+
print(f"\n[Split] Train: {len(train_texts)} | Val: {len(val_texts)}")
|
| 205 |
+
|
| 206 |
+
train_dataset = ToxicCommentDataset(train_texts, train_labels, tokenizer)
|
| 207 |
+
val_dataset = ToxicCommentDataset(val_texts, val_labels, tokenizer)
|
| 208 |
+
|
| 209 |
+
# ── TRAINING ARGS ──────────────────────────────────────────────────────
|
| 210 |
+
# NOTE: `eval_strategy` replaces deprecated `evaluation_strat` (HF >= 4.41)
|
| 211 |
+
training_args = TrainingArguments(
|
| 212 |
+
output_dir=MODEL_OUTPUT_DIR,
|
| 213 |
+
num_train_epochs=args.epochs,
|
| 214 |
+
per_device_train_batch_size=args.batch_size,
|
| 215 |
+
per_device_eval_batch_size=args.batch_size * 2,
|
| 216 |
+
learning_rate=args.learning_rate,
|
| 217 |
+
weight_decay=0.01,
|
| 218 |
+
warmup_ratio=0.1,
|
| 219 |
+
eval_strategy="epoch", # Fixed: was `evaluation_strat` (deprecated)
|
| 220 |
+
save_strategy="epoch",
|
| 221 |
+
load_best_model_at_end=True,
|
| 222 |
+
metric_for_best_model="f1_macro",
|
| 223 |
+
greater_is_better=True,
|
| 224 |
+
logging_dir=os.path.join(MODEL_OUTPUT_DIR, "logs"),
|
| 225 |
+
logging_steps=50,
|
| 226 |
+
save_total_limit=2,
|
| 227 |
+
save_safetensors=True,
|
| 228 |
+
fp16=torch.cuda.is_available(),
|
| 229 |
+
dataloader_num_workers=2,
|
| 230 |
+
report_to=["mlflow"] if args.mlflow else [],
|
| 231 |
+
run_name=f"toxiguard-{args.model}-{datetime.now().strftime('%Y%m%d-%H%M')}",
|
| 232 |
+
)
|
| 233 |
+
|
| 234 |
+
# ── TRAINER ────────────────────────────────────────────────────────────
|
| 235 |
+
trainer = Trainer(
|
| 236 |
+
model=model,
|
| 237 |
+
args=training_args,
|
| 238 |
+
train_dataset=train_dataset,
|
| 239 |
+
eval_dataset=val_dataset,
|
| 240 |
+
compute_metrics=compute_metrics,
|
| 241 |
+
callbacks=[EarlyStoppingCallback(early_stopping_patience=2)],
|
| 242 |
+
)
|
| 243 |
+
|
| 244 |
+
# ── TRAIN ──────────────────────────────────────────────────────────────
|
| 245 |
+
print(f"\n[Train] Starting — epochs={args.epochs} | batch={args.batch_size} | lr={args.learning_rate}")
|
| 246 |
+
trainer.train()
|
| 247 |
+
|
| 248 |
+
# ── EVALUATE ───────────────────────────────────────────────────────────
|
| 249 |
+
print("\n[Eval] Running final evaluation...")
|
| 250 |
+
eval_results = trainer.evaluate()
|
| 251 |
+
|
| 252 |
+
print("\n" + "=" * 60)
|
| 253 |
+
print("EVALUATION RESULTS")
|
| 254 |
+
print("=" * 60)
|
| 255 |
+
for key, value in sorted(eval_results.items()):
|
| 256 |
+
if isinstance(value, float):
|
| 257 |
+
print(f" {key}: {value:.4f}")
|
| 258 |
+
|
| 259 |
+
# ── SAVE MODEL ─────────────────────────────────────────────────────────
|
| 260 |
+
print(f"\n[Save] Saving to {MODEL_OUTPUT_DIR}...")
|
| 261 |
+
trainer.save_model(MODEL_OUTPUT_DIR)
|
| 262 |
+
tokenizer.save_pretrained(MODEL_OUTPUT_DIR)
|
| 263 |
+
|
| 264 |
+
# Save training metrics JSON
|
| 265 |
+
metrics_path = os.path.join(MODEL_OUTPUT_DIR, "training_metrics.json")
|
| 266 |
+
metrics_data = {
|
| 267 |
+
"model": model_name,
|
| 268 |
+
"trained_at": datetime.now().isoformat(),
|
| 269 |
+
"epochs": args.epochs,
|
| 270 |
+
"batch_size": args.batch_size,
|
| 271 |
+
"learning_rate": args.learning_rate,
|
| 272 |
+
"train_samples": len(train_texts),
|
| 273 |
+
"val_samples": len(val_texts),
|
| 274 |
+
"label_columns": LABEL_COLUMNS,
|
| 275 |
+
"metrics": {
|
| 276 |
+
k: round(v, 4) if isinstance(v, float) else v
|
| 277 |
+
for k, v in eval_results.items()
|
| 278 |
+
},
|
| 279 |
+
}
|
| 280 |
+
with open(metrics_path, "w") as f:
|
| 281 |
+
json.dump(metrics_data, f, indent=2)
|
| 282 |
+
print(f"[Save] Metrics saved to {metrics_path}")
|
| 283 |
+
|
| 284 |
+
# ── PER-CLASS REPORT ────────────────────────────────────────────────────
|
| 285 |
+
print("\n[Report] Per-class Classification Report:")
|
| 286 |
+
val_preds = trainer.predict(val_dataset)
|
| 287 |
+
probs = torch.sigmoid(torch.tensor(val_preds.predictions)).numpy()
|
| 288 |
+
preds = (probs >= 0.5).astype(int)
|
| 289 |
+
print(classification_report(
|
| 290 |
+
val_labels, preds,
|
| 291 |
+
target_names=LABEL_COLUMNS,
|
| 292 |
+
zero_division=0,
|
| 293 |
+
))
|
| 294 |
+
|
| 295 |
+
# ── ONNX EXPORT (optional) ──────────────────────────────────────────────
|
| 296 |
+
if args.export_onnx:
|
| 297 |
+
print("\n[ONNX] Exporting to ONNX Runtime format...")
|
| 298 |
+
try:
|
| 299 |
+
from ml.export_onnx import export_onnx
|
| 300 |
+
export_onnx(MODEL_OUTPUT_DIR)
|
| 301 |
+
except ImportError as exc:
|
| 302 |
+
print(f"[ONNX] Export skipped — missing packages: {exc}")
|
| 303 |
+
print(" Run: pip install optimum[onnxruntime] onnx")
|
| 304 |
+
|
| 305 |
+
print("\n[Done] Training complete!")
|
| 306 |
+
return eval_results
|
| 307 |
+
|
| 308 |
+
# ── CLI ────────────────────────────────────────────────────────────────────────
|
| 309 |
+
|
| 310 |
+
if __name__ == "__main__":
|
| 311 |
+
parser = argparse.ArgumentParser(
|
| 312 |
+
description="ToxiGuard AI — Fine-tune DeBERTa for toxicity classification"
|
| 313 |
+
)
|
| 314 |
+
parser.add_argument(
|
| 315 |
+
"--model", type=str, default="deberta-v3-base",
|
| 316 |
+
choices=list(MODEL_REGISTRY.keys()),
|
| 317 |
+
help="Base model to fine-tune (default: deberta-v3-base)",
|
| 318 |
+
)
|
| 319 |
+
parser.add_argument("--epochs", type=int, default=3, help="Training epochs")
|
| 320 |
+
parser.add_argument("--batch_size", type=int, default=16, help="Batch size per device")
|
| 321 |
+
parser.add_argument("--learning_rate", type=float, default=2e-5, help="Learning rate")
|
| 322 |
+
parser.add_argument("--sample_size", type=int, default=None, help="Dataset size limit (for testing)")
|
| 323 |
+
parser.add_argument("--mlflow", action="store_true", help="Enable MLflow tracking")
|
| 324 |
+
parser.add_argument("--export_onnx", action="store_true", help="Export to ONNX after training")
|
| 325 |
+
|
| 326 |
+
train(parser.parse_args())
|
models.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# models.py
|
| 2 |
+
|
| 3 |
+
from sqlalchemy import Column, Integer, String, DateTime, Float, Boolean, Text
|
| 4 |
+
from datetime import datetime
|
| 5 |
+
|
| 6 |
+
from database import Base
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class User(Base):
|
| 10 |
+
__tablename__ = "users"
|
| 11 |
+
|
| 12 |
+
id = Column(Integer, primary_key=True, index=True)
|
| 13 |
+
|
| 14 |
+
email = Column(String, unique=True, index=True, nullable=False)
|
| 15 |
+
|
| 16 |
+
password_hash = Column(String, nullable=False)
|
| 17 |
+
|
| 18 |
+
created_at = Column(DateTime, default=datetime.utcnow)
|
| 19 |
+
|
| 20 |
+
# SaaS fields
|
| 21 |
+
api_key = Column(String, unique=True, index=True, nullable=True)
|
| 22 |
+
|
| 23 |
+
plan = Column(String, default="free")
|
| 24 |
+
|
| 25 |
+
usage_count = Column(Integer, default=0)
|
| 26 |
+
|
| 27 |
+
# Tracks the last time the user made an authenticated API request.
|
| 28 |
+
# Useful for analytics, stale account cleanup, and pro dashboard UX.
|
| 29 |
+
last_used = Column(DateTime, nullable=True, default=None)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class PredictionLog(Base):
|
| 33 |
+
"""
|
| 34 |
+
Stores every /predict response for drift monitoring and analytics.
|
| 35 |
+
Used by DriftMonitor to compute rolling confidence stats.
|
| 36 |
+
"""
|
| 37 |
+
__tablename__ = "prediction_logs"
|
| 38 |
+
|
| 39 |
+
id = Column(Integer, primary_key=True, index=True)
|
| 40 |
+
user_id = Column(Integer, nullable=True, index=True)
|
| 41 |
+
text_hash = Column(String, nullable=True) # SHA256 of input (no PII)
|
| 42 |
+
text_length = Column(Integer, nullable=True)
|
| 43 |
+
confidence = Column(Float, nullable=False) # 0.0–1.0 toxicity probability
|
| 44 |
+
toxic = Column(Boolean, nullable=False)
|
| 45 |
+
severity = Column(String, nullable=True) # low/medium/high
|
| 46 |
+
source = Column(String, nullable=True) # transformer_onnx / rules / etc.
|
| 47 |
+
timestamp = Column(DateTime, default=datetime.utcnow, index=True)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
class FeedbackSample(Base):
|
| 51 |
+
"""
|
| 52 |
+
Active Learning feedback table.
|
| 53 |
+
Users flag incorrect predictions (FP/FN) to build a correction queue.
|
| 54 |
+
"""
|
| 55 |
+
__tablename__ = "feedback_samples"
|
| 56 |
+
|
| 57 |
+
id = Column(Integer, primary_key=True, index=True)
|
| 58 |
+
user_id = Column(Integer, nullable=True, index=True)
|
| 59 |
+
input_text = Column(Text, nullable=False) # original text
|
| 60 |
+
predicted_toxic = Column(Boolean, nullable=False) # what model said
|
| 61 |
+
correct_label = Column(Boolean, nullable=False) # what user says is right
|
| 62 |
+
feedback_type = Column(String, nullable=False) # "false_positive" / "false_negative"
|
| 63 |
+
confidence_at_time = Column(Float, nullable=True) # model confidence when flagged
|
| 64 |
+
notes = Column(Text, nullable=True) # optional user comment
|
| 65 |
+
abusive_words = Column(Text, nullable=True) # specific abusive words identified by user
|
| 66 |
+
explanation = Column(Text, nullable=True) # explanation/why it is abusive
|
| 67 |
+
reviewed = Column(Boolean, default=False) # admin reviewed flag
|
| 68 |
+
created_at = Column(DateTime, default=datetime.utcnow, index=True)
|
requirements-prod.txt
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ToxiGuard AI — Production Dependencies (Hugging Face Spaces & Cloud)
|
| 2 |
+
# ==============================================================================
|
| 3 |
+
# Modern compatible dependency specifications for zero-conflict builds with Gradio 6 / FastAPI.
|
| 4 |
+
|
| 5 |
+
# Web Framework & Server
|
| 6 |
+
fastapi>=0.115.2
|
| 7 |
+
uvicorn[standard]>=0.30.0
|
| 8 |
+
pydantic[email]>=2.8.0
|
| 9 |
+
email-validator>=2.2.0
|
| 10 |
+
pydantic-settings>=2.4.0
|
| 11 |
+
|
| 12 |
+
# Database & Security
|
| 13 |
+
sqlalchemy>=2.0.0
|
| 14 |
+
passlib[bcrypt]>=1.7.4
|
| 15 |
+
bcrypt>=4.0.1
|
| 16 |
+
python-jose[cryptography]>=3.3.0
|
| 17 |
+
cryptography>=42.0.0
|
| 18 |
+
slowapi>=0.1.9
|
| 19 |
+
|
| 20 |
+
# Machine Learning & ONNX Runtime (Inference Engine)
|
| 21 |
+
onnxruntime>=1.19.0
|
| 22 |
+
transformers>=4.48.0
|
| 23 |
+
tokenizers>=0.20.0
|
| 24 |
+
sentencepiece>=0.2.0
|
| 25 |
+
scikit-learn>=1.5.0
|
| 26 |
+
joblib>=1.4.0
|
| 27 |
+
numpy>=1.26.0
|
| 28 |
+
scipy>=1.13.0
|
| 29 |
+
|
| 30 |
+
# NLP & Sentiment Preprocessing
|
| 31 |
+
nltk>=3.9.0
|
| 32 |
+
textblob>=0.18.0
|
| 33 |
+
vaderSentiment>=3.3.2
|
| 34 |
+
|
| 35 |
+
# LLM Fallback & Utils
|
| 36 |
+
openai>=1.50.0
|
| 37 |
+
httpx>=0.27.0
|
| 38 |
+
python-dotenv>=1.0.0
|
| 39 |
+
pandas>=2.2.0
|
| 40 |
+
aiofiles>=24.1.0
|
| 41 |
+
python-multipart>=0.0.18
|
| 42 |
+
loguru>=0.7.2
|
requirements.txt
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ToxiGuard AI — Production Dependencies (Hugging Face Spaces & Cloud)
|
| 2 |
+
# ==============================================================================
|
| 3 |
+
# Modern compatible dependency specifications for zero-conflict builds with Gradio 6 / FastAPI.
|
| 4 |
+
|
| 5 |
+
# Web Framework & Server
|
| 6 |
+
fastapi>=0.115.2
|
| 7 |
+
uvicorn[standard]>=0.30.0
|
| 8 |
+
pydantic[email]>=2.8.0
|
| 9 |
+
email-validator>=2.2.0
|
| 10 |
+
pydantic-settings>=2.4.0
|
| 11 |
+
|
| 12 |
+
# Database & Security
|
| 13 |
+
sqlalchemy>=2.0.0
|
| 14 |
+
passlib[bcrypt]>=1.7.4
|
| 15 |
+
bcrypt>=4.0.1
|
| 16 |
+
python-jose[cryptography]>=3.3.0
|
| 17 |
+
cryptography>=42.0.0
|
| 18 |
+
slowapi>=0.1.9
|
| 19 |
+
|
| 20 |
+
# Machine Learning & ONNX Runtime (Inference Engine)
|
| 21 |
+
onnxruntime>=1.19.0
|
| 22 |
+
transformers>=4.48.0
|
| 23 |
+
tokenizers>=0.20.0
|
| 24 |
+
sentencepiece>=0.2.0
|
| 25 |
+
scikit-learn>=1.5.0
|
| 26 |
+
joblib>=1.4.0
|
| 27 |
+
numpy>=1.26.0
|
| 28 |
+
scipy>=1.13.0
|
| 29 |
+
|
| 30 |
+
# NLP & Sentiment Preprocessing
|
| 31 |
+
nltk>=3.9.0
|
| 32 |
+
textblob>=0.18.0
|
| 33 |
+
vaderSentiment>=3.3.2
|
| 34 |
+
|
| 35 |
+
# LLM Fallback & Utils
|
| 36 |
+
openai>=1.50.0
|
| 37 |
+
httpx>=0.27.0
|
| 38 |
+
python-dotenv>=1.0.0
|
| 39 |
+
pandas>=2.2.0
|
| 40 |
+
aiofiles>=24.1.0
|
| 41 |
+
python-multipart>=0.0.18
|
| 42 |
+
loguru>=0.7.2
|