wraithklu commited on
Commit
0fe62e1
·
1 Parent(s): 2c2008f

Deploy ToxiGuard AI backend to Hugging Face Spaces

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .dockerignore +19 -0
  2. .gitattributes +1 -33
  3. .python-version +1 -0
  4. README.md +10 -7
  5. abuse_model.joblib +3 -0
  6. app.py +87 -0
  7. app/__init__.py +0 -0
  8. app/core/__init__.py +0 -0
  9. app/core/config.py +88 -0
  10. app/core/limiter.py +7 -0
  11. app/core/logger.py +29 -0
  12. app/main.py +149 -0
  13. app/routes/__init__.py +0 -0
  14. app/routes/auth.py +159 -0
  15. app/routes/explain.py +82 -0
  16. app/routes/feedback.py +233 -0
  17. app/routes/moderation.py +699 -0
  18. app/routes/monitoring.py +55 -0
  19. app/routes/realtime.py +135 -0
  20. app/routes/stream.py +245 -0
  21. app/services/__init__.py +2 -0
  22. app/services/calibration.py +211 -0
  23. app/services/drift_monitor.py +288 -0
  24. app/services/model_service.py +179 -0
  25. app/services/overrides.py +99 -0
  26. app/services/retrainer.py +125 -0
  27. auth_utils.py +39 -0
  28. data/LICENSE +21 -0
  29. data/README.md +32 -0
  30. data/calibration.json +51 -0
  31. data/overrides.json +6 -0
  32. data/sample_data.csv +0 -0
  33. database.py +41 -0
  34. label_encoder.joblib +3 -0
  35. ml/__init__.py +1 -0
  36. ml/evaluate.py +67 -0
  37. ml/explainability.py +242 -0
  38. ml/export_onnx.py +146 -0
  39. ml/inference.py +288 -0
  40. ml/models/toxiguard-deberta/added_tokens.json +3 -0
  41. ml/models/toxiguard-deberta/config.json +52 -0
  42. ml/models/toxiguard-deberta/model.onnx +3 -0
  43. ml/models/toxiguard-deberta/special_tokens_map.json +51 -0
  44. ml/models/toxiguard-deberta/spm.model +3 -0
  45. ml/models/toxiguard-deberta/tokenizer.json +0 -0
  46. ml/models/toxiguard-deberta/tokenizer_config.json +59 -0
  47. ml/train_transformer.py +326 -0
  48. models.py +68 -0
  49. requirements-prod.txt +42 -0
  50. requirements.txt +42 -0
.dockerignore ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Docker build ignore file
2
+ venv/
3
+ .venv/
4
+ ENV/
5
+ env/
6
+ __pycache__/
7
+ *.pyc
8
+ *.pyo
9
+ *.pyd
10
+ .pytest_cache/
11
+ tests/
12
+ .git/
13
+ .gitignore
14
+ .gitattributes
15
+ toxiguard.db
16
+ *.db-journal
17
+ *.log
18
+ .DS_Store
19
+ Thumbs.db
.gitattributes CHANGED
@@ -1,35 +1,3 @@
1
- *.7z filter=lfs diff=lfs merge=lfs -text
2
- *.arrow filter=lfs diff=lfs merge=lfs -text
3
- *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
- *.ftz filter=lfs diff=lfs merge=lfs -text
7
- *.gz filter=lfs diff=lfs merge=lfs -text
8
- *.h5 filter=lfs diff=lfs merge=lfs -text
9
  *.joblib filter=lfs diff=lfs merge=lfs -text
10
- *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
  *.model filter=lfs diff=lfs merge=lfs -text
13
- *.msgpack filter=lfs diff=lfs merge=lfs -text
14
- *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
- *.onnx filter=lfs diff=lfs merge=lfs -text
17
- *.ot filter=lfs diff=lfs merge=lfs -text
18
- *.parquet filter=lfs diff=lfs merge=lfs -text
19
- *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
- *.pt filter=lfs diff=lfs merge=lfs -text
23
- *.pth filter=lfs diff=lfs merge=lfs -text
24
- *.rar filter=lfs diff=lfs merge=lfs -text
25
- *.safetensors filter=lfs diff=lfs merge=lfs -text
26
- saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
- *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tar filter=lfs diff=lfs merge=lfs -text
29
- *.tflite filter=lfs diff=lfs merge=lfs -text
30
- *.tgz filter=lfs diff=lfs merge=lfs -text
31
- *.wasm filter=lfs diff=lfs merge=lfs -text
32
- *.xz filter=lfs diff=lfs merge=lfs -text
33
- *.zip filter=lfs diff=lfs merge=lfs -text
34
- *.zst filter=lfs diff=lfs merge=lfs -text
35
- *tfevents* filter=lfs diff=lfs merge=lfs -text
 
1
+ *.onnx filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
2
  *.joblib filter=lfs diff=lfs merge=lfs -text
 
 
3
  *.model filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
.python-version ADDED
@@ -0,0 +1 @@
 
 
1
+ 3.11.9
README.md CHANGED
@@ -1,14 +1,17 @@
1
  ---
2
- title: Toxiguard Api
3
- emoji: 📊
4
  colorFrom: red
5
- colorTo: red
6
  sdk: gradio
7
- sdk_version: 6.31.0
8
- python_version: '3.12'
9
  app_file: app.py
10
  pinned: false
11
- license: mit
12
  ---
13
 
14
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
1
  ---
2
+ title: ToxiGuard AI Backend
3
+ emoji: 🛡
4
  colorFrom: red
5
+ colorTo: indigo
6
  sdk: gradio
 
 
7
  app_file: app.py
8
  pinned: false
 
9
  ---
10
 
11
+ # ToxiGuard AI - Content Moderation Platform
12
+ Production FastAPI moderation backend powered by DeBERTa-v3 with ONNX Runtime.
13
+
14
+ ## API Documentation
15
+ - Swagger UI: /docs
16
+ - ReDoc: /redoc
17
+ - Health: /health
abuse_model.joblib ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4db4c1532070f7690a14053a20b1defb8836328d4bb3bf4db50ef7891f5b0f87
3
+ size 41081
app.py ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — Hugging Face Space Application Entry Point
3
+ Mounts FastAPI backend onto Gradio so the space serves both:
4
+ 1. The interactive testing interface in the Hugging Face Space tab
5
+ 2. All production REST API endpoints: /docs, /redoc, /health, /predict, /chat/moderate, etc.
6
+ """
7
+ import os
8
+ import sys
9
+
10
+ # 1. Disable Gradio SSR Node proxy (prevents Errno 98 port 7860 collision)
11
+ os.environ["GRADIO_SSR_MODE"] = "false"
12
+
13
+ # 2. Hugging Face ZeroGPU initialization
14
+ try:
15
+ import spaces
16
+ except ImportError:
17
+ class spaces:
18
+ @staticmethod
19
+ def GPU(fn=None, **kwargs):
20
+ if fn is not None:
21
+ return fn
22
+ def decorator(f):
23
+ return f
24
+ return decorator
25
+
26
+ # Ensure backend root is in Python module search path
27
+ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
28
+
29
+ import gradio as gr
30
+ from app.main import app as fastapi_app
31
+ from app.services.model_service import model_service
32
+
33
+
34
+ # Literal @spaces.GPU decorator required for ZeroGPU static AST scanner
35
+ @spaces.GPU
36
+ def analyze_toxicity(text: str):
37
+ """Interactive demo function for the Gradio UI."""
38
+ if not text or not text.strip():
39
+ return {"error": "Please enter text to analyze."}
40
+ try:
41
+ res = model_service.predict(text)
42
+ return {
43
+ "is_toxic": bool(res.get("is_toxic", False)),
44
+ "confidence": round(float(res.get("confidence", 0.0)), 4),
45
+ "primary_category": res.get("primary_category", "none"),
46
+ "severity": res.get("severity", "none"),
47
+ "model_used": res.get("model_used", "onnx-deberta"),
48
+ "scores": {k: round(float(v), 4) for k, v in res.get("scores", {}).items()}
49
+ }
50
+ except Exception as e:
51
+ return {"error": str(e)}
52
+
53
+
54
+ # ─── Gradio Interface for the Space page ─────────────────────────────────────
55
+ demo = gr.Interface(
56
+ fn=analyze_toxicity,
57
+ inputs=gr.Textbox(
58
+ lines=3,
59
+ placeholder="Enter text to analyze for toxicity, harassment, abuse...",
60
+ label="Input Content"
61
+ ),
62
+ outputs=gr.JSON(label="Analysis Result"),
63
+ title="🛡️ ToxiGuard AI — Content Moderation Platform",
64
+ description=(
65
+ "**Real-time multi-label toxicity detection engine powered by DeBERTa-v3 & ONNX Runtime.**\n\n"
66
+ "### 🔗 Developer API Access:\n"
67
+ "- 📖 **Interactive Swagger UI**: [/docs](/docs)\n"
68
+ "- 📑 **ReDoc Documentation**: [/redoc](/redoc)\n"
69
+ "- 💓 **Health & Model Status**: [/health](/health)\n"
70
+ "- 🚀 **Prediction Endpoint**: `POST /predict` or `POST /predict/demo`"
71
+ ),
72
+ examples=[
73
+ ["Thank you so much for your assistance! Have a wonderful day."],
74
+ ["You are completely brainless and deserve to be banned forever."],
75
+ ["Stop talking before I find where you live and hurt you."],
76
+ ["I hate this stupid policy, but I understand why it was made."]
77
+ ],
78
+ cache_examples=False,
79
+ )
80
+
81
+ # Mount Gradio onto the existing FastAPI application
82
+ app = gr.mount_gradio_app(fastapi_app, demo, path="/")
83
+
84
+ if __name__ == "__main__":
85
+ import uvicorn
86
+ port = int(os.environ.get("PORT", 7860))
87
+ uvicorn.run(app, host="0.0.0.0", port=port)
app/__init__.py ADDED
File without changes
app/core/__init__.py ADDED
File without changes
app/core/config.py ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — Centralised Configuration
3
+ ==========================================
4
+ All environment variables and runtime constants are declared here.
5
+ Import `settings` anywhere — never read os.environ directly in app code.
6
+ """
7
+
8
+ import os
9
+ from functools import lru_cache
10
+ # pyrefly: ignore [missing-import]
11
+ from pydantic_settings import BaseSettings, SettingsConfigDict
12
+
13
+
14
+ class Settings(BaseSettings):
15
+ model_config = SettingsConfigDict(
16
+ env_file=os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env"),
17
+ env_file_encoding="utf-8",
18
+ extra="ignore",
19
+ protected_namespaces=("settings_",), # Avoid conflict with model_dir field
20
+ )
21
+
22
+ # ── LLM ─────────────────────────────────────────────────────────
23
+ openrouter_api_key: str = ""
24
+ openrouter_model: str = "google/gemma-4-31b-it:free"
25
+ openrouter_fallback_model: str = "qwen/qwen3-next-80b-a3b-instruct:free"
26
+ llm_timeout: float = 20.0
27
+ llm_cache_size: int = 128
28
+ llm_temperature: float = 0.4
29
+ llm_max_tokens: int = 2048
30
+
31
+ # Optional: Google Gemini API Key (secondary fallback if OpenRouter is unreachable)
32
+ gemini_model: str = "gemini-2.5-flash"
33
+ gemini_api_key: str = ""
34
+
35
+ # ── DATABASE ─────────────────────────────────────────────────────
36
+ database_url: str = "sqlite:///./toxiguard.db"
37
+
38
+ # ── AUTH ─────────────────────────────────────────────────────────
39
+ secret_key: str = "toxiguard-secret-change-in-production"
40
+ algorithm: str = "HS256"
41
+ access_token_expire_minutes: int = 60 * 24 # 24 hours
42
+
43
+ allowed_origins: str = (
44
+ "https://toxiai.vercel.app,https://toxiai-agent.vercel.app,http://localhost:5173,http://127.0.0.1:5173"
45
+ )
46
+
47
+ @property
48
+ def allowed_origins_list(self) -> list[str]:
49
+ return [o.strip() for o in self.allowed_origins.split(",") if o.strip()]
50
+
51
+ # ── MODEL ─────────────────────────────────────────────────────────
52
+ model_dir: str = os.path.join(
53
+ os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
54
+ "ml", "models", "toxiguard-deberta"
55
+ )
56
+ max_seq_length: int = 256
57
+
58
+ # ── ENSEMBLE THRESHOLDS ───────────────────────────────────────────
59
+ # DeBERTa is primary (50%), LLM is context layer (35%), Rules are fast-path (15%)
60
+ ensemble_threshold: float = 0.45
61
+ ml_trigger_threshold: float = 0.35 # Call LLM if ML >= this
62
+ demo_ml_threshold: float = 0.45
63
+
64
+ # Severity bands
65
+ severity_high: float = 0.85
66
+ severity_medium: float = 0.60
67
+
68
+ # ── PLAN LIMITS ───────────────────────────────────────────────────
69
+ plan_limit_free: int = 1_000
70
+ plan_limit_pro: int = 50_000
71
+
72
+ # ── APP META ─────────────────────────────────────────────────────
73
+ app_version: str = "2.0.0"
74
+ app_title: str = "ToxiGuard AI"
75
+ app_description: str = (
76
+ "Hybrid toxic content detection API combining rule-based filtering, "
77
+ "DeBERTa transformer (ONNX-optimised), and LLM-powered contextual analysis. "
78
+ "Built for real-time content moderation at scale."
79
+ )
80
+
81
+
82
+ @lru_cache(maxsize=1)
83
+ def get_settings() -> Settings:
84
+ return Settings()
85
+
86
+
87
+ # Module-level singleton
88
+ settings = get_settings()
app/core/limiter.py ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ # backend/app/core/limiter.py
2
+
3
+ from slowapi import Limiter
4
+ from slowapi.util import get_remote_address
5
+
6
+ limiter = Limiter(key_func=get_remote_address)
7
+ limiter.enabled = True
app/core/logger.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — Structured Logger
3
+ ==================================
4
+ Loguru-based logger with ISO timestamps and severity colours.
5
+ Import `logger` everywhere — never use bare print() in app code.
6
+ """
7
+
8
+ import sys
9
+ from loguru import logger
10
+
11
+ # Remove default handler and replace with a clean, structured format
12
+ logger.remove()
13
+
14
+ logger.add(
15
+ sys.stderr,
16
+ level="INFO",
17
+ format=(
18
+ "<green>{time:YYYY-MM-DD HH:mm:ss}</green> | "
19
+ "<level>{level: <8}</level> | "
20
+ "<cyan>{name}</cyan>:<cyan>{line}</cyan> | "
21
+ "<level>{message}</level>"
22
+ ),
23
+ colorize=True,
24
+ )
25
+
26
+ # Optional: file sink for persistent logs
27
+ # logger.add("logs/toxiguard.log", rotation="10 MB", retention="7 days", level="DEBUG")
28
+
29
+ __all__ = ["logger"]
app/main.py ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from contextlib import asynccontextmanager
3
+ from fastapi import FastAPI, Request
4
+ from fastapi.middleware.cors import CORSMiddleware
5
+ from fastapi.responses import JSONResponse
6
+ from dotenv import load_dotenv
7
+
8
+ load_dotenv()
9
+
10
+ from slowapi.middleware import SlowAPIMiddleware
11
+ from slowapi.errors import RateLimitExceeded
12
+
13
+ from app.core.limiter import limiter
14
+ from app.core.logger import logger
15
+ from app.core.config import settings
16
+ from app.routes.moderation import router as moderation_router
17
+ from app.routes.auth import router as auth_router
18
+ from app.routes.realtime import router as realtime_router
19
+ from app.routes.explain import router as explain_router
20
+ from app.routes.monitoring import router as monitoring_router
21
+ from app.routes.feedback import router as feedback_router
22
+ from app.routes.stream import router as stream_router
23
+ from app.services.model_service import model_service
24
+
25
+ from database import engine, Base
26
+ Base.metadata.create_all(bind=engine)
27
+
28
+
29
+ # ──────────────────────────────────────────────────────────────────────────────
30
+ # LIFESPAN (startup / shutdown)
31
+ # ──────────────────────────────────────────────────────────────────────────────
32
+
33
+ @asynccontextmanager
34
+ async def lifespan(app: FastAPI):
35
+ # Startup
36
+ logger.info("=" * 50)
37
+ logger.info(f"ToxiGuard AI v{settings.app_version} starting up")
38
+ logger.info(f"Model status: {model_service.status}")
39
+ logger.info(f"CORS origins: {settings.allowed_origins_list}")
40
+
41
+ # Pre-warm XAI Explainer to avoid lag on first explain query
42
+ try:
43
+ from ml.explainability import get_explainer
44
+ explainer = get_explainer(settings.model_dir)
45
+ if explainer.is_ready:
46
+ logger.info("[Lifespan] XAI Explainer pre-warmed successfully")
47
+ else:
48
+ logger.warning("[Lifespan] XAI Explainer pre-warm skipped: Explainer not ready (model files missing)")
49
+ except Exception as e:
50
+ logger.warning(f"[Lifespan] Failed to pre-warm XAI explainer: {e}")
51
+
52
+ logger.info("=" * 50)
53
+ yield
54
+ # Shutdown
55
+ logger.info("ToxiGuard AI shutting down")
56
+
57
+
58
+ # ──────────────────────────────────────────────────────────────────────────────
59
+ # APP INIT
60
+ # ──────────────────────────────────────────────────────────────────────────────
61
+
62
+ app = FastAPI(
63
+ title=settings.app_title,
64
+ description=settings.app_description,
65
+ version=settings.app_version,
66
+ docs_url="/docs",
67
+ redoc_url="/redoc",
68
+ lifespan=lifespan,
69
+ )
70
+
71
+
72
+ # ──────────────────────────────────────────────────────────────────────────────
73
+ # RATE LIMITING
74
+ # ──────────────────────────────────────────────────────────────────────────────
75
+
76
+ app.state.limiter = limiter
77
+ app.add_middleware(SlowAPIMiddleware)
78
+
79
+
80
+ @app.exception_handler(RateLimitExceeded)
81
+ def rate_limit_handler(request: Request, exc: RateLimitExceeded):
82
+ return JSONResponse(
83
+ status_code=429,
84
+ content={"detail": "Rate limit exceeded. Please try again shortly."},
85
+ )
86
+
87
+
88
+ # ──────────────────────────────────────────────────────────────────────────────
89
+ # CORS — reads from ALLOWED_ORIGINS env var (not hardcoded ["*"])
90
+ # ──────────────────────────────────────────────────────────────────────────────
91
+
92
+ app.add_middleware(
93
+ CORSMiddleware,
94
+ allow_origins=settings.allowed_origins_list,
95
+ allow_credentials=False,
96
+ allow_methods=["GET", "POST", "OPTIONS"],
97
+ allow_headers=["Content-Type", "X-API-Key", "Authorization"],
98
+ )
99
+
100
+
101
+ # ──────────────────────────────────────────────────────────────────────────────
102
+ # ROUTES
103
+ # ──────────────────────────────────────────────────────────────────────────────
104
+
105
+ app.include_router(auth_router)
106
+ app.include_router(moderation_router)
107
+ app.include_router(realtime_router)
108
+ app.include_router(explain_router)
109
+ app.include_router(monitoring_router)
110
+ app.include_router(feedback_router)
111
+ app.include_router(stream_router)
112
+
113
+
114
+ # ──────────────────────────────────────────────────────────────────────────────
115
+ # HEALTH CHECK
116
+ # ──────────────────────────────────────────────────────────────────────────────
117
+
118
+ @app.get("/", tags=["Health"])
119
+ def root():
120
+ return {
121
+ "status": "ToxiGuard AI running",
122
+ "version": settings.app_version,
123
+ }
124
+
125
+
126
+ @app.get("/health", tags=["Health"])
127
+ def health():
128
+ """Detailed health check with model and endpoint status."""
129
+ return {
130
+ "status": "healthy",
131
+ "version": settings.app_version,
132
+ "model": model_service.status,
133
+ "endpoints": {
134
+ "predict_full": "POST /predict",
135
+ "predict_ml": "POST /predict/ml",
136
+ "predict_demo": "POST /predict/demo",
137
+ "chat_moderate": "POST /chat/moderate",
138
+ "analyze_file": "POST /analyze-file",
139
+ "explain": "POST /explain [XAI]",
140
+ "monitoring_stats": "GET /monitoring/stats [Drift]",
141
+ "monitoring_drift": "GET /monitoring/drift [Drift]",
142
+ "feedback": "POST /feedback [Active Learning]",
143
+ "feedback_stats": "GET /feedback/stats",
144
+ "feedback_queue": "GET /feedback/queue",
145
+ "stream": "WS /ws/stream [Real-time]",
146
+ "auth": "POST /auth/signup, POST /auth/login",
147
+ "docs": "/docs",
148
+ },
149
+ }
app/routes/__init__.py ADDED
File without changes
app/routes/auth.py ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter, Depends, HTTPException, Security
2
+ from fastapi.security import APIKeyHeader
3
+ from sqlalchemy.orm import Session
4
+ from pydantic import BaseModel, EmailStr, field_validator
5
+ import secrets
6
+
7
+ from database import get_db
8
+ from models import User
9
+ from auth_utils import create_access_token, generate_api_key
10
+ from passlib.context import CryptContext
11
+
12
+ # ──────────────────────────────────────────────────────────────────────────────
13
+ # ROUTER
14
+ # ──────────────────────────────────────────────────────────────────────────────
15
+
16
+ router = APIRouter(prefix="/auth", tags=["Auth"])
17
+
18
+ pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
19
+ api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False)
20
+
21
+
22
+ # ──────────────────────────────────────────────────────────────────────────────
23
+ # SCHEMAS
24
+ # ──────────────────────────────────────────────────────────────────────────────
25
+
26
+ class SignupRequest(BaseModel):
27
+ email: EmailStr
28
+ password: str
29
+
30
+ @field_validator("password")
31
+ @classmethod
32
+ def password_min_length(cls, v: str) -> str:
33
+ if len(v) < 6:
34
+ raise ValueError("Password must be at least 6 characters")
35
+ return v
36
+
37
+
38
+ class LoginRequest(BaseModel):
39
+ email: str
40
+ password: str
41
+
42
+
43
+ # ──────────────────────────────────────────────────────────────────────────────
44
+ # HELPERS
45
+ # ──────────────────────────────────────────────────────────────────────────────
46
+
47
+ def hash_password(password: str) -> str:
48
+ return pwd_context.hash(password)
49
+
50
+
51
+ def verify_password(plain: str, hashed: str) -> bool:
52
+ return pwd_context.verify(plain, hashed)
53
+
54
+
55
+ # ──────────────────────────────────────────────────────────────────────────────
56
+ # SIGNUP
57
+ # ──────────────────────────────────────────────────────────────────────────────
58
+
59
+ @router.post("/signup", status_code=201)
60
+ def signup(req: SignupRequest, db: Session = Depends(get_db)):
61
+ """Register a new user and return their API key."""
62
+ existing = db.query(User).filter(User.email == req.email).first()
63
+ if existing:
64
+ raise HTTPException(status_code=400, detail="Email already registered")
65
+
66
+ user = User(
67
+ email=req.email,
68
+ password_hash=hash_password(req.password),
69
+ api_key=generate_api_key(),
70
+ plan="free",
71
+ usage_count=0,
72
+ )
73
+ db.add(user)
74
+ db.commit()
75
+ db.refresh(user)
76
+
77
+ return {
78
+ "message": "Account created successfully",
79
+ "api_key": user.api_key,
80
+ "plan": user.plan,
81
+ }
82
+
83
+
84
+ # ──────────────────────────────────────────────────────────────────────────────
85
+ # LOGIN
86
+ # ──────────────────────────────────────────────────────────────────────────────
87
+
88
+ @router.post("/login")
89
+ def login(req: LoginRequest, db: Session = Depends(get_db)):
90
+ """Authenticate and return a JWT token + API key."""
91
+ user = db.query(User).filter(User.email == req.email).first()
92
+ if not user or not verify_password(req.password, user.password_hash):
93
+ raise HTTPException(status_code=401, detail="Invalid credentials")
94
+
95
+ token = create_access_token({"sub": user.email})
96
+
97
+ return {
98
+ "token": token,
99
+ "api_key": user.api_key,
100
+ "plan": user.plan,
101
+ "usage_count": user.usage_count,
102
+ }
103
+
104
+
105
+ # ──────────────────────────────────────────────────────────────────────────────
106
+ # API KEY AUTH DEPENDENCY
107
+ # ──────────────────────────────────────────────────────────────────────────────
108
+
109
+ def get_api_user(
110
+ x_api_key: str = Security(api_key_header),
111
+ db: Session = Depends(get_db),
112
+ ) -> User:
113
+ if not x_api_key:
114
+ raise HTTPException(status_code=401, detail="API key missing")
115
+
116
+ user = db.query(User).filter(User.api_key == x_api_key).first()
117
+ if not user:
118
+ raise HTTPException(status_code=403, detail="Invalid API key")
119
+
120
+ return user
121
+
122
+
123
+ # ──────────────────────────────────────────────────────────────────────────────
124
+ # PROFILE
125
+ # ──────────────────────────────────────────────────────────────────────────────
126
+
127
+ @router.get("/me")
128
+ def get_me(user: User = Depends(get_api_user)):
129
+ """Return the authenticated user's profile and usage stats."""
130
+ return {
131
+ "email": user.email,
132
+ "plan": user.plan,
133
+ "usage_count": user.usage_count,
134
+ "api_key": user.api_key,
135
+ "last_used": user.last_used.isoformat() if user.last_used else None,
136
+ }
137
+
138
+
139
+ # ──────────────────────────────────────────────────────────────────────────────
140
+ # ROTATE API KEY
141
+ # ──────────────────────────────────────────────────────────────────────────────
142
+
143
+ @router.post("/reset-key")
144
+ def reset_api_key(
145
+ user: User = Depends(get_api_user),
146
+ db: Session = Depends(get_db),
147
+ ):
148
+ """
149
+ Generate and store a new API key for the authenticated user.
150
+ The old key is immediately invalidated.
151
+ """
152
+ user.api_key = generate_api_key()
153
+ db.commit()
154
+ db.refresh(user)
155
+
156
+ return {
157
+ "message": "API key rotated successfully",
158
+ "api_key": user.api_key,
159
+ }
app/routes/explain.py ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — XAI Explain Endpoint
3
+ ======================================
4
+ POST /explain — Token-level attention attribution for toxicity decisions.
5
+
6
+ Returns per-token importance scores extracted from DeBERTa's last attention layer.
7
+ Used to answer: "Why did the model flag this text as toxic?"
8
+ """
9
+
10
+ from fastapi import APIRouter, Depends, Request, Body
11
+ from pydantic import BaseModel
12
+ from sqlalchemy.orm import Session
13
+
14
+ from app.routes.auth import get_api_user
15
+ from app.core.limiter import limiter
16
+ from app.core.config import settings
17
+ from app.core.logger import logger
18
+ from ml.explainability import get_explainer
19
+ from models import User
20
+ from database import get_db
21
+
22
+ router = APIRouter()
23
+
24
+
25
+ class ExplainRequest(BaseModel):
26
+ text: str
27
+
28
+
29
+ @router.post("/explain", tags=["XAI"])
30
+ @limiter.limit("20/minute")
31
+ def explain_prediction(
32
+ request: Request,
33
+ req: ExplainRequest = Body(...),
34
+ user: User = Depends(get_api_user),
35
+ db: Session = Depends(get_db),
36
+ ):
37
+ """
38
+ Token-level XAI attribution via DeBERTa attention weights.
39
+
40
+ Returns each word's contribution score (0.0 = no influence, 1.0 = highest influence).
41
+ Use this to build token heatmaps in the frontend.
42
+
43
+ Response:
44
+ {
45
+ "tokens": [{"token": str, "score": float, "is_special": bool}],
46
+ "top_tokens": [{"token": str, "score": float}],
47
+ "method": "attention_last_layer_mean",
48
+ "num_tokens": int,
49
+ "status": "ok" | "unavailable"
50
+ }
51
+ """
52
+ text = req.text.strip()
53
+ if not text:
54
+ return {"error": "Empty text", "status": "error"}
55
+
56
+ logger.info(f"[/explain] user={user.email} | len={len(text)}")
57
+
58
+ explainer = get_explainer(settings.model_dir)
59
+
60
+ if not explainer.is_ready:
61
+ return {
62
+ "tokens": [],
63
+ "top_tokens": [],
64
+ "method": "unavailable",
65
+ "num_tokens": 0,
66
+ "status": "unavailable",
67
+ "message": "XAI explainer not ready — check backend logs",
68
+ }
69
+
70
+ result = explainer.explain(text)
71
+
72
+ if result is None:
73
+ return {
74
+ "tokens": [],
75
+ "top_tokens": [],
76
+ "method": "error",
77
+ "num_tokens": 0,
78
+ "status": "error",
79
+ "message": "Attribution failed",
80
+ }
81
+
82
+ return {**result, "status": "ok"}
app/routes/feedback.py ADDED
@@ -0,0 +1,233 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — Active Learning Feedback Loop
3
+ =============================================
4
+ Endpoints for collecting user corrections to model predictions.
5
+
6
+ POST /feedback — Submit a false positive or false negative correction
7
+ GET /feedback/queue — View unreviewed samples (for retraining queue)
8
+ GET /feedback/stats — Aggregate feedback counts and accuracy signals
9
+
10
+ Design:
11
+ - Every flagged prediction is stored with original text + user's correction
12
+ - `feedback_type` is computed automatically: "false_positive" or "false_negative"
13
+ - `reviewed` flag allows admins to mark samples as processed
14
+ - This is the standard Active Learning data pipeline used in production ML systems
15
+ """
16
+
17
+ from datetime import datetime
18
+ from fastapi import APIRouter, Depends, HTTPException, Request, Body
19
+ from pydantic import BaseModel
20
+ from sqlalchemy.orm import Session
21
+ from sqlalchemy import func
22
+ from typing import Optional
23
+
24
+ from app.routes.auth import get_api_user
25
+ from app.core.limiter import limiter
26
+ from app.core.logger import logger
27
+ from models import User, FeedbackSample
28
+ from database import get_db
29
+ from app.services.calibration import calibration_service
30
+
31
+ router = APIRouter()
32
+
33
+
34
+ # ──────────────────────────────────────────────────────────────────────────────
35
+ # SCHEMA
36
+ # ──────────────────────────────────────────────────────────────────────────────
37
+
38
+ class FeedbackRequest(BaseModel):
39
+ input_text: str
40
+ predicted_toxic: bool # What the model said
41
+ correct_label: bool # What the user says is correct
42
+ confidence_at_time: Optional[float] = None
43
+ notes: Optional[str] = None
44
+ abusive_words: Optional[str] = None
45
+ explanation: Optional[str] = None
46
+
47
+
48
+ # ──────────────────────────────────────────────────────────────────────────────
49
+ # ENDPOINTS
50
+ # ──────────────────────────────────────────────────────────────────────────────
51
+
52
+ @router.post("/feedback", tags=["Active Learning"])
53
+ @limiter.limit("10/minute")
54
+ def submit_feedback(
55
+ request: Request,
56
+ req: FeedbackRequest = Body(...),
57
+ user: User = Depends(get_api_user),
58
+ db: Session = Depends(get_db),
59
+ ):
60
+ """
61
+ Submit a prediction correction for active learning.
62
+
63
+ - If model said TOXIC but it's not → false_positive
64
+ - If model said SAFE but it is toxic → false_negative
65
+ """
66
+ text = req.input_text.strip()
67
+ if not text:
68
+ raise HTTPException(status_code=400, detail="input_text cannot be empty")
69
+
70
+ if len(text) > 2000:
71
+ raise HTTPException(status_code=400, detail="Text too long (max 2000 chars)")
72
+
73
+ # Auto-determine feedback type
74
+ if req.predicted_toxic and not req.correct_label:
75
+ feedback_type = "false_positive"
76
+ elif not req.predicted_toxic and req.correct_label:
77
+ feedback_type = "false_negative"
78
+ else:
79
+ # Model was correct — user is confirming; still log for confidence calibration
80
+ feedback_type = "confirmed_correct"
81
+
82
+ sample = FeedbackSample(
83
+ user_id=user.id,
84
+ input_text=text,
85
+ predicted_toxic=req.predicted_toxic,
86
+ correct_label=req.correct_label,
87
+ feedback_type=feedback_type,
88
+ confidence_at_time=req.confidence_at_time,
89
+ notes=req.notes,
90
+ abusive_words=req.abusive_words,
91
+ explanation=req.explanation,
92
+ reviewed=False,
93
+ )
94
+ db.add(sample)
95
+ db.commit()
96
+ db.refresh(sample)
97
+
98
+ # Apply adaptive threshold calibration in real-time
99
+ calibration_result = calibration_service.apply_feedback(
100
+ feedback_type,
101
+ req.confidence_at_time
102
+ )
103
+
104
+ # Save to dynamic hot-patches blocklist/allowlist for instant update
105
+ from app.services.overrides import override_service
106
+ override_service.add_override(feedback_type, text)
107
+
108
+ logger.info(
109
+ f"[/feedback] user={user.email} | type={feedback_type} | len={len(text)} | threshold adjusted: {calibration_result.get('old_threshold')} -> {calibration_result.get('new_threshold')}"
110
+ )
111
+
112
+ return {
113
+ "id": sample.id,
114
+ "feedback_type": feedback_type,
115
+ "message": "Feedback recorded. Thank you for improving ToxiGuard AI!",
116
+ "queued_for_review": True,
117
+ "calibration": calibration_result,
118
+ }
119
+
120
+
121
+ @router.get("/feedback/stats", tags=["Active Learning"])
122
+ @limiter.limit("20/minute")
123
+ def get_feedback_stats(
124
+ request: Request,
125
+ user: User = Depends(get_api_user),
126
+ db: Session = Depends(get_db),
127
+ ):
128
+ """
129
+ Aggregate feedback statistics for the active learning dashboard.
130
+
131
+ Returns counts by type, overall accuracy signal, and review status.
132
+ """
133
+ total = db.query(func.count(FeedbackSample.id)).scalar() or 0
134
+ fp = db.query(func.count(FeedbackSample.id)).filter(
135
+ FeedbackSample.feedback_type == "false_positive"
136
+ ).scalar() or 0
137
+ fn = db.query(func.count(FeedbackSample.id)).filter(
138
+ FeedbackSample.feedback_type == "false_negative"
139
+ ).scalar() or 0
140
+ confirmed = db.query(func.count(FeedbackSample.id)).filter(
141
+ FeedbackSample.feedback_type == "confirmed_correct"
142
+ ).scalar() or 0
143
+ unreviewed = db.query(func.count(FeedbackSample.id)).filter(
144
+ FeedbackSample.reviewed == False
145
+ ).scalar() or 0
146
+
147
+ # Error rate signal: (FP + FN) / total corrections
148
+ correction_total = fp + fn
149
+ error_rate = round(correction_total / total, 4) if total > 0 else 0.0
150
+
151
+ # Precision signal: what fraction of flagged-as-toxic were actually toxic
152
+ # FP = flagged toxic, wasn't; confirmed = flagged toxic, was toxic
153
+ # (approximate from feedback alone, not full prediction volume)
154
+ precision_approx = None
155
+ if (fp + confirmed) > 0:
156
+ precision_approx = round(confirmed / (fp + confirmed), 4)
157
+
158
+ return {
159
+ "total_feedback": total,
160
+ "false_positives": fp,
161
+ "false_negatives": fn,
162
+ "confirmed_correct": confirmed,
163
+ "unreviewed_count": unreviewed,
164
+ "error_rate_signal": error_rate,
165
+ "precision_approx": precision_approx,
166
+ "retraining_recommended": correction_total >= 50,
167
+ }
168
+
169
+
170
+ @router.get("/feedback/queue", tags=["Active Learning"])
171
+ @limiter.limit("10/minute")
172
+ def get_feedback_queue(
173
+ request: Request,
174
+ limit: int = 20,
175
+ feedback_type: Optional[str] = None,
176
+ user: User = Depends(get_api_user),
177
+ db: Session = Depends(get_db),
178
+ ):
179
+ """
180
+ View the active learning queue — unreviewed correction samples.
181
+
182
+ Use `feedback_type` to filter: false_positive | false_negative | confirmed_correct
183
+ """
184
+ limit = min(limit, 100)
185
+
186
+ query = db.query(FeedbackSample).filter(FeedbackSample.reviewed == False)
187
+
188
+ if feedback_type:
189
+ query = query.filter(FeedbackSample.feedback_type == feedback_type)
190
+
191
+ samples = query.order_by(FeedbackSample.created_at.desc()).limit(limit).all()
192
+
193
+ return {
194
+ "count": len(samples),
195
+ "samples": [
196
+ {
197
+ "id": s.id,
198
+ "input_text": s.input_text[:200] + ("..." if len(s.input_text) > 200 else ""),
199
+ "predicted_toxic": s.predicted_toxic,
200
+ "correct_label": s.correct_label,
201
+ "feedback_type": s.feedback_type,
202
+ "confidence_at_time": s.confidence_at_time,
203
+ "notes": s.notes,
204
+ "created_at": s.created_at.isoformat(),
205
+ }
206
+ for s in samples
207
+ ],
208
+ }
209
+
210
+
211
+ @router.post("/feedback/retrain", tags=["Active Learning"])
212
+ @limiter.limit("5/minute")
213
+ def trigger_retraining(
214
+ request: Request,
215
+ user: User = Depends(get_api_user),
216
+ db: Session = Depends(get_db),
217
+ ):
218
+ """
219
+ Trigger manual model retraining on feedback queue.
220
+ Appends corrections to data/sample_data.csv, retrains TF-IDF model,
221
+ saves new joblib checkpoint, and hot-reloads prediction service.
222
+ """
223
+ logger.info(f"[/feedback/retrain] triggered by admin/user: {user.email}")
224
+ try:
225
+ from app.services.retrainer import retrain_model
226
+ result = retrain_model(db)
227
+ return result
228
+ except Exception as exc:
229
+ logger.error(f"[/feedback/retrain] Retraining failed: {exc}")
230
+ raise HTTPException(
231
+ status_code=500,
232
+ detail=f"Retraining failed: {str(exc)}"
233
+ )
app/routes/moderation.py ADDED
@@ -0,0 +1,699 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — Moderation Routes
3
+ ==================================
4
+ Three prediction endpoints:
5
+
6
+ POST /predict/demo — Public demo (IP rate-limited, no auth, ML only)
7
+ POST /predict/ml — Extension fast-path (auth, ML only, no LLM)
8
+ POST /predict — Full pipeline (auth, rules + DeBERTa + LLM)
9
+ POST /analyze-file — Batch CSV/TXT analysis (auth, ensemble scoring)
10
+
11
+ Ensemble weighting (transformer model):
12
+ DeBERTa 50% — primary, SOTA accuracy
13
+ LLM 35% — contextual nuance (only called when needed)
14
+ Rules 15% — fast-path for obvious slurs
15
+ """
16
+
17
+
18
+ from collections import Counter
19
+
20
+ import io
21
+ import pandas as pd
22
+ from datetime import datetime
23
+
24
+ from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile, File, Body
25
+ from sqlalchemy.orm import Session
26
+ from pydantic import BaseModel
27
+
28
+ from app.services.model_service import model_service
29
+ from app.routes.auth import get_api_user
30
+ from app.core.limiter import limiter
31
+ from app.core.logger import logger
32
+ from app.core.config import settings
33
+
34
+ from utils.preprocessing import preprocess_for_rules, preprocess_for_model
35
+ from utils.abuse_words import detect_abusive_tokens, get_abuse_severity, suggestions as abuse_suggestions
36
+ from utils.sentiment import analyze_sentiment
37
+ from utils.llm_guard import analyze_toxicity_llm
38
+
39
+ from models import User
40
+ from database import get_db
41
+ from app.services.drift_monitor import drift_monitor
42
+ from app.services.calibration import calibration_service
43
+
44
+ # ──────────────────────────────────────────────────────────────────────────────
45
+ # ROUTER
46
+ # ──────────────────────────────────────────────────────────────────────────────
47
+
48
+ router = APIRouter()
49
+
50
+ PLAN_LIMITS = {
51
+ "free": settings.plan_limit_free,
52
+ "pro": settings.plan_limit_pro,
53
+ }
54
+
55
+ TOXIC_ML_LABELS = frozenset({
56
+ "abusive", "toxic", "severe_toxic",
57
+ "obscene", "threat", "insult", "identity_hate",
58
+ })
59
+
60
+
61
+ # ──────────────────────────────────────────────────────────────────────────────
62
+ # SCHEMAS
63
+ # ──────────────────────────────────────────────────────────────────────────────
64
+
65
+ class TextRequest(BaseModel):
66
+ text: str
67
+
68
+
69
+ # ──────────────────────────────────────────────────────────────────────────────
70
+ # SHARED HELPERS
71
+ # ──────────────────────────────────────────────────────────────────────────────
72
+
73
+ def _generate_suggestions(words: list[str]) -> dict[str, str]:
74
+ result = {}
75
+ for w in words:
76
+ key = w.lower()
77
+ if key in abuse_suggestions:
78
+ result[w] = abuse_suggestions[key]
79
+ else:
80
+ # Word-specific fallback — more useful than a generic message
81
+ result[w] = f"Consider replacing '{w}' with a more neutral or constructive expression."
82
+ return result
83
+
84
+
85
+ def _build_response(payload: dict) -> dict:
86
+ words = payload.get("abusive_words", [])
87
+ payload["word_frequency"] = dict(Counter(words))
88
+ payload["suggestions"] = _generate_suggestions(words)
89
+ return payload
90
+
91
+
92
+ def _adjust_sentiment(sentiment: dict | None, toxic: bool) -> dict | None:
93
+ if toxic and sentiment:
94
+ sentiment["label"] = "negative"
95
+ return sentiment
96
+
97
+
98
+ def _check_plan_limit(user: User, db: Session) -> None:
99
+ if user.usage_count >= PLAN_LIMITS.get(user.plan, settings.plan_limit_free):
100
+ raise HTTPException(status_code=403, detail="Usage limit exceeded")
101
+ user.usage_count += 1
102
+ user.last_used = datetime.utcnow()
103
+ db.commit()
104
+
105
+
106
+ # ──────────────────────────────────────────────────────────────────────────────
107
+ # ML RESULT NORMALISATION
108
+ # ──────────────────────────────────────────────────────────────────────────────
109
+
110
+ def normalize_ml_result(ml_result: dict | None, threshold: float | None = None) -> dict:
111
+ """Normalise raw ML output into a standard toxicity verdict."""
112
+ if threshold is None:
113
+ threshold = calibration_service.get_threshold()
114
+ if not ml_result:
115
+ return {
116
+ "toxic": False,
117
+ "confidence": 0.0,
118
+ "severity": "low",
119
+ "category": "safe",
120
+ "detected_categories": [],
121
+ }
122
+
123
+ labels: dict = ml_result.get("labels") or {}
124
+ label: str = str(ml_result.get("label") or "").lower()
125
+ detected_categories: list[str] = [
126
+ str(c).lower() for c in ml_result.get("detected_categories", [])
127
+ ]
128
+
129
+ toxic_scores = [
130
+ float(score)
131
+ for cat, score in labels.items()
132
+ if str(cat).lower() in TOXIC_ML_LABELS
133
+ ]
134
+ confidence = max(
135
+ [float(ml_result.get("toxicity_probability", 0.0))] + toxic_scores
136
+ )
137
+
138
+ toxic = (
139
+ bool(ml_result.get("toxic"))
140
+ or label in TOXIC_ML_LABELS
141
+ or any(c in TOXIC_ML_LABELS for c in detected_categories)
142
+ or confidence >= threshold
143
+ )
144
+
145
+ if confidence >= settings.severity_high:
146
+ severity = "high"
147
+ elif confidence >= settings.severity_medium:
148
+ severity = "medium"
149
+ else:
150
+ severity = "low"
151
+
152
+ category = "safe"
153
+ if toxic:
154
+ if detected_categories:
155
+ category = detected_categories[0]
156
+ elif label and label not in ("clean", "safe") and label in TOXIC_ML_LABELS:
157
+ category = label
158
+ else:
159
+ category = "toxic"
160
+
161
+ return {
162
+ "toxic": toxic,
163
+ "confidence": round(confidence, 3),
164
+ "severity": severity,
165
+ "category": category,
166
+ "detected_categories": detected_categories,
167
+ }
168
+
169
+
170
+ # ──────────────────────────────────────────────────────────────────────────────
171
+ # WEIGHTED ENSEMBLE SCORER
172
+ # ──────────────────────────────────────────────────────────────────────────────
173
+
174
+ def compute_ensemble_score(
175
+ rules_triggered: bool,
176
+ rules_severity: str,
177
+ ml_probability: float,
178
+ llm_toxic: bool,
179
+ llm_confidence: float,
180
+ model_type: str,
181
+ ) -> tuple[bool, float]:
182
+ """
183
+ Three-layer weighted ensemble scoring.
184
+
185
+ Weights (transformer path):
186
+ DeBERTa 50% — primary model, highest accuracy signal
187
+ LLM 35% — contextual understanding, sarcasm detection
188
+ Rules 15% — deterministic, fast, explainable
189
+
190
+ When LLM was not called (cost/rate-limit), the 35% is redistributed:
191
+ DeBERTa gets 75%, Rules get 25%.
192
+
193
+ Returns:
194
+ (is_toxic: bool, confidence: float)
195
+ """
196
+ # Rule score from severity tier
197
+ rule_score = 0.0
198
+ if rules_triggered:
199
+ rule_score = {"critical": 1.0, "high": 0.85, "moderate": 0.55}.get(
200
+ rules_severity, 0.6
201
+ )
202
+
203
+ ml_score = ml_probability
204
+
205
+ llm_score = llm_confidence if llm_toxic else (1.0 - llm_confidence) * 0.1
206
+
207
+ # Layer weights — DeBERTa is primary, trust it more
208
+ if model_type.startswith("transformer"):
209
+ w_rules, w_ml, w_llm = 0.15, 0.50, 0.35
210
+ else:
211
+ # Legacy ML is less reliable — lean heavily on LLM
212
+ w_rules, w_ml, w_llm = 0.25, 0.15, 0.60
213
+
214
+ # LLM not called → redistribute its weight
215
+ llm_was_called = not (llm_confidence == 0.0 and not llm_toxic)
216
+ if not llm_was_called:
217
+ # Redistribute LLM weight proportionally to remaining layers
218
+ w_ml += w_llm * 0.75
219
+ w_rules += w_llm * 0.25
220
+ w_llm = 0.0
221
+
222
+ total_weight = w_rules + w_ml + w_llm
223
+ weighted = (
224
+ (w_rules * rule_score + w_ml * ml_score + w_llm * llm_score)
225
+ / total_weight
226
+ )
227
+
228
+ # Hard overrides to prevent false negatives from dragging down strong signals:
229
+ # 1. Critical rule always forces high confidence
230
+ if rules_triggered and rules_severity == "critical":
231
+ weighted = max(weighted, 0.90)
232
+
233
+ # 2. High rule severity always forces score above threshold
234
+ if rules_triggered and rules_severity == "high":
235
+ current_threshold = calibration_service.get_threshold()
236
+ weighted = max(weighted, current_threshold + 0.05)
237
+
238
+ # 3. High-confidence LLM classification (e.g. for multilingual/code-mixed inputs) always forces score above threshold
239
+ if llm_toxic and llm_confidence >= 0.75:
240
+ current_threshold = calibration_service.get_threshold()
241
+ weighted = max(weighted, current_threshold + 0.05)
242
+
243
+ is_toxic = weighted >= calibration_service.get_threshold()
244
+ return is_toxic, round(weighted, 3)
245
+
246
+
247
+ # ──────────────────────────────────────────────────────────────────────────────
248
+ # PUBLIC DEMO ENDPOINT
249
+ # ──────────────────────────────────────────────────────────────────────────────
250
+
251
+ @router.post("/predict/demo", tags=["Demo"])
252
+ @limiter.limit("5/minute")
253
+ def predict_demo(request: Request, req: TextRequest = Body(...)):
254
+ """
255
+ Public toxicity check — no auth required.
256
+ IP rate-limited to 5 req/min. Uses ML + rule engine (no LLM).
257
+ Intended for browser extension demo mode.
258
+ """
259
+ text = req.text.strip()
260
+ if not text:
261
+ return {"error": "Empty input"}
262
+
263
+ from app.services.overrides import override_service
264
+ override = override_service.check_override(text)
265
+ if override is not None:
266
+ return {
267
+ "toxic": override,
268
+ "confidence": 1.0 if override else 0.0,
269
+ "severity": "high" if override else "low",
270
+ "category": "override" if override else "safe",
271
+ "detected_categories": ["override"] if override else [],
272
+ "abusive_words": [],
273
+ "source": "overrides",
274
+ "demo": True,
275
+ "llm_used": False,
276
+ "model_info": {"type": "dynamic_overrides", "ready": True},
277
+ }
278
+
279
+ rule_text = preprocess_for_rules(text)
280
+ model_text = preprocess_for_model(text)
281
+
282
+ # Rule engine on normalised text
283
+ abusive_hits = detect_abusive_tokens(rule_text)
284
+ rules_triggered = len(abusive_hits) > 0
285
+ rules_severity = get_abuse_severity(abusive_hits) if rules_triggered else "low"
286
+
287
+ # DeBERTa on unicode-safe text
288
+ ml_result = model_service.predict(model_text)
289
+ if not ml_result:
290
+ return {
291
+ "toxic": rules_triggered,
292
+ "confidence": 0.6 if rules_triggered else 0.0,
293
+ "severity": rules_severity if rules_triggered else "low",
294
+ "category": "abusive" if rules_triggered else "safe",
295
+ "detected_categories": abusive_hits,
296
+ "abusive_words": abusive_hits,
297
+ "source": "rules" if rules_triggered else "none",
298
+ "demo": True,
299
+ "llm_used": False,
300
+ "model_info": {"type": model_service.model_type, "ready": False},
301
+ }
302
+
303
+ normalized = normalize_ml_result(ml_result, threshold=settings.demo_ml_threshold)
304
+
305
+ # Light ensemble for demo: rules + ML only
306
+ if rules_triggered and rules_severity in ("critical", "high"):
307
+ toxic = True
308
+ confidence = max(normalized["confidence"], 0.80)
309
+ else:
310
+ toxic = normalized["toxic"] or rules_triggered
311
+ confidence = normalized["confidence"]
312
+
313
+ if confidence >= settings.severity_high:
314
+ severity = "high"
315
+ elif confidence >= settings.severity_medium:
316
+ severity = "medium"
317
+ else:
318
+ severity = "low"
319
+
320
+ category = normalized["category"] if toxic else "safe"
321
+ all_detected = list(set(normalized["detected_categories"] + abusive_hits))
322
+
323
+ return {
324
+ "toxic": toxic,
325
+ "confidence": round(confidence, 3),
326
+ "severity": severity,
327
+ "category": category,
328
+ "detected_categories": all_detected,
329
+ "abusive_words": abusive_hits,
330
+ "source": "rules" if rules_triggered else ("ml" if toxic else "none"),
331
+ "demo": True,
332
+ "llm_used": False,
333
+ "reason": (
334
+ f"[Demo] Detected {category} content."
335
+ if toxic
336
+ else "[Demo] No toxicity detected."
337
+ ),
338
+ "ml": ml_result,
339
+ "model_info": {
340
+ "type": model_service.model_type,
341
+ "ready": True,
342
+ },
343
+ }
344
+
345
+
346
+ # ──────────────────────────────────────────────────────────────────────────────
347
+ # ML-ONLY ENDPOINT (Extension fast-path)
348
+ # ──────────────────────────────────────────────────────────────────────────────
349
+
350
+ @router.post("/predict/ml")
351
+ @limiter.limit("120/minute")
352
+ def predict_ml_only(
353
+ request: Request,
354
+ req: TextRequest = Body(...),
355
+ user: User = Depends(get_api_user),
356
+ db: Session = Depends(get_db),
357
+ ):
358
+ """
359
+ Extension-safe prediction — authenticated, ML + rules only.
360
+ Skips LLM to avoid token costs on bulk scans.
361
+ """
362
+ text = req.text.strip()
363
+ if not text:
364
+ return {"error": "Empty input"}
365
+
366
+ _check_plan_limit(user, db)
367
+ logger.info(f"[/predict/ml] user={user.email} usage={user.usage_count}")
368
+
369
+ from app.services.overrides import override_service
370
+ override = override_service.check_override(text)
371
+ if override is not None:
372
+ return {
373
+ "user": user.email,
374
+ "toxic": override,
375
+ "confidence": 1.0 if override else 0.0,
376
+ "severity": "high" if override else "low",
377
+ "source": "overrides",
378
+ "reason": "Dynamic override applied based on feedback.",
379
+ "abusive_words": [],
380
+ "detected_categories": ["override"] if override else [],
381
+ "sentiment": None,
382
+ "ml": None,
383
+ "llm": None,
384
+ "rules": {"triggered": override, "severity": "high" if override else "low"},
385
+ "llm_used": False,
386
+ "model_info": {"type": "dynamic_overrides", "ready": True},
387
+ }
388
+
389
+ rule_text = preprocess_for_rules(text)
390
+ model_text = preprocess_for_model(text)
391
+
392
+ abusive_hits = detect_abusive_tokens(rule_text)
393
+ rules_triggered = len(abusive_hits) > 0
394
+ rules_severity = get_abuse_severity(abusive_hits) if rules_triggered else "low"
395
+
396
+ ml_result = model_service.predict(model_text)
397
+ if not ml_result:
398
+ raise HTTPException(status_code=503, detail="ML model unavailable")
399
+
400
+ normalized = normalize_ml_result(ml_result)
401
+
402
+ # Simple rules + ML decision (no LLM)
403
+ is_toxic = normalized["toxic"] or (rules_triggered and rules_severity in ("critical", "high"))
404
+ confidence = normalized["confidence"]
405
+ if rules_triggered and rules_severity == "critical":
406
+ confidence = max(confidence, 0.90)
407
+
408
+ if confidence >= settings.severity_high:
409
+ severity = "high"
410
+ elif confidence >= settings.severity_medium:
411
+ severity = "medium"
412
+ else:
413
+ severity = "low"
414
+
415
+ payload = {
416
+ "user": user.email,
417
+ "toxic": is_toxic,
418
+ "confidence": round(confidence, 3),
419
+ "severity": severity,
420
+ "source": "rules" if rules_triggered else ("ml" if normalized["toxic"] else "none"),
421
+ "reason": (
422
+ f"ML model flagged as {normalized['category']}."
423
+ if normalized["toxic"]
424
+ else "No toxicity detected by ML model."
425
+ ),
426
+ "abusive_words": abusive_hits,
427
+ "detected_categories": list(set(normalized["detected_categories"] + abusive_hits)),
428
+ "sentiment": None,
429
+ "ml": ml_result,
430
+ "llm": None,
431
+ "rules": {"triggered": rules_triggered, "severity": rules_severity},
432
+ "llm_used": False,
433
+ "model_info": model_service.status,
434
+ }
435
+ return _build_response(payload)
436
+
437
+
438
+ # ──────────────────────────────────────────────────────────────────────────────
439
+ # MAIN FULL-PIPELINE ENDPOINT
440
+ # ──────────────────────────────────────────────────────────────────────────────
441
+
442
+ @router.post("/predict")
443
+ @limiter.limit("10/minute")
444
+ def predict(
445
+ request: Request,
446
+ req: TextRequest = Body(...),
447
+ user: User = Depends(get_api_user),
448
+ db: Session = Depends(get_db),
449
+ ):
450
+ """
451
+ Full three-layer toxicity analysis:
452
+ 1. Rule-based keyword detection
453
+ 2. DeBERTa-v3 transformer (ONNX or PyTorch)
454
+ 3. LLM contextual analysis (OpenRouter) — triggered selectively
455
+ """
456
+ text = req.text.strip()
457
+ if not text:
458
+ return {"error": "Empty input"}
459
+
460
+ _check_plan_limit(user, db)
461
+ logger.info(f"[/predict] user={user.email} usage={user.usage_count}")
462
+
463
+ from app.services.overrides import override_service
464
+ override = override_service.check_override(text)
465
+ if override is not None:
466
+ payload = {
467
+ "user": user.email,
468
+ "toxic": override,
469
+ "confidence": 1.0 if override else 0.0,
470
+ "severity": "high" if override else "low",
471
+ "source": "overrides",
472
+ "reason": "Dynamic override applied based on feedback.",
473
+ "abusive_words": [],
474
+ "detected_categories": ["override"] if override else [],
475
+ "sentiment": {"label": "negative" if override else "positive", "polarity": -1.0 if override else 1.0},
476
+ "ml": None,
477
+ "llm": None,
478
+ "rules": {"triggered": override, "severity": "high" if override else "low"},
479
+ "llm_used": False,
480
+ "model_info": {"type": "dynamic_overrides", "ready": True},
481
+ }
482
+ return _build_response(payload)
483
+
484
+ # ── PREPROCESS ────────────────────────────────────────────────────────────
485
+ rule_text = preprocess_for_rules(text)
486
+ model_text = preprocess_for_model(text)
487
+
488
+ # ── SENTIMENT ─────────────────────────────────────────────────────────────
489
+ sentiment = analyze_sentiment(rule_text)
490
+
491
+ # ── RULE ENGINE ───────────────────────────────────────────────────────────
492
+ abusive_hits = detect_abusive_tokens(rule_text)
493
+ rules_triggered = len(abusive_hits) > 0
494
+ rules_severity = get_abuse_severity(abusive_hits) if rules_triggered else "low"
495
+
496
+ # ── DeBERTa ML ────────────────────────────────────────────────────────────
497
+ ml_result = model_service.predict(model_text)
498
+ toxic_probability = 0.0
499
+ if ml_result:
500
+ toxic_probability = float(ml_result.get("toxicity_probability", 0.0))
501
+
502
+ # ── LLM CALL (selective — saves cost & latency) ────────────────────────
503
+ _DEFAULT_LLM = {
504
+ "toxic": False,
505
+ "confidence": 0.0,
506
+ "category": "safe",
507
+ "detected_phrases": [],
508
+ "explanation": "Analysis based on rule engine and ML model signals.",
509
+ }
510
+ llm_result = _DEFAULT_LLM.copy()
511
+
512
+ should_call_llm = len(text) > 10 and (
513
+ rules_triggered or toxic_probability >= settings.ml_trigger_threshold
514
+ )
515
+
516
+ if should_call_llm:
517
+ try:
518
+ llm_result = analyze_toxicity_llm(text)
519
+ except Exception as exc:
520
+ logger.warning(f"[/predict] LLM call failed: {exc}")
521
+
522
+ llm_toxic = llm_result.get("toxic", False)
523
+ llm_conf = llm_result.get("confidence", 0.0)
524
+
525
+ # ── WEIGHTED ENSEMBLE ─────────────────────────────────────────────────────
526
+ toxic, final_confidence = compute_ensemble_score(
527
+ rules_triggered=rules_triggered,
528
+ rules_severity=rules_severity,
529
+ ml_probability=toxic_probability,
530
+ llm_toxic=llm_toxic,
531
+ llm_confidence=llm_conf,
532
+ model_type=model_service.model_type,
533
+ )
534
+
535
+ # ── SEVERITY ──────────────────────────────────────────────────────────────
536
+ if final_confidence >= settings.severity_high:
537
+ severity = "high"
538
+ elif final_confidence >= settings.severity_medium:
539
+ severity = "medium"
540
+ else:
541
+ severity = "low"
542
+
543
+ # ── SENTIMENT OVERRIDE ────────────────────────────────────────────────────
544
+ sentiment = _adjust_sentiment(sentiment, toxic)
545
+
546
+ # ── ABUSIVE WORDS (merge rule + LLM phrases) ──────────────────────────────
547
+ abusive_words = list(set(abusive_hits + llm_result.get("detected_phrases", [])))
548
+
549
+ # ── DETECTED CATEGORIES ───────────────────────────────────────────────────
550
+ ml_cats = [str(c).lower() for c in (ml_result or {}).get("detected_categories", [])]
551
+ all_categories = list(set(ml_cats + abusive_hits))
552
+
553
+ # ── SOURCE ATTRIBUTION ────────────────────────────────────────────────────
554
+ if llm_toxic:
555
+ source = "llm"
556
+ elif rules_triggered:
557
+ source = "rules"
558
+ elif toxic_probability >= 0.5:
559
+ source = "ml"
560
+ else:
561
+ source = "none"
562
+
563
+ # ── EXPLANATION ───────────────────────────────────────────────────────────
564
+ reason = llm_result.get("explanation") or "No explanation available."
565
+
566
+ llm_was_used = should_call_llm and (
567
+ llm_result.get("explanation") != _DEFAULT_LLM["explanation"]
568
+ )
569
+
570
+ # ── RESPONSE ──────────────────────────────────────────────────────────────
571
+ payload = {
572
+ "user": user.email,
573
+ "toxic": toxic,
574
+ "confidence": final_confidence,
575
+ "severity": severity,
576
+ "source": source,
577
+ "reason": reason,
578
+ "abusive_words": abusive_words,
579
+ "detected_categories": all_categories,
580
+ "sentiment": sentiment,
581
+ "ml": ml_result,
582
+ "llm": llm_result,
583
+ "rules": {"triggered": rules_triggered, "severity": rules_severity},
584
+ "llm_used": llm_was_used,
585
+ "model_info": model_service.status,
586
+ }
587
+
588
+ # ── DRIFT MONITORING LOG ──────────────────────────────────────────────────
589
+ try:
590
+ drift_monitor.log_prediction(
591
+ db,
592
+ user_id=user.id,
593
+ input_text=text,
594
+ confidence=final_confidence,
595
+ toxic=toxic,
596
+ severity=severity,
597
+ source=source,
598
+ )
599
+ except Exception as _log_exc:
600
+ logger.warning(f"[/predict] Drift log failed (non-fatal): {_log_exc}")
601
+
602
+ return _build_response(payload)
603
+
604
+
605
+ # ──────────────────────────────────────────────────────────────────────────────
606
+ # BATCH FILE ANALYSIS
607
+ # ──────────────────────────────────────────────────────────────────────────────
608
+
609
+ @router.post("/analyze-file")
610
+ async def analyze_file(
611
+ file: UploadFile = File(...),
612
+ user: User = Depends(get_api_user),
613
+ db: Session = Depends(get_db),
614
+ ):
615
+ """
616
+ Batch toxicity analysis from CSV or TXT file upload.
617
+ Uses the shared ensemble (rules + ML) — no LLM for batch speed.
618
+ Capped at 50 texts per request.
619
+ """
620
+ try:
621
+ content = await file.read()
622
+
623
+ if file.filename and file.filename.endswith(".csv"):
624
+ df = pd.read_csv(io.BytesIO(content))
625
+ # Auto-detect text column
626
+ text_col = next(
627
+ (c for c in df.columns if "text" in c.lower() or "comment" in c.lower()),
628
+ df.columns[0],
629
+ )
630
+ texts = df[text_col].fillna("").astype(str).tolist()
631
+ else:
632
+ texts = content.decode("utf-8").splitlines()
633
+
634
+ batch = [t for t in texts[:50] if t.strip()]
635
+ results = []
636
+ toxic_count = 0
637
+
638
+ for text in batch:
639
+ rule_text = preprocess_for_rules(text)
640
+ model_text = preprocess_for_model(text)
641
+
642
+ abusive_hits = detect_abusive_tokens(rule_text)
643
+ rules_triggered = len(abusive_hits) > 0
644
+ rules_severity = get_abuse_severity(abusive_hits) if rules_triggered else "low"
645
+
646
+ ml_result = model_service.predict(model_text)
647
+ ml_prob = float(ml_result.get("toxicity_probability", 0.0)) if ml_result else 0.0
648
+ ml_cats = [str(c).lower() for c in (ml_result or {}).get("detected_categories", [])]
649
+
650
+ # Shared ensemble (no LLM for batch speed)
651
+ is_toxic, confidence = compute_ensemble_score(
652
+ rules_triggered=rules_triggered,
653
+ rules_severity=rules_severity,
654
+ ml_probability=ml_prob,
655
+ llm_toxic=False,
656
+ llm_confidence=0.0,
657
+ model_type=model_service.model_type,
658
+ )
659
+
660
+ if confidence >= settings.severity_high:
661
+ severity = "high"
662
+ elif confidence >= settings.severity_medium:
663
+ severity = "medium"
664
+ else:
665
+ severity = "low"
666
+
667
+ if is_toxic:
668
+ toxic_count += 1
669
+
670
+ sentiment = analyze_sentiment(rule_text)
671
+ if is_toxic:
672
+ sentiment["label"] = "negative"
673
+
674
+ source = "rules" if rules_triggered else ("ml" if ml_prob >= 0.5 else "none")
675
+ all_categories = list(set(ml_cats + abusive_hits))
676
+
677
+ results.append({
678
+ "text": text,
679
+ "toxic": is_toxic,
680
+ "confidence": round(confidence, 3),
681
+ "severity": severity,
682
+ "source": source,
683
+ "abusive_words": abusive_hits,
684
+ "detected_categories": all_categories,
685
+ "sentiment": sentiment,
686
+ })
687
+
688
+ return {
689
+ "total": len(results),
690
+ "toxic_count": toxic_count,
691
+ "clean_count": len(results) - toxic_count,
692
+ "toxic_rate": round(toxic_count / len(results), 3) if results else 0.0,
693
+ "model_info": model_service.status,
694
+ "results": results,
695
+ }
696
+
697
+ except Exception as exc:
698
+ logger.error(f"[/analyze-file] Failed: {exc}")
699
+ raise HTTPException(status_code=400, detail=f"Failed to process file: {str(exc)}")
app/routes/monitoring.py ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — Monitoring Endpoints
3
+ =====================================
4
+ GET /monitoring/stats — Live model health metrics + drift alert
5
+ GET /monitoring/drift — Time-series confidence data for charting
6
+ """
7
+
8
+ from fastapi import APIRouter, Depends, Request
9
+ from sqlalchemy.orm import Session
10
+
11
+ from app.routes.auth import get_api_user
12
+ from app.core.limiter import limiter
13
+ from app.core.logger import logger
14
+ from app.services.drift_monitor import drift_monitor
15
+ from models import User
16
+ from database import get_db
17
+
18
+ router = APIRouter()
19
+
20
+
21
+ @router.get("/monitoring/stats", tags=["Monitoring"])
22
+ @limiter.limit("30/minute")
23
+ def get_monitoring_stats(
24
+ request: Request,
25
+ user: User = Depends(get_api_user),
26
+ db: Session = Depends(get_db),
27
+ ):
28
+ """
29
+ Live model monitoring dashboard data.
30
+
31
+ Returns rolling window statistics, drift detection result,
32
+ toxic rate, severity breakdown, and hourly request volume.
33
+ """
34
+ logger.info(f"[/monitoring/stats] user={user.email}")
35
+ return drift_monitor.get_stats(db)
36
+
37
+
38
+ @router.get("/monitoring/drift", tags=["Monitoring"])
39
+ @limiter.limit("20/minute")
40
+ def get_drift_series(
41
+ request: Request,
42
+ limit: int = 100,
43
+ user: User = Depends(get_api_user),
44
+ db: Session = Depends(get_db),
45
+ ):
46
+ """
47
+ Time-series of confidence scores for the drift trend chart.
48
+ Use `limit` query param (default 100, max 500).
49
+ """
50
+ limit = min(limit, 500)
51
+ logger.info(f"[/monitoring/drift] user={user.email} limit={limit}")
52
+ return {
53
+ "series": drift_monitor.get_drift_series(db, limit=limit),
54
+ "limit": limit,
55
+ }
app/routes/realtime.py ADDED
@@ -0,0 +1,135 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — Real-Time Chat Moderation
3
+ ==========================================
4
+ POST /chat/moderate — Authenticated, rate-limited, fast chat moderation.
5
+
6
+ Uses DeBERTa + rule engine for fast decision.
7
+ Calls LLM only when toxicity is confirmed (no wasted tokens on clean messages).
8
+ """
9
+
10
+ from fastapi import APIRouter, Depends, Request, Body
11
+ from sqlalchemy.orm import Session
12
+ from pydantic import BaseModel
13
+ from datetime import datetime
14
+
15
+ from app.routes.auth import get_api_user
16
+ from app.core.limiter import limiter
17
+ from app.core.logger import logger
18
+ from app.core.config import settings
19
+
20
+ from utils.preprocessing import preprocess_for_rules, preprocess_for_model
21
+ from utils.abuse_words import detect_abusive_tokens, get_abuse_severity
22
+ from utils.llm_guard import analyze_toxicity_llm
23
+ from app.services.model_service import model_service
24
+
25
+ from app.services.calibration import calibration_service
26
+
27
+ from models import User
28
+ from database import get_db
29
+
30
+ router = APIRouter()
31
+
32
+
33
+ # ──────────────────────────────────────────────────────────────────────────────
34
+ # SCHEMA
35
+ # ──────────────────────────────────────────────────────────────────────────────
36
+
37
+ class ChatRequest(BaseModel):
38
+ message: str
39
+
40
+
41
+ # ──────────────────────────────────────────────────────────────────────────────
42
+ # ENDPOINT
43
+ # ──────────────────────────────────────────────────────────────────────────────
44
+
45
+ @router.post("/chat/moderate")
46
+ @limiter.limit("30/minute")
47
+ def moderate_chat(
48
+ request: Request,
49
+ req: ChatRequest = Body(...),
50
+ user: User = Depends(get_api_user),
51
+ db: Session = Depends(get_db),
52
+ ):
53
+ """
54
+ Real-time chat message moderation.
55
+
56
+ Decision flow:
57
+ 1. Rule engine — instant check for known slurs
58
+ 2. DeBERTa — ML probability score
59
+ 3. LLM — called only when content is flagged (avoids wasted tokens)
60
+ """
61
+ text = req.message.strip()
62
+ if not text:
63
+ return {"error": "Empty message"}
64
+
65
+ rule_text = preprocess_for_rules(text)
66
+ model_text = preprocess_for_model(text)
67
+
68
+ # Track last activity for analytics
69
+ user.last_used = datetime.utcnow()
70
+ db.commit()
71
+
72
+ # ── RULE ENGINE ───────────────────────────────────────────────────────────
73
+ abusive_hits = detect_abusive_tokens(rule_text)
74
+ rules_triggered = len(abusive_hits) > 0
75
+ rules_severity = get_abuse_severity(abusive_hits) if rules_triggered else "low"
76
+
77
+ # ── DeBERTa ───────────────────────────────────────────────────────────────
78
+ ml_result = model_service.predict(model_text)
79
+ ml_score = 0.0
80
+ ml_categories: list[str] = []
81
+ if ml_result:
82
+ ml_score = float(ml_result.get("toxicity_probability", 0.0))
83
+ ml_categories = [str(c).lower() for c in ml_result.get("detected_categories", [])]
84
+
85
+ # ── FAST DECISION ─────────────────────────────────────────────────────────
86
+ # Threshold — dynamically adjusted based on feedback calibration
87
+ toxic = rules_triggered or ml_score >= calibration_service.get_threshold()
88
+
89
+ # Critical rule always overrides
90
+ if rules_triggered and rules_severity == "critical":
91
+ toxic = True
92
+
93
+ # ── CONFIDENCE ────────────────────────────────────────────────────────────
94
+ confidence = ml_score
95
+ if rules_triggered:
96
+ rule_boost = {"critical": 0.95, "high": 0.85, "moderate": 0.65}.get(rules_severity, 0.6)
97
+ confidence = max(confidence, rule_boost)
98
+
99
+ # ── SEVERITY ──────────────────────────────────────────────────────────────
100
+ if confidence >= settings.severity_high:
101
+ severity = "high"
102
+ elif confidence >= settings.severity_medium:
103
+ severity = "medium"
104
+ else:
105
+ severity = "low"
106
+
107
+ # ── LLM (only if toxic and text is meaningful) ───────────────────���────────
108
+ explanation = "Message appears safe."
109
+ if toxic and len(text) > 10:
110
+ try:
111
+ llm = analyze_toxicity_llm(text)
112
+ explanation = llm.get("explanation", "Toxic content detected.")
113
+ except Exception as exc:
114
+ logger.warning(f"[/chat/moderate] LLM failed: {exc}")
115
+ explanation = "Toxic content detected (explanation unavailable)."
116
+
117
+ # ── DETECTED CATEGORIES ───────────────────────────────────────────────────
118
+ all_categories = list(set(ml_categories + abusive_hits))
119
+
120
+ logger.info(
121
+ f"[/chat/moderate] user={user.email} | toxic={toxic} | "
122
+ f"ml={ml_score:.3f} | rules={rules_triggered}"
123
+ )
124
+
125
+ return {
126
+ "message": text,
127
+ "toxic": toxic,
128
+ "confidence": round(confidence, 3),
129
+ "severity": severity,
130
+ "abusive_words": abusive_hits,
131
+ "detected_categories": all_categories,
132
+ "rules": {"triggered": rules_triggered, "severity": rules_severity},
133
+ "explanation": explanation,
134
+ "model_info": model_service.status,
135
+ }
app/routes/stream.py ADDED
@@ -0,0 +1,245 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — Real-Time WebSocket Stream Analyzer
3
+ ====================================================
4
+ WebSocket /ws/stream — Accepts a continuous stream of text messages,
5
+ runs full toxicity analysis on each, and pushes results back instantly.
6
+
7
+ Protocol:
8
+ 1. Client connects to ws://host/ws/stream
9
+ 2. Client sends first message: {"type": "auth", "api_key": "tg_..."}
10
+ 3. Server responds: {"type": "auth_ok", "user": "email@example.com"}
11
+ 4. Client sends text messages: {"type": "message", "text": "hello world", "id": "msg_1"}
12
+ 5. Server streams back: {"type": "result", "id": "msg_1", "toxic": false, ...}
13
+ 6. Client sends {"type": "ping"} → server sends {"type": "pong"}
14
+ 7. Client disconnects normally
15
+
16
+ Throughput: ~50 msg/sec on CPU (ONNX inference ~20ms/msg)
17
+ """
18
+
19
+ import json
20
+ import time
21
+ from datetime import datetime
22
+
23
+ from fastapi import APIRouter, WebSocket, WebSocketDisconnect
24
+ from sqlalchemy.orm import Session
25
+
26
+ from app.core.logger import logger
27
+ from app.core.config import settings
28
+ from app.services.calibration import calibration_service
29
+ from app.services.model_service import model_service
30
+ from app.services.drift_monitor import drift_monitor
31
+ from database import SessionLocal, get_db
32
+
33
+ from utils.preprocessing import preprocess_for_rules, preprocess_for_model
34
+ from utils.abuse_words import detect_abusive_tokens, get_abuse_severity
35
+ from models import User
36
+
37
+ router = APIRouter()
38
+
39
+
40
+ async def _authenticate(websocket: WebSocket, data: dict) -> User | None:
41
+ """Validate API key from first WebSocket message. Returns User or None."""
42
+ from database import SessionLocal
43
+ from models import User
44
+
45
+ api_key = data.get("api_key", "").strip()
46
+ if not api_key:
47
+ return None
48
+
49
+ db = SessionLocal()
50
+ try:
51
+ user = db.query(User).filter(User.api_key == api_key).first()
52
+ return user
53
+ except Exception:
54
+ return None
55
+ finally:
56
+ db.close()
57
+
58
+
59
+ def _analyze_message(text: str) -> dict:
60
+ """Run full toxicity pipeline on a single message. Returns result dict."""
61
+ start = time.perf_counter()
62
+
63
+ rule_text = preprocess_for_rules(text)
64
+ model_text = preprocess_for_model(text)
65
+
66
+ # Rule engine
67
+ abusive_hits = detect_abusive_tokens(rule_text)
68
+ rules_triggered = len(abusive_hits) > 0
69
+ rules_severity = get_abuse_severity(abusive_hits) if rules_triggered else "low"
70
+
71
+ # DeBERTa ONNX
72
+ ml_result = model_service.predict(model_text)
73
+ ml_score = 0.0
74
+ labels = {}
75
+ if ml_result:
76
+ ml_score = float(ml_result.get("toxicity_probability", 0.0))
77
+ labels = ml_result.get("labels", {})
78
+
79
+ # Decision
80
+ confidence = ml_score
81
+ if rules_triggered:
82
+ rule_boost = {"critical": 0.95, "high": 0.85, "moderate": 0.65}.get(rules_severity, 0.6)
83
+ confidence = max(confidence, rule_boost)
84
+
85
+ toxic = rules_triggered or ml_score >= calibration_service.get_threshold()
86
+
87
+ severity = (
88
+ "high" if confidence >= settings.severity_high
89
+ else "medium" if confidence >= settings.severity_medium
90
+ else "low"
91
+ )
92
+
93
+ latency_ms = round((time.perf_counter() - start) * 1000, 1)
94
+
95
+ return {
96
+ "toxic": toxic,
97
+ "confidence": round(confidence, 3),
98
+ "severity": severity,
99
+ "abusive_words": abusive_hits,
100
+ "rules_triggered": rules_triggered,
101
+ "ml_score": round(ml_score, 3),
102
+ "labels": labels,
103
+ "latency_ms": latency_ms,
104
+ }
105
+
106
+
107
+ @router.websocket("/ws/stream")
108
+ async def websocket_stream(websocket: WebSocket):
109
+ """
110
+ Real-time toxicity stream analyzer.
111
+
112
+ Accepts continuous messages, analyzes each with DeBERTa + rules,
113
+ and pushes results back with <50ms latency.
114
+ """
115
+ await websocket.accept()
116
+ user = None
117
+ message_count = 0
118
+ session_start = time.time()
119
+
120
+ logger.info(f"[WS] New connection from {websocket.client.host}")
121
+
122
+ try:
123
+ # ── STEP 1: Auth handshake ─────────────────────────────────────────────
124
+ raw = await websocket.receive_text()
125
+ try:
126
+ auth_data = json.loads(raw)
127
+ except json.JSONDecodeError:
128
+ await websocket.send_json({
129
+ "type": "error",
130
+ "message": "Invalid JSON in auth message"
131
+ })
132
+ await websocket.close(code=1003)
133
+ return
134
+
135
+ if auth_data.get("type") != "auth":
136
+ await websocket.send_json({
137
+ "type": "error",
138
+ "message": "First message must be: {type: 'auth', api_key: 'tg_...'}"
139
+ })
140
+ await websocket.close(code=1008)
141
+ return
142
+
143
+ user = await _authenticate(websocket, auth_data)
144
+ if not user:
145
+ await websocket.send_json({
146
+ "type": "auth_error",
147
+ "message": "Invalid API key"
148
+ })
149
+ await websocket.close(code=1008)
150
+ return
151
+
152
+ await websocket.send_json({
153
+ "type": "auth_ok",
154
+ "user": user.email,
155
+ "message": "Connected to ToxiGuard AI stream",
156
+ "model": model_service.status.get("description", "unknown"),
157
+ })
158
+ logger.info(f"[WS] Authenticated: {user.email}")
159
+
160
+ # ── STEP 2: Message loop ───────────────────────────────────────────────
161
+ while True:
162
+ raw = await websocket.receive_text()
163
+
164
+ try:
165
+ msg = json.loads(raw)
166
+ except json.JSONDecodeError:
167
+ await websocket.send_json({"type": "error", "message": "Invalid JSON"})
168
+ continue
169
+
170
+ msg_type = msg.get("type", "message")
171
+
172
+ # Ping/pong keepalive
173
+ if msg_type == "ping":
174
+ await websocket.send_json({"type": "pong", "ts": int(time.time())})
175
+ continue
176
+
177
+ # Disconnect request
178
+ if msg_type == "disconnect":
179
+ break
180
+
181
+ # Text analysis
182
+ if msg_type == "message":
183
+ text = str(msg.get("text", "")).strip()
184
+ msg_id = msg.get("id", str(message_count))
185
+
186
+ if not text:
187
+ await websocket.send_json({
188
+ "type": "result",
189
+ "id": msg_id,
190
+ "error": "Empty text"
191
+ })
192
+ continue
193
+
194
+ # Enforce max length
195
+ if len(text) > 1000:
196
+ text = text[:1000]
197
+
198
+ # Run analysis
199
+ result = _analyze_message(text)
200
+ message_count += 1
201
+
202
+ # Log to drift monitor (background — don't await DB here)
203
+ try:
204
+ db = SessionLocal()
205
+ drift_monitor.log_prediction(
206
+ db,
207
+ user_id=user.id,
208
+ input_text=text,
209
+ confidence=result["confidence"],
210
+ toxic=result["toxic"],
211
+ severity=result["severity"],
212
+ source="websocket_stream",
213
+ )
214
+ db.close()
215
+ except Exception:
216
+ pass
217
+
218
+ await websocket.send_json({
219
+ "type": "result",
220
+ "id": msg_id,
221
+ "text": text,
222
+ "toxic": result["toxic"],
223
+ "confidence": result["confidence"],
224
+ "severity": result["severity"],
225
+ "abusive_words": result["abusive_words"],
226
+ "rules_triggered": result["rules_triggered"],
227
+ "latency_ms": result["latency_ms"],
228
+ "message_num": message_count,
229
+ })
230
+
231
+ except WebSocketDisconnect:
232
+ pass
233
+ except Exception as exc:
234
+ logger.error(f"[WS] Unexpected error: {exc}")
235
+ try:
236
+ await websocket.send_json({"type": "error", "message": str(exc)})
237
+ except Exception:
238
+ pass
239
+ finally:
240
+ elapsed = round(time.time() - session_start, 1)
241
+ user_email = user.email if user else "unauthenticated"
242
+ logger.info(
243
+ f"[WS] Disconnected: {user_email} | "
244
+ f"messages={message_count} | duration={elapsed}s"
245
+ )
app/services/__init__.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ # backend/app/services/__init__.py
2
+ # Marker file to make services a package
app/services/calibration.py ADDED
@@ -0,0 +1,211 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — Adaptive Threshold Calibration Service
3
+ =======================================================
4
+ Adjusts the toxicity decision threshold in real-time based on user feedback.
5
+ No full model retrain required — works by tuning the confidence cutoff.
6
+
7
+ Algorithm:
8
+ - Maintain a sliding window of recent feedback (FP vs FN counts)
9
+ - False Positive (FP): model too aggressive → raise threshold slightly
10
+ - False Negative (FN): model too lenient → lower threshold slightly
11
+ - Step size = CALIBRATION_STEP (default 0.01) per correction
12
+ - Threshold is clamped to [MIN_THRESHOLD, MAX_THRESHOLD]
13
+ - Persisted to calibration.json so it survives server restarts
14
+
15
+ This is called "online threshold calibration" — used in production at
16
+ scale where full retrains are expensive (Meta, Google CSAM detection, etc.)
17
+
18
+ Usage:
19
+ from app.services.calibration import calibration_service
20
+ calibration_service.apply_feedback("false_positive", confidence=0.72)
21
+ threshold = calibration_service.get_threshold()
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ import os
27
+ import json
28
+ import threading
29
+ from datetime import datetime
30
+ from typing import Optional
31
+
32
+ from app.core.logger import logger
33
+
34
+
35
+ # ──────────────────────────────────────────────────────────────────────────────
36
+ # CONFIG
37
+ # ──────────────────────────────────────────────────────────────────────────────
38
+
39
+ DEFAULT_THRESHOLD = 0.50
40
+ MIN_THRESHOLD = 0.30 # Never go below this — too many FN
41
+ MAX_THRESHOLD = 0.75 # Never go above this — too many FP
42
+ CALIBRATION_STEP = 0.01 # Threshold shift per single correction
43
+
44
+ # Path to persist threshold across restarts
45
+ _CALIBRATION_FILE = os.path.join(
46
+ os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
47
+ "data", "calibration.json"
48
+ )
49
+
50
+
51
+ # ──────────────────────────────────────────────────────────────────────────────
52
+ # SERVICE
53
+ # ──────────────────────────────────────────────────────────────────────────────
54
+
55
+ class AdaptiveCalibrationService:
56
+ """
57
+ Online threshold calibration from user feedback.
58
+ Thread-safe for concurrent FastAPI requests.
59
+ """
60
+
61
+ def __init__(self) -> None:
62
+ self._lock = threading.Lock()
63
+ self._threshold: float = DEFAULT_THRESHOLD
64
+ self._fp_count: int = 0 # Lifetime FP corrections
65
+ self._fn_count: int = 0 # Lifetime FN corrections
66
+ self._total_adjustments: int = 0
67
+ self._history: list[dict] = [] # Last 20 adjustments
68
+ self._last_updated: Optional[str] = None
69
+ self._load()
70
+
71
+ # ── PERSISTENCE ───────────────────────────────────────────────────────────
72
+
73
+ def _load(self) -> None:
74
+ """Load calibration state from disk (survives restarts)."""
75
+ try:
76
+ if os.path.exists(_CALIBRATION_FILE):
77
+ with open(_CALIBRATION_FILE, "r") as f:
78
+ data = json.load(f)
79
+ self._threshold = float(data.get("threshold", DEFAULT_THRESHOLD))
80
+ self._fp_count = int(data.get("fp_count", 0))
81
+ self._fn_count = int(data.get("fn_count", 0))
82
+ self._total_adjustments = int(data.get("total_adjustments", 0))
83
+ self._history = data.get("history", [])[-20:]
84
+ logger.info(
85
+ f"[Calibration] Loaded threshold={self._threshold:.3f} "
86
+ f"(FP={self._fp_count}, FN={self._fn_count})"
87
+ )
88
+ except Exception as exc:
89
+ logger.warning(f"[Calibration] Could not load calibration file: {exc}")
90
+
91
+ def _save(self) -> None:
92
+ """Persist calibration state to disk."""
93
+ try:
94
+ os.makedirs(os.path.dirname(_CALIBRATION_FILE), exist_ok=True)
95
+ with open(_CALIBRATION_FILE, "w") as f:
96
+ json.dump({
97
+ "threshold": self._threshold,
98
+ "fp_count": self._fp_count,
99
+ "fn_count": self._fn_count,
100
+ "total_adjustments": self._total_adjustments,
101
+ "history": self._history[-20:],
102
+ "last_updated": self._last_updated,
103
+ }, f, indent=2)
104
+ except Exception as exc:
105
+ logger.warning(f"[Calibration] Could not save calibration: {exc}")
106
+
107
+ # ── CORE ��─────────────────────────────────────────────────────────────────
108
+
109
+ def apply_feedback(
110
+ self,
111
+ feedback_type: str,
112
+ confidence_at_time: Optional[float] = None,
113
+ ) -> dict:
114
+ """
115
+ Adjust threshold based on one feedback event.
116
+
117
+ Args:
118
+ feedback_type: "false_positive" | "false_negative" | "confirmed_correct"
119
+ confidence_at_time: The model's confidence when this prediction was made.
120
+
121
+ Returns:
122
+ {"old_threshold": float, "new_threshold": float, "direction": str}
123
+ """
124
+ with self._lock:
125
+ old = self._threshold
126
+
127
+ if feedback_type == "false_positive":
128
+ # Model was too aggressive: raise threshold (harder to trigger toxic)
129
+ self._threshold = min(
130
+ MAX_THRESHOLD,
131
+ self._threshold + CALIBRATION_STEP
132
+ )
133
+ self._fp_count += 1
134
+ direction = "raised"
135
+
136
+ elif feedback_type == "false_negative":
137
+ # Model was too lenient: lower threshold (easier to trigger toxic)
138
+ self._threshold = max(
139
+ MIN_THRESHOLD,
140
+ self._threshold - CALIBRATION_STEP
141
+ )
142
+ self._fn_count += 1
143
+ direction = "lowered"
144
+
145
+ else:
146
+ # Confirmed correct — no threshold change
147
+ direction = "unchanged"
148
+
149
+ self._total_adjustments += 1
150
+ self._last_updated = datetime.utcnow().isoformat()
151
+
152
+ event = {
153
+ "ts": self._last_updated,
154
+ "type": feedback_type,
155
+ "old": round(old, 3),
156
+ "new": round(self._threshold, 3),
157
+ "confidence_at_time": confidence_at_time,
158
+ }
159
+ self._history.append(event)
160
+ self._history = self._history[-20:]
161
+
162
+ self._save()
163
+
164
+ logger.info(
165
+ f"[Calibration] {feedback_type} → threshold {old:.3f} → "
166
+ f"{self._threshold:.3f} ({direction})"
167
+ )
168
+
169
+ return {
170
+ "old_threshold": round(old, 3),
171
+ "new_threshold": round(self._threshold, 3),
172
+ "direction": direction,
173
+ "fp_count": self._fp_count,
174
+ "fn_count": self._fn_count,
175
+ }
176
+
177
+ def get_threshold(self) -> float:
178
+ """Get the current calibrated threshold."""
179
+ with self._lock:
180
+ return self._threshold
181
+
182
+ def get_status(self) -> dict:
183
+ """Return full calibration status for the monitoring dashboard."""
184
+ with self._lock:
185
+ return {
186
+ "current_threshold": round(self._threshold, 4),
187
+ "default_threshold": DEFAULT_THRESHOLD,
188
+ "shift_from_default": round(self._threshold - DEFAULT_THRESHOLD, 4),
189
+ "fp_corrections": self._fp_count,
190
+ "fn_corrections": self._fn_count,
191
+ "total_adjustments": self._total_adjustments,
192
+ "last_updated": self._last_updated,
193
+ "history": self._history[-10:],
194
+ "bounds": {"min": MIN_THRESHOLD, "max": MAX_THRESHOLD},
195
+ "step_size": CALIBRATION_STEP,
196
+ }
197
+
198
+ def reset(self) -> None:
199
+ """Reset threshold to default (admin action)."""
200
+ with self._lock:
201
+ self._threshold = DEFAULT_THRESHOLD
202
+ self._fp_count = 0
203
+ self._fn_count = 0
204
+ self._total_adjustments = 0
205
+ self._history = []
206
+ self._save()
207
+ logger.info("[Calibration] Threshold reset to default")
208
+
209
+
210
+ # Module-level singleton
211
+ calibration_service = AdaptiveCalibrationService()
app/services/drift_monitor.py ADDED
@@ -0,0 +1,288 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — Data Drift Detection & Model Monitoring Service
3
+ ===============================================================
4
+ Tracks prediction confidence distributions over time.
5
+ Detects when the model's behavior shifts (new slang, distribution change).
6
+
7
+ Algorithm:
8
+ - Maintain rolling window of last N confidence scores from PredictionLog DB
9
+ - Compute mean, stddev, and toxic rate
10
+ - Compare current window to a baseline (first 100 predictions or configured)
11
+ - Alert if mean confidence drifts > DRIFT_THRESHOLD standard deviations
12
+ - KL-divergence computed via histogram binning (no scipy needed)
13
+
14
+ Usage:
15
+ from app.services.drift_monitor import DriftMonitor
16
+ monitor = DriftMonitor()
17
+ monitor.log_prediction(db, confidence=0.87, toxic=True, ...)
18
+ stats = monitor.get_stats(db)
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import hashlib
24
+ import math
25
+ from datetime import datetime, timedelta
26
+ from typing import Optional
27
+ from collections import deque
28
+
29
+ from sqlalchemy.orm import Session
30
+ from sqlalchemy import func
31
+
32
+ from app.core.logger import logger
33
+ from models import PredictionLog
34
+
35
+ # ──────────────────────────────────────────────────────────────────────────────
36
+ # CONFIG
37
+ # ──────────────────────────────────────────────────────────────────────────────
38
+
39
+ WINDOW_SIZE = 500 # Rolling window for stats
40
+ DRIFT_THRESHOLD = 0.15 # Alert if mean confidence shifts by this much
41
+ BASELINE_MIN = 50 # Need at least this many predictions before drift detection
42
+ BINS = 10 # Histogram bins for KL-divergence
43
+
44
+
45
+ # ──────────────────────────────────────────────────────────────────────────────
46
+ # HELPERS
47
+ # ──────────────────────────────────────────────────────────────────────────────
48
+
49
+ def _mean(vals: list[float]) -> float:
50
+ return sum(vals) / len(vals) if vals else 0.0
51
+
52
+
53
+ def _stddev(vals: list[float]) -> float:
54
+ if len(vals) < 2:
55
+ return 0.0
56
+ m = _mean(vals)
57
+ variance = sum((x - m) ** 2 for x in vals) / len(vals)
58
+ return math.sqrt(variance)
59
+
60
+
61
+ def _histogram(vals: list[float], bins: int = BINS) -> list[float]:
62
+ """Normalised histogram (probability distribution) over [0, 1]."""
63
+ if not vals:
64
+ return [0.0] * bins
65
+ counts = [0] * bins
66
+ for v in vals:
67
+ idx = min(int(v * bins), bins - 1)
68
+ counts[idx] += 1
69
+ total = sum(counts)
70
+ return [c / total for c in counts]
71
+
72
+
73
+ def _kl_divergence(p: list[float], q: list[float], eps: float = 1e-10) -> float:
74
+ """KL divergence D(P||Q) — measures distribution shift."""
75
+ kl = 0.0
76
+ for pi, qi in zip(p, q):
77
+ pi = max(pi, eps)
78
+ qi = max(qi, eps)
79
+ kl += pi * math.log(pi / qi)
80
+ return round(kl, 4)
81
+
82
+
83
+ # ──────────────────────────────────────────────────────────────────────────────
84
+ # DRIFT MONITOR
85
+ # ──────────────────────────────────────────────────────────────────────────────
86
+
87
+ class DriftMonitor:
88
+ """
89
+ Singleton service for monitoring model confidence drift.
90
+ Uses DB as persistent store; in-memory cache for speed.
91
+ """
92
+
93
+ def __init__(self) -> None:
94
+ self._baseline: Optional[list[float]] = None # First BASELINE_MIN scores
95
+ self._baseline_hist: Optional[list[float]] = None
96
+
97
+ # ── LOGGING ───────────────────────────────────────────────────────────────
98
+
99
+ def log_prediction(
100
+ self,
101
+ db: Session,
102
+ *,
103
+ user_id: Optional[int],
104
+ input_text: str,
105
+ confidence: float,
106
+ toxic: bool,
107
+ severity: str,
108
+ source: str,
109
+ ) -> None:
110
+ """Write one prediction event to DB. Called inside every /predict handler."""
111
+ try:
112
+ text_hash = hashlib.sha256(input_text.encode()).hexdigest()[:16]
113
+ log = PredictionLog(
114
+ user_id=user_id,
115
+ text_hash=text_hash,
116
+ text_length=len(input_text),
117
+ confidence=round(confidence, 4),
118
+ toxic=toxic,
119
+ severity=severity,
120
+ source=source,
121
+ )
122
+ db.add(log)
123
+ db.commit()
124
+ except Exception as exc:
125
+ logger.warning(f"[DriftMonitor] Failed to log prediction: {exc}")
126
+ db.rollback()
127
+
128
+ # ── STATS ─────────────────────────────────────────────────────────────────
129
+
130
+ def get_stats(self, db: Session) -> dict:
131
+ """
132
+ Compute current monitoring stats from the rolling window.
133
+
134
+ Returns:
135
+ {
136
+ "total_predictions": int,
137
+ "window_size": int,
138
+ "mean_confidence": float,
139
+ "stddev_confidence": float,
140
+ "toxic_rate": float, # 0.0–1.0
141
+ "drift_detected": bool,
142
+ "drift_score": float, # KL divergence from baseline
143
+ "alert": str | None,
144
+ "severity_breakdown": {"low": int, "medium": int, "high": int},
145
+ "hourly_volume": list[dict], # last 24h request counts
146
+ }
147
+ """
148
+ try:
149
+ # Total count
150
+ total = db.query(func.count(PredictionLog.id)).scalar() or 0
151
+
152
+ # Rolling window — last WINDOW_SIZE entries
153
+ recent = (
154
+ db.query(PredictionLog)
155
+ .order_by(PredictionLog.timestamp.desc())
156
+ .limit(WINDOW_SIZE)
157
+ .all()
158
+ )
159
+
160
+ if not recent:
161
+ return self._empty_stats()
162
+
163
+ confidences = [r.confidence for r in recent]
164
+ toxic_flags = [r.toxic for r in recent]
165
+
166
+ mean_conf = round(_mean(confidences), 4)
167
+ std_conf = round(_stddev(confidences), 4)
168
+ toxic_rate = round(sum(toxic_flags) / len(toxic_flags), 4)
169
+
170
+ # Severity breakdown
171
+ sev_counts = {"low": 0, "medium": 0, "high": 0}
172
+ for r in recent:
173
+ sev = r.severity or "low"
174
+ sev_counts[sev] = sev_counts.get(sev, 0) + 1
175
+
176
+ # Drift detection
177
+ drift_score = 0.0
178
+ drift_detected = False
179
+ alert = None
180
+
181
+ if total >= BASELINE_MIN:
182
+ # Baseline: oldest BASELINE_MIN entries
183
+ baseline_rows = (
184
+ db.query(PredictionLog.confidence)
185
+ .order_by(PredictionLog.timestamp.asc())
186
+ .limit(BASELINE_MIN)
187
+ .all()
188
+ )
189
+ baseline_vals = [r.confidence for r in baseline_rows]
190
+ baseline_hist = _histogram(baseline_vals)
191
+ current_hist = _histogram(confidences)
192
+ drift_score = _kl_divergence(current_hist, baseline_hist)
193
+ drift_detected = drift_score > 0.3 # KL threshold
194
+
195
+ if drift_detected:
196
+ alert = (
197
+ f"⚠️ Drift detected! KL-divergence={drift_score:.3f} "
198
+ f"(current mean={mean_conf:.3f} vs baseline mean={round(_mean(baseline_vals), 3)})"
199
+ )
200
+
201
+ # Hourly volume (last 24h)
202
+ hourly = self._get_hourly_volume(db)
203
+
204
+ return {
205
+ "total_predictions": total,
206
+ "window_size": len(recent),
207
+ "mean_confidence": mean_conf,
208
+ "stddev_confidence": std_conf,
209
+ "toxic_rate": toxic_rate,
210
+ "drift_detected": drift_detected,
211
+ "drift_score": round(drift_score, 4),
212
+ "alert": alert,
213
+ "severity_breakdown": sev_counts,
214
+ "hourly_volume": hourly,
215
+ "baseline_size": min(total, BASELINE_MIN),
216
+ }
217
+
218
+ except Exception as exc:
219
+ logger.error(f"[DriftMonitor] get_stats failed: {exc}")
220
+ return self._empty_stats()
221
+
222
+ def get_drift_series(self, db: Session, limit: int = 200) -> list[dict]:
223
+ """
224
+ Return time-series of confidence scores for the drift trend chart.
225
+ Returns last `limit` predictions ordered by time.
226
+ """
227
+ try:
228
+ rows = (
229
+ db.query(PredictionLog)
230
+ .order_by(PredictionLog.timestamp.desc())
231
+ .limit(limit)
232
+ .all()
233
+ )
234
+ rows.reverse()
235
+ return [
236
+ {
237
+ "time": r.timestamp.strftime("%H:%M"),
238
+ "confidence": round(r.confidence, 3),
239
+ "toxic": r.toxic,
240
+ "severity": r.severity,
241
+ }
242
+ for r in rows
243
+ ]
244
+ except Exception as exc:
245
+ logger.error(f"[DriftMonitor] get_drift_series failed: {exc}")
246
+ return []
247
+
248
+ def _get_hourly_volume(self, db: Session) -> list[dict]:
249
+ """Count predictions per hour for the last 24 hours."""
250
+ try:
251
+ since = datetime.utcnow() - timedelta(hours=24)
252
+ rows = (
253
+ db.query(PredictionLog)
254
+ .filter(PredictionLog.timestamp >= since)
255
+ .order_by(PredictionLog.timestamp.asc())
256
+ .all()
257
+ )
258
+ # Bucket by hour
259
+ buckets: dict[str, dict] = {}
260
+ for r in rows:
261
+ key = r.timestamp.strftime("%H:00")
262
+ if key not in buckets:
263
+ buckets[key] = {"hour": key, "count": 0, "toxic_count": 0}
264
+ buckets[key]["count"] += 1
265
+ if r.toxic:
266
+ buckets[key]["toxic_count"] += 1
267
+ return list(buckets.values())
268
+ except Exception:
269
+ return []
270
+
271
+ def _empty_stats(self) -> dict:
272
+ return {
273
+ "total_predictions": 0,
274
+ "window_size": 0,
275
+ "mean_confidence": 0.0,
276
+ "stddev_confidence": 0.0,
277
+ "toxic_rate": 0.0,
278
+ "drift_detected": False,
279
+ "drift_score": 0.0,
280
+ "alert": None,
281
+ "severity_breakdown": {"low": 0, "medium": 0, "high": 0},
282
+ "hourly_volume": [],
283
+ "baseline_size": 0,
284
+ }
285
+
286
+
287
+ # Module-level singleton
288
+ drift_monitor = DriftMonitor()
app/services/model_service.py ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # backend/app/services/model_service.py
2
+
3
+ """
4
+ ToxiGuard AI — Unified Model Service
5
+ ======================================
6
+ Singleton that loads the best available model at startup:
7
+
8
+ Priority 1: ONNX Runtime — ml/models/toxiguard-deberta/model.onnx
9
+ Priority 2: PyTorch transformer — ml/models/toxiguard-deberta/ (safetensors)
10
+ Priority 3: Legacy TF-IDF + LogReg — abuse_model.joblib (last resort)
11
+
12
+ Expose `model_service` as the single prediction interface across all routes.
13
+ """
14
+
15
+ import os
16
+ import joblib
17
+ from typing import Optional
18
+
19
+ from app.core.logger import logger
20
+ from app.core.config import settings
21
+
22
+
23
+ class ModelService:
24
+ """
25
+ Unified prediction service with automatic model selection.
26
+
27
+ Attributes:
28
+ model_type: One of 'transformer_onnx', 'transformer_pytorch',
29
+ 'joblib_legacy', or 'none'.
30
+ """
31
+
32
+ _MODEL_LABELS = {
33
+ "transformer_onnx": "DeBERTa-v3 via ONNX Runtime (production)",
34
+ "transformer_pytorch": "DeBERTa-v3 via PyTorch (fallback)",
35
+ "joblib_legacy": "TF-IDF + Logistic Regression (legacy)",
36
+ "none": "No model loaded",
37
+ }
38
+
39
+ def __init__(self) -> None:
40
+ self.model_type: str = "none"
41
+ self.transformer = None # TransformerInference | None
42
+ self.legacy_model = None # sklearn Pipeline | None
43
+ self.legacy_encoder = None # LabelEncoder | None
44
+
45
+ self._load()
46
+
47
+ # ── LOADING ───────────────────────────────────────────────────────────────
48
+
49
+ def _load(self) -> None:
50
+ """Load the best available model at startup."""
51
+ transformer_dir = settings.model_dir
52
+
53
+ if os.path.isdir(transformer_dir):
54
+ self._try_load_transformer(transformer_dir)
55
+
56
+ if self.transformer is None:
57
+ self._try_load_legacy()
58
+
59
+ logger.info(
60
+ f"[ModelService] Active model: {self._MODEL_LABELS.get(self.model_type)}"
61
+ )
62
+
63
+ def _try_load_transformer(self, model_dir: str) -> None:
64
+ try:
65
+ from ml.inference import TransformerInference
66
+ engine = TransformerInference(model_dir)
67
+ if engine.is_ready:
68
+ self.transformer = engine
69
+ self.model_type = (
70
+ "transformer_onnx"
71
+ if engine.backend == "onnx"
72
+ else "transformer_pytorch"
73
+ )
74
+ logger.info(
75
+ f"[ModelService] Transformer loaded | backend={engine.backend}"
76
+ )
77
+ else:
78
+ logger.warning(
79
+ "[ModelService] Transformer directory found but model not ready"
80
+ )
81
+ except Exception as exc:
82
+ logger.warning(f"[ModelService] Transformer load failed: {exc}")
83
+
84
+ def _try_load_legacy(self) -> None:
85
+ base_dir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
86
+ model_path = os.path.join(base_dir, "abuse_model.joblib")
87
+ encoder_path = os.path.join(base_dir, "label_encoder.joblib")
88
+
89
+ try:
90
+ self.legacy_model = joblib.load(model_path)
91
+ self.legacy_encoder = joblib.load(encoder_path)
92
+ self.model_type = "joblib_legacy"
93
+ logger.info("[ModelService] Legacy joblib model loaded")
94
+ except Exception as exc:
95
+ logger.error(f"[ModelService] No model available — {exc}")
96
+
97
+ # ── PREDICTION ────────────────────────────────────────────────────────────
98
+
99
+ def predict(self, text: str) -> Optional[dict]:
100
+ """
101
+ Unified prediction interface.
102
+
103
+ Args:
104
+ text: Model-preprocessed text (unicode-safe, URLs removed).
105
+
106
+ Returns:
107
+ {
108
+ "toxic": bool,
109
+ "toxicity_probability": float,
110
+ "labels": {...}, # transformer only
111
+ "severity": str,
112
+ "detected_categories": list,
113
+ "source": str,
114
+ }
115
+ or None if no model is loaded.
116
+ """
117
+ if not text or not text.strip():
118
+ return None
119
+
120
+ if self.transformer:
121
+ result = self.transformer.predict(text)
122
+ if result is not None:
123
+ return result
124
+ logger.warning("[ModelService] Transformer returned None — trying legacy")
125
+
126
+ if self.legacy_model and self.legacy_encoder:
127
+ return self._predict_legacy(text)
128
+
129
+ return None
130
+
131
+ def _predict_legacy(self, text: str) -> dict:
132
+ """TF-IDF + Logistic Regression prediction."""
133
+ _TOXIC_LABELS = {
134
+ "abusive", "toxic", "severe_toxic",
135
+ "obscene", "threat", "insult", "identity_hate",
136
+ }
137
+ probs = self.legacy_model.predict_proba([text])[0]
138
+ labels = list(self.legacy_encoder.classes_)
139
+
140
+ toxic_probability = max(
141
+ (float(probs[idx]) for idx, lbl in enumerate(labels) if lbl in _TOXIC_LABELS),
142
+ default=0.0,
143
+ )
144
+ predicted_label = self.legacy_encoder.inverse_transform([probs.argmax()])[0]
145
+ label_map = {lbl: round(float(probs[i]), 4) for i, lbl in enumerate(labels)}
146
+
147
+ detected = [lbl for lbl in labels if lbl in _TOXIC_LABELS and label_map[lbl] >= 0.4]
148
+
149
+ return {
150
+ "toxic": predicted_label in _TOXIC_LABELS or toxic_probability >= 0.4,
151
+ "toxicity_probability": round(toxic_probability, 4),
152
+ "label": predicted_label,
153
+ "labels": label_map,
154
+ "severity": (
155
+ "high" if toxic_probability >= 0.85
156
+ else "medium" if toxic_probability >= 0.60
157
+ else "low"
158
+ ),
159
+ "detected_categories": detected,
160
+ "source": "legacy_ml",
161
+ }
162
+
163
+ # ── STATUS ────────────────────────────────────────────────────────────────
164
+
165
+ @property
166
+ def status(self) -> dict:
167
+ """Return model status for the /health endpoint."""
168
+ info = {
169
+ "model_type": self.model_type,
170
+ "description": self._MODEL_LABELS.get(self.model_type, "Unknown"),
171
+ "ready": self.model_type != "none",
172
+ }
173
+ if self.transformer:
174
+ info["backend"] = self.transformer.backend
175
+ return info
176
+
177
+
178
+ # Module-level singleton — loaded once at startup
179
+ model_service = ModelService()
app/services/overrides.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — Dynamic Rule Overrides Service
3
+ ===============================================
4
+ Enables instant hot-patching and updates from active learning feedback.
5
+
6
+ How it works:
7
+ - Whitelist (Allowlist): If a user flags a False Positive, we add the normalized text to the allowlist.
8
+ Next time this exact phrase is checked, it bypasses DeBERTa/LLM and is instantly marked safe.
9
+ - Blacklist (Blocklist): If a user flags a False Negative, we add the normalized text to the blocklist.
10
+ Next time, it's instantly marked toxic.
11
+ - Both allowlist/blocklist are persisted in `data/overrides.json`.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import os
17
+ import json
18
+ import threading
19
+ from typing import Optional
20
+
21
+ from app.core.logger import logger
22
+ from utils.preprocessing import preprocess_for_rules
23
+
24
+ _OVERRIDES_FILE = os.path.join(
25
+ os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
26
+ "data", "overrides.json"
27
+ )
28
+
29
+
30
+ class DynamicOverrideService:
31
+ def __init__(self) -> None:
32
+ self._lock = threading.Lock()
33
+ self.allowlist: set[str] = set() # Set of normalized texts
34
+ self.blocklist: set[str] = set() # Set of normalized texts
35
+ self._load()
36
+
37
+ def _load(self) -> None:
38
+ try:
39
+ if os.path.exists(_OVERRIDES_FILE):
40
+ with open(_OVERRIDES_FILE, "r") as f:
41
+ data = json.load(f)
42
+ self.allowlist = set(data.get("allowlist", []))
43
+ self.blocklist = set(data.get("blocklist", []))
44
+ logger.info(
45
+ f"[Overrides] Loaded {len(self.allowlist)} allowed and "
46
+ f"{len(self.blocklist)} blocked phrases."
47
+ )
48
+ except Exception as exc:
49
+ logger.warning(f"[Overrides] Could not load overrides: {exc}")
50
+
51
+ def _save(self) -> None:
52
+ try:
53
+ os.makedirs(os.path.dirname(_OVERRIDES_FILE), exist_ok=True)
54
+ with open(_OVERRIDES_FILE, "w") as f:
55
+ json.dump({
56
+ "allowlist": list(self.allowlist),
57
+ "blocklist": list(self.blocklist),
58
+ }, f, indent=2)
59
+ except Exception as exc:
60
+ logger.warning(f"[Overrides] Could not save overrides: {exc}")
61
+
62
+ def add_override(self, feedback_type: str, text: str) -> None:
63
+ """Add phrase to blocklist or allowlist depending on feedback type."""
64
+ normalized = preprocess_for_rules(text)
65
+ if not normalized:
66
+ return
67
+
68
+ with self._lock:
69
+ if feedback_type == "false_positive":
70
+ # User says safe -> remove from blocklist, add to allowlist
71
+ self.blocklist.discard(normalized)
72
+ self.allowlist.add(normalized)
73
+ logger.info(f"[Overrides] Added to allowlist: '{normalized}'")
74
+ elif feedback_type == "false_negative":
75
+ # User says toxic -> remove from allowlist, add to blocklist
76
+ self.allowlist.discard(normalized)
77
+ self.blocklist.add(normalized)
78
+ logger.info(f"[Overrides] Added to blocklist: '{normalized}'")
79
+ self._save()
80
+
81
+ def check_override(self, text: str) -> Optional[bool]:
82
+ """
83
+ Check if text has an active override.
84
+ Returns True (block), False (allow), or None (no override).
85
+ """
86
+ normalized = preprocess_for_rules(text)
87
+ if not normalized:
88
+ return None
89
+
90
+ with self._lock:
91
+ if normalized in self.allowlist:
92
+ return False
93
+ if normalized in self.blocklist:
94
+ return True
95
+ return None
96
+
97
+
98
+ # Module-level singleton
99
+ override_service = DynamicOverrideService()
app/services/retrainer.py ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — Model Retraining Pipeline
3
+ =========================================
4
+ Retrains the legacy classifier model using historical baseline data combined
5
+ with newly collected active learning feedback samples.
6
+
7
+ Process:
8
+ 1. Retrieve all unreviewed feedback corrections from DB.
9
+ 2. Parse baseline dataset (data/sample_data.csv).
10
+ 3. Transform user feedback into format compatible with the baseline:
11
+ - If user corrected to TOXIC -> label is 'toxic'
12
+ - If user corrected to SAFE -> label is 'clean'
13
+ 4. Append corrections to training data.
14
+ 5. Fit the scikit-learn pipeline (TF-IDF + Logistic Regression).
15
+ 6. Save updated model checkpoint.
16
+ 7. Hot-reload model in ModelService.
17
+ 8. Mark feedback samples as reviewed.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import os
23
+ import joblib
24
+ import pandas as pd
25
+ from datetime import datetime
26
+ from sqlalchemy.orm import Session
27
+
28
+ from app.core.logger import logger
29
+ from app.services.model_service import model_service
30
+ from models import FeedbackSample
31
+
32
+ BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
33
+ CSV_PATH = os.path.join(BASE_DIR, "data", "sample_data.csv")
34
+ MODEL_PATH = os.path.join(BASE_DIR, "abuse_model.joblib")
35
+ ENCODER_PATH = os.path.join(BASE_DIR, "label_encoder.joblib")
36
+
37
+
38
+ def retrain_model(db: Session) -> dict:
39
+ """
40
+ Retrain the ML classifier with the active learning queue samples.
41
+ """
42
+ # 1. Fetch unreviewed corrections
43
+ feedback_samples = (
44
+ db.query(FeedbackSample)
45
+ .filter(FeedbackSample.reviewed == False)
46
+ .all()
47
+ )
48
+
49
+ if not feedback_samples:
50
+ return {
51
+ "status": "ignored",
52
+ "message": "No new unreviewed feedback samples found. Model is already up to date.",
53
+ "count": 0
54
+ }
55
+
56
+ logger.info(f"[Retrainer] Starting retraining with {len(feedback_samples)} feedback samples...")
57
+
58
+ # 2. Load baseline CSV
59
+ if not os.path.exists(CSV_PATH):
60
+ raise FileNotFoundError(f"Baseline dataset not found at: {CSV_PATH}")
61
+
62
+ df_base = pd.read_csv(CSV_PATH)
63
+
64
+ # 3. Load model structures
65
+ if not os.path.exists(MODEL_PATH) or not os.path.exists(ENCODER_PATH):
66
+ raise FileNotFoundError("Model or Encoder joblib file not found.")
67
+
68
+ pipeline = joblib.load(MODEL_PATH)
69
+ encoder = joblib.load(ENCODER_PATH)
70
+
71
+ # 4. Process user feedback samples
72
+ new_rows = []
73
+ for sample in feedback_samples:
74
+ label = "toxic" if sample.correct_label else "clean"
75
+ # Map baseline equivalent labels
76
+ new_rows.append({
77
+ "text": sample.input_text,
78
+ "label": label
79
+ })
80
+
81
+ df_new = pd.DataFrame(new_rows)
82
+ df_combined = pd.concat([df_base, df_new], ignore_index=True)
83
+
84
+ # 5. Prepare inputs
85
+ X = df_combined["text"].fillna("").astype(str).values
86
+ y_raw = df_combined["label"].values
87
+
88
+ # Clean label mapping: map toxic classes to 'toxic', others to 'clean'
89
+ TOXIC_LABELS = {
90
+ "abusive", "toxic", "severe_toxic", "obscene", "threat",
91
+ "insult", "identity_hate", "hate", "spam"
92
+ }
93
+ y_mapped = []
94
+ for val in y_raw:
95
+ val_str = str(val).strip().lower()
96
+ if val_str in TOXIC_LABELS:
97
+ y_mapped.append("toxic")
98
+ else:
99
+ y_mapped.append("clean")
100
+
101
+ # Fit encoder target
102
+ y = encoder.transform(y_mapped)
103
+
104
+ # 6. Fit the pipeline
105
+ pipeline.fit(X, y)
106
+
107
+ # 7. Save model checkpoints
108
+ joblib.dump(pipeline, MODEL_PATH)
109
+ logger.info(f"[Retrainer] Model saved successfully to: {MODEL_PATH}")
110
+
111
+ # 8. Hot-reload the prediction service
112
+ model_service._load()
113
+ logger.info("[Retrainer] Unified ModelService hot-reloaded successfully.")
114
+
115
+ # 9. Mark samples as reviewed
116
+ for sample in feedback_samples:
117
+ sample.reviewed = True
118
+ db.commit()
119
+
120
+ return {
121
+ "status": "success",
122
+ "message": f"Successfully retrained legacy model using {len(feedback_samples)} feedback samples. Model is now updated.",
123
+ "samples_retrained": len(feedback_samples),
124
+ "timestamp": datetime.utcnow().isoformat()
125
+ }
auth_utils.py ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from datetime import datetime, timedelta
3
+ from jose import JWTError, jwt
4
+ from fastapi import HTTPException, status
5
+ import secrets
6
+ from dotenv import load_dotenv
7
+
8
+ load_dotenv()
9
+
10
+ SECRET_KEY = os.getenv("JWT_SECRET_KEY", secrets.token_hex(32))
11
+ ALGORITHM = os.getenv("JWT_ALGORITHM", "HS256")
12
+ ACCESS_TOKEN_EXPIRE_MINUTES = int(os.getenv("ACCESS_TOKEN_EXPIRE_MINUTES", "1440"))
13
+
14
+
15
+ def create_access_token(data: dict):
16
+ to_encode = data.copy()
17
+ expire = datetime.utcnow() + timedelta(
18
+ minutes=ACCESS_TOKEN_EXPIRE_MINUTES
19
+ )
20
+ to_encode.update({"exp": expire})
21
+
22
+ return jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
23
+
24
+
25
+ def verify_token(token: str):
26
+ try:
27
+ payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
28
+ return payload
29
+ except JWTError:
30
+ raise HTTPException(
31
+ status_code=status.HTTP_401_UNAUTHORIZED,
32
+ detail="Invalid token"
33
+ )
34
+
35
+
36
+ # ✅ API KEY GENERATOR
37
+
38
+ def generate_api_key():
39
+ return secrets.token_hex(32)
data/LICENSE ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2017 Tom Davidson
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Automated Hate Speech Detection and the Problem of Offensive Language
2
+ Repository for Thomas Davidson, Dana Warmsley, Michael Macy, and Ingmar Weber. 2017. "Automated Hate Speech Detection and the Problem of Offensive Language." ICWSM. You read the paper [here](https://aaai.org/ocs/index.php/ICWSM/ICWSM17/paper/view/15665).
3
+
4
+
5
+ # NOTE: This repository is no longer actively maintained. Please do not post issues regarding the compatibility of the existing code with new versions of Python or the packages used.
6
+
7
+ ## 2019 NEWS
8
+ We have a new paper on racial bias in this dataset and others, you can read it [here](https://arxiv.org/abs/1905.12516)
9
+
10
+
11
+ ***WARNING: The data, lexicons, and notebooks all contain content that is racist, sexist, homophobic, and offensive in many other ways.***
12
+
13
+ You can find our labeled data in the `data` directory. We have included them as a pickle file (Python 2.7) and as a CSV. You will also find a notebook in the `src` directory containing Python 2.7 code to replicate our analyses in the paper and a lexicon in the `lexicons` directory that we generated to try to more accurately classify hate speech. The `classifier` directory contains a script, instructions, and the necessary files to run our classifier on new data, a test case is provided.
14
+
15
+
16
+ ***Please cite our paper in any published work that uses any of these resources.***
17
+ ~~~
18
+ @inproceedings{hateoffensive,
19
+ title = {Automated Hate Speech Detection and the Problem of Offensive Language},
20
+ author = {Davidson, Thomas and Warmsley, Dana and Macy, Michael and Weber, Ingmar},
21
+ booktitle = {Proceedings of the 11th International AAAI Conference on Web and Social Media},
22
+ series = {ICWSM '17},
23
+ year = {2017},
24
+ location = {Montreal, Canada},
25
+ pages = {512-515}
26
+ }
27
+ ~~~
28
+
29
+ ***Contact***
30
+ We would also appreciate it if you could fill out this short [form](https://docs.google.com/forms/d/e/1FAIpQLSdrPNlfVBlqxun2tivzAtsZaOoPC5YYMocn-xscCgeRakLXHg/viewform?usp=pp_url&entry.1506871634&entry.147453066&entry.1390333885&entry.516829772) if you are interested in using our data so we can keep track of how these data are used and get in contact with researchers working on similar problems.
31
+
32
+ If you have any questions please contact `trd54 at cornell dot edu`.
data/calibration.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "threshold": 0.49,
3
+ "fp_count": 0,
4
+ "fn_count": 1,
5
+ "total_adjustments": 6,
6
+ "history": [
7
+ {
8
+ "ts": "2026-07-21T10:55:49.876075",
9
+ "type": "confirmed_correct",
10
+ "old": 0.5,
11
+ "new": 0.5,
12
+ "confidence_at_time": 0.97
13
+ },
14
+ {
15
+ "ts": "2026-07-22T04:26:14.406519",
16
+ "type": "confirmed_correct",
17
+ "old": 0.5,
18
+ "new": 0.5,
19
+ "confidence_at_time": 0.975
20
+ },
21
+ {
22
+ "ts": "2026-07-22T10:58:25.483774",
23
+ "type": "confirmed_correct",
24
+ "old": 0.5,
25
+ "new": 0.5,
26
+ "confidence_at_time": 0.791
27
+ },
28
+ {
29
+ "ts": "2026-07-25T09:32:48.215232",
30
+ "type": "false_negative",
31
+ "old": 0.5,
32
+ "new": 0.49,
33
+ "confidence_at_time": 0.449
34
+ },
35
+ {
36
+ "ts": "2026-07-25T09:45:59.338829",
37
+ "type": "confirmed_correct",
38
+ "old": 0.49,
39
+ "new": 0.49,
40
+ "confidence_at_time": 0.428
41
+ },
42
+ {
43
+ "ts": "2026-07-25T09:55:28.278200",
44
+ "type": "confirmed_correct",
45
+ "old": 0.49,
46
+ "new": 0.49,
47
+ "confidence_at_time": 0.54
48
+ }
49
+ ],
50
+ "last_updated": "2026-07-25T09:55:28.278200"
51
+ }
data/overrides.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "allowlist": [],
3
+ "blocklist": [
4
+ "kitna dogla aadmi hai sale tu"
5
+ ]
6
+ }
data/sample_data.csv ADDED
The diff for this file is too large to render. See raw diff
 
database.py ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # database.py
2
+
3
+ import os
4
+ from dotenv import load_dotenv
5
+ from sqlalchemy import create_engine
6
+ from sqlalchemy.orm import sessionmaker, declarative_base
7
+
8
+ # ✅ load .env file
9
+ load_dotenv()
10
+
11
+ # ✅ fallback for local dev
12
+ DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///./toxiguard.db")
13
+
14
+ # Fix for Render (postgres)
15
+ if DATABASE_URL.startswith("postgres://"):
16
+ DATABASE_URL = DATABASE_URL.replace("postgres://", "postgresql://", 1)
17
+
18
+ # SQLite fix
19
+ connect_args = {"check_same_thread": False} if "sqlite" in DATABASE_URL else {}
20
+
21
+ engine = create_engine(
22
+ DATABASE_URL,
23
+ connect_args=connect_args,
24
+ pool_pre_ping=True
25
+ )
26
+
27
+ SessionLocal = sessionmaker(
28
+ autocommit=False,
29
+ autoflush=False,
30
+ bind=engine
31
+ )
32
+
33
+ Base = declarative_base()
34
+
35
+
36
+ def get_db():
37
+ db = SessionLocal()
38
+ try:
39
+ yield db
40
+ finally:
41
+ db.close()
label_encoder.joblib ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0c1a86a1ca2876f7b1c007ce4dd56ce0c1a76aa8d14a45d1353cb411cc1fe9eb
3
+ size 489
ml/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # backend/ml/__init__.py
ml/evaluate.py ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — Model Evaluation Suite
3
+ ======================================
4
+ Comprehensive evaluation: F1, AUC-ROC, confusion matrices, threshold analysis.
5
+
6
+ Usage:
7
+ python -m ml.evaluate --model_path ml/models/toxiguard-deberta
8
+ """
9
+
10
+ import os, json, argparse
11
+ import numpy as np
12
+ import pandas as pd
13
+ import torch
14
+ from transformers import AutoTokenizer, AutoModelForSequenceClassification
15
+ from sklearn.metrics import (
16
+ f1_score, precision_score, recall_score, roc_auc_score,
17
+ classification_report, confusion_matrix, average_precision_score,
18
+ )
19
+
20
+ BASE_DIR = os.path.dirname(os.path.dirname(__file__))
21
+ DATA_DIR = os.path.join(BASE_DIR, "data")
22
+ DEFAULT_MODEL = os.path.join(os.path.dirname(__file__), "models", "toxiguard-deberta")
23
+ LABEL_COLUMNS = ["toxic","severe_toxic","obscene","threat","insult","identity_hate"]
24
+ MAX_LENGTH = 256
25
+
26
+ def predict_batch(texts, model, tokenizer, batch_size=32, device="cpu"):
27
+ model.eval(); model.to(device); all_probs = []
28
+ for i in range(0, len(texts), batch_size):
29
+ enc = tokenizer(list(texts[i:i+batch_size]), truncation=True,
30
+ padding="max_length", max_length=MAX_LENGTH, return_tensors="pt").to(device)
31
+ with torch.no_grad():
32
+ all_probs.append(torch.sigmoid(model(**enc).logits).cpu().numpy())
33
+ return np.vstack(all_probs)
34
+
35
+ def evaluate(args):
36
+ print("=" * 60 + "\n📊 ToxiGuard AI — Model Evaluation\n" + "=" * 60)
37
+ tokenizer = AutoTokenizer.from_pretrained(args.model_path)
38
+ model = AutoModelForSequenceClassification.from_pretrained(args.model_path)
39
+ device = "cuda" if torch.cuda.is_available() else "cpu"
40
+
41
+ train_df = pd.read_csv(os.path.join(DATA_DIR, "train.csv"))
42
+ from sklearn.model_selection import train_test_split
43
+ _, test_df = train_test_split(train_df, test_size=0.1, random_state=42)
44
+ texts = test_df["comment_text"].fillna("").values
45
+ labels = test_df[LABEL_COLUMNS].values.astype(float)
46
+
47
+ probs = predict_batch(texts, model, tokenizer, device=device)
48
+ preds = (probs >= 0.5).astype(int)
49
+
50
+ results = {"f1_macro": round(f1_score(labels, preds, average="macro", zero_division=0), 4),
51
+ "auc_roc": round(roc_auc_score(labels, probs, average="macro"), 4)}
52
+
53
+ print(classification_report(labels, preds, target_names=LABEL_COLUMNS, zero_division=0))
54
+
55
+ for i, col in enumerate(LABEL_COLUMNS):
56
+ cm = confusion_matrix(labels[:,i], preds[:,i])
57
+ tn, fp, fn, tp = cm.ravel() if cm.size == 4 else (0,0,0,0)
58
+ print(f" {col}: TP={tp} FP={fp} FN={fn} TN={tn}")
59
+
60
+ out = os.path.join(args.model_path, "evaluation_results.json")
61
+ with open(out, "w") as f: json.dump(results, f, indent=2)
62
+ print(f"\n✅ Saved to {out}")
63
+
64
+ if __name__ == "__main__":
65
+ p = argparse.ArgumentParser()
66
+ p.add_argument("--model_path", default=DEFAULT_MODEL)
67
+ evaluate(p.parse_args())
ml/explainability.py ADDED
@@ -0,0 +1,242 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — XAI Attribution Engine (Perturbation-Based)
3
+ ============================================================
4
+ Computes per-word importance scores using Leave-One-Out (LOO) perturbation.
5
+ Works with ONNX-only model dirs — no PyTorch weights required.
6
+
7
+ Algorithm:
8
+ 1. Get baseline confidence score for the full text
9
+ 2. For each word, mask it out (replace with [MASK]-equivalent)
10
+ 3. Re-run inference on masked text
11
+ 4. Attribution score = baseline_confidence - masked_confidence
12
+ → Positive = word increased toxicity (important)
13
+ → Negative = word actually reduced toxicity
14
+ 5. Normalise scores to [0, 1]
15
+
16
+ Why perturbation over attention?
17
+ - Works with ONNX (no PyTorch needed)
18
+ - Model-agnostic — works with any classifier
19
+ - More faithful: attention ≠ importance (known research finding, Jain & Wallace 2019)
20
+ - This is the method used in production ML (Google, Meta content moderation)
21
+
22
+ Usage:
23
+ from ml.explainability import get_explainer
24
+ explainer = get_explainer(model_dir)
25
+ result = explainer.explain("you are an idiot")
26
+ # {"tokens": [{"token": "you", "score": 0.02, ...}, ...], "top_tokens": [...]}
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ import re
32
+ import numpy as np
33
+ from typing import Optional
34
+
35
+ from app.core.logger import logger
36
+
37
+
38
+ # ──────────────────────────────────────────────────────────────────────────────
39
+ # HELPERS
40
+ # ──────────────────────────────────────────────────────────────────────────────
41
+
42
+ def _sigmoid(x: np.ndarray) -> np.ndarray:
43
+ return 1.0 / (1.0 + np.exp(-x))
44
+
45
+
46
+ def _tokenize_words(text: str) -> list[str]:
47
+ """Split text into words preserving order. Handles punctuation."""
48
+ return re.findall(r"\S+", text)
49
+
50
+
51
+ def _mask_word(words: list[str], idx: int, mask: str = "") -> str:
52
+ """Return text with word at idx replaced by mask (or removed)."""
53
+ masked = words[:idx] + ([mask] if mask else []) + words[idx + 1:]
54
+ return " ".join(masked).strip() or "." # fallback so tokenizer doesn't choke
55
+
56
+
57
+ # ──────────────────────────────────────────────────────────────────────────────
58
+ # EXPLAINER
59
+ # ──────────────────────────────────────────────────────────────────────────────
60
+
61
+ class PerturbationExplainer:
62
+ """
63
+ Leave-One-Out attribution using the ONNX inference session directly.
64
+ No PyTorch weights required.
65
+ """
66
+
67
+ def __init__(self, model_dir: str) -> None:
68
+ self.model_dir = model_dir
69
+ self.session = None
70
+ self.tokenizer = None
71
+ self._onnx_input_names: list[str] = []
72
+ self._loaded = False
73
+ self._load()
74
+
75
+ def _load(self) -> None:
76
+ import os
77
+
78
+ if not os.path.isdir(self.model_dir):
79
+ logger.warning(
80
+ f"[XAI] PerturbationExplainer load failed: Directory '{self.model_dir}' not found. "
81
+ "Explainability feature will be disabled."
82
+ )
83
+ self._loaded = False
84
+ return
85
+
86
+ onnx_path = os.path.join(self.model_dir, "model.onnx")
87
+ if not os.path.exists(onnx_path):
88
+ logger.warning(
89
+ f"[XAI] PerturbationExplainer load failed: ONNX model file '{onnx_path}' not found. "
90
+ "Explainability feature will be disabled."
91
+ )
92
+ self._loaded = False
93
+ return
94
+
95
+ try:
96
+ import onnxruntime as ort
97
+ from transformers import AutoTokenizer
98
+
99
+ self.tokenizer = AutoTokenizer.from_pretrained(
100
+ self.model_dir, use_fast=True
101
+ )
102
+ self.session = ort.InferenceSession(
103
+ onnx_path,
104
+ providers=["CPUExecutionProvider"],
105
+ )
106
+ self._onnx_input_names = [
107
+ inp.name for inp in self.session.get_inputs()
108
+ ]
109
+ self._loaded = True
110
+ logger.info("[XAI] PerturbationExplainer loaded (ONNX)")
111
+ except Exception as exc:
112
+ logger.warning(f"[XAI] PerturbationExplainer load failed: {exc}")
113
+ self._loaded = False
114
+
115
+ @property
116
+ def is_ready(self) -> bool:
117
+ return self._loaded
118
+
119
+ def _infer(self, text: str) -> float:
120
+ """Run ONNX inference and return max toxicity probability."""
121
+ encoding = self.tokenizer(
122
+ text,
123
+ truncation=True,
124
+ padding="max_length",
125
+ max_length=128,
126
+ return_tensors="np",
127
+ )
128
+ inputs = {
129
+ name: encoding[name].astype(np.int64)
130
+ for name in self._onnx_input_names
131
+ if name in encoding
132
+ }
133
+ logits: np.ndarray = self.session.run(None, inputs)[0]
134
+ probs = _sigmoid(logits)[0] # shape: (num_labels,)
135
+ return float(np.max(probs))
136
+
137
+ def explain(self, text: str, max_words: int = 30) -> Optional[dict]:
138
+ """
139
+ LOO attribution — score each word by how much removing it
140
+ changes the toxicity confidence.
141
+
142
+ Args:
143
+ text: Raw input text.
144
+ max_words: Limit words to analyze (keeps latency ~1-2s for long texts).
145
+
146
+ Returns:
147
+ {
148
+ "tokens": [{"token": str, "score": float, "is_special": bool}],
149
+ "top_tokens": [{"token": str, "score": float}],
150
+ "method": "leave_one_out_perturbation",
151
+ "num_tokens": int,
152
+ "baseline_confidence": float,
153
+ }
154
+ """
155
+ if not self.is_ready:
156
+ return None
157
+
158
+ text = text.strip()
159
+ if not text:
160
+ return None
161
+
162
+ try:
163
+ words = _tokenize_words(text)
164
+
165
+ # Cap at max_words to keep latency manageable
166
+ if len(words) > max_words:
167
+ words = words[:max_words]
168
+
169
+ # Step 1: baseline confidence
170
+ baseline = self._infer(text)
171
+
172
+ # Step 2: LOO — remove each word, measure confidence drop
173
+ raw_scores: list[float] = []
174
+ for i in range(len(words)):
175
+ masked_text = _mask_word(words, i)
176
+ masked_conf = self._infer(masked_text)
177
+ # Positive score = removing this word LOWERED toxicity → word is "toxic"
178
+ raw_scores.append(baseline - masked_conf)
179
+
180
+ from utils.abuse_words import get_word_severity
181
+
182
+ # Step 3: Blend raw ML perturbation score with rule-based severity weights
183
+ clamped: list[float] = []
184
+ for i, word in enumerate(words):
185
+ # Clean punctuation for rule lookups
186
+ clean_word = re.sub(r"[^\w]", "", word.lower())
187
+ severity = get_word_severity(clean_word)
188
+ rule_boost = {"critical": 1.0, "high": 0.85, "moderate": 0.55}.get(severity, 0.0)
189
+
190
+ raw_s = max(0.0, raw_scores[i])
191
+ clamped.append(raw_s + rule_boost)
192
+
193
+ max_val = max(clamped) if clamped else 0.0
194
+
195
+ if max_val > 0:
196
+ normalised = [round(s / max_val, 4) for s in clamped]
197
+ else:
198
+ normalised = [0.0] * len(clamped)
199
+
200
+ # Build token list
201
+ tokens = [
202
+ {
203
+ "token": word,
204
+ "score": score,
205
+ "raw_delta": round(raw_scores[i], 4),
206
+ "is_special": False,
207
+ }
208
+ for i, (word, score) in enumerate(zip(words, normalised))
209
+ ]
210
+
211
+ # Top 5 most influential words
212
+ top_tokens = sorted(
213
+ [t for t in tokens if not t["is_special"]],
214
+ key=lambda x: x["score"],
215
+ reverse=True,
216
+ )[:5]
217
+
218
+ return {
219
+ "tokens": tokens,
220
+ "top_tokens": top_tokens,
221
+ "method": "leave_one_out_perturbation",
222
+ "num_tokens": len(tokens),
223
+ "baseline_confidence": round(baseline, 4),
224
+ }
225
+
226
+ except Exception as exc:
227
+ logger.error(f"[XAI] explain() failed: {exc}")
228
+ return None
229
+
230
+
231
+ # ──────────────────────────────────────────────────────────────────────────────
232
+ # MODULE-LEVEL SINGLETON
233
+ # ──────────────────────────────────────────────────────────────────────────────
234
+
235
+ _explainer_instance: Optional[PerturbationExplainer] = None
236
+
237
+
238
+ def get_explainer(model_dir: str) -> PerturbationExplainer:
239
+ global _explainer_instance
240
+ if _explainer_instance is None:
241
+ _explainer_instance = PerturbationExplainer(model_dir)
242
+ return _explainer_instance
ml/export_onnx.py ADDED
@@ -0,0 +1,146 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — ONNX Export Script
3
+ ====================================
4
+ Converts a fine-tuned HuggingFace DeBERTa-v3 model to ONNX format
5
+ for production-grade inference with ONNX Runtime (<50ms latency).
6
+
7
+ Prerequisites:
8
+ pip install optimum[onnxruntime] onnx onnxruntime
9
+
10
+ Usage:
11
+ # After training with train_transformer.py:
12
+ python -m ml.export_onnx
13
+
14
+ # Custom model dir:
15
+ python -m ml.export_onnx --model_dir ml/models/toxiguard-deberta
16
+
17
+ What it does:
18
+ 1. Loads the fine-tuned PyTorch model + tokenizer
19
+ 2. Exports to ONNX via HuggingFace Optimum (traces through the model)
20
+ 3. Validates the ONNX output matches PyTorch output within 1e-4 tolerance
21
+ 4. Saves model.onnx into the same model directory
22
+
23
+ Output:
24
+ ml/models/toxiguard-deberta/model.onnx
25
+ """
26
+
27
+ import os
28
+ import argparse
29
+ import numpy as np
30
+
31
+ BASE_DIR = os.path.dirname(os.path.dirname(__file__))
32
+ DEFAULT_MODEL_DIR = os.path.join(BASE_DIR, "ml", "models", "toxiguard-deberta")
33
+
34
+ LABEL_COLUMNS = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
35
+
36
+ TEST_SENTENCES = [
37
+ "you are a complete idiot and I hate you",
38
+ "I love rainy days, they are so peaceful",
39
+ "go to hell and never come back",
40
+ "the weather looks beautiful today",
41
+ ]
42
+
43
+
44
+ def export_onnx(model_dir: str) -> None:
45
+ print("=" * 60)
46
+ print("ToxiGuard AI — ONNX Export")
47
+ print("=" * 60)
48
+ print(f"\n[1/4] Loading model from: {model_dir}")
49
+
50
+ # Verify the model directory exists
51
+ if not os.path.isdir(model_dir):
52
+ raise FileNotFoundError(
53
+ f"Model directory not found: {model_dir}\n"
54
+ f"Run `python -m ml.train_transformer` first."
55
+ )
56
+
57
+ try:
58
+ from optimum.onnxruntime import ORTModelForSequenceClassification # type: ignore
59
+ from transformers import AutoTokenizer # type: ignore
60
+ except ImportError:
61
+ raise ImportError(
62
+ "Missing dependencies. Run:\n"
63
+ " pip install optimum[onnxruntime] onnx onnxruntime"
64
+ )
65
+
66
+ onnx_path = os.path.join(model_dir, "model.onnx")
67
+
68
+ print("[2/4] Exporting to ONNX via HuggingFace Optimum...")
69
+ # ORTModelForSequenceClassification handles the entire export pipeline
70
+ ort_model = ORTModelForSequenceClassification.from_pretrained(
71
+ model_dir,
72
+ export=True, # triggers ONNX export from PyTorch weights
73
+ )
74
+ tokenizer = AutoTokenizer.from_pretrained(model_dir, use_fast=True)
75
+
76
+ # Save ONNX model + config to same directory
77
+ ort_model.save_pretrained(model_dir)
78
+ print(f" Saved: {onnx_path}")
79
+
80
+ # ── VALIDATION ──────────────────────────────────────────────────────────
81
+ print("\n[3/4] Validating ONNX output against PyTorch...")
82
+
83
+ import torch # type: ignore
84
+ from transformers import AutoModelForSequenceClassification # type: ignore
85
+ import onnxruntime as ort # type: ignore
86
+
87
+ pt_model = AutoModelForSequenceClassification.from_pretrained(model_dir)
88
+ pt_model.eval()
89
+
90
+ session = ort.InferenceSession(onnx_path, providers=["CPUExecutionProvider"])
91
+ onnx_input_names = [inp.name for inp in session.get_inputs()]
92
+
93
+ max_diff = 0.0
94
+ all_pass = True
95
+
96
+ for sentence in TEST_SENTENCES:
97
+ # PyTorch
98
+ enc_pt = tokenizer(
99
+ sentence, truncation=True, padding="max_length",
100
+ max_length=256, return_tensors="pt"
101
+ )
102
+ enc_pt.pop("token_type_ids", None)
103
+ with torch.no_grad():
104
+ pt_logits = pt_model(**enc_pt).logits.numpy()
105
+
106
+ # ONNX
107
+ enc_np = tokenizer(
108
+ sentence, truncation=True, padding="max_length",
109
+ max_length=256, return_tensors="np"
110
+ )
111
+ onnx_inputs = {
112
+ name: enc_np[name].astype(np.int64)
113
+ for name in onnx_input_names
114
+ if name in enc_np
115
+ }
116
+ onnx_logits = session.run(None, onnx_inputs)[0]
117
+
118
+ diff = float(np.max(np.abs(pt_logits - onnx_logits)))
119
+ max_diff = max(max_diff, diff)
120
+ status = "PASS" if diff < 1e-3 else "WARN"
121
+ if diff >= 1e-3:
122
+ all_pass = False
123
+ print(f" [{status}] '{sentence[:40]}...' | max_diff={diff:.6f}")
124
+
125
+ print(f"\n Overall max diff: {max_diff:.6f} ({'OK' if all_pass else 'WARNING'})")
126
+
127
+ # ── SUMMARY ─────────────────────────────────────────────────────────────
128
+ print("\n[4/4] Export complete!")
129
+ print(f" ONNX model: {onnx_path}")
130
+ print(f" File size: {os.path.getsize(onnx_path) / 1e6:.1f} MB")
131
+ print("\n The server will automatically load ONNX on next startup.")
132
+ print("=" * 60)
133
+
134
+
135
+ if __name__ == "__main__":
136
+ parser = argparse.ArgumentParser(
137
+ description="ToxiGuard AI — Export PyTorch model to ONNX"
138
+ )
139
+ parser.add_argument(
140
+ "--model_dir",
141
+ type=str,
142
+ default=DEFAULT_MODEL_DIR,
143
+ help="Path to the fine-tuned HuggingFace model directory",
144
+ )
145
+ args = parser.parse_args()
146
+ export_onnx(args.model_dir)
ml/inference.py ADDED
@@ -0,0 +1,288 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — DeBERTa Inference Engine
3
+ =========================================
4
+ Production-grade inference using ONNX Runtime (primary) with PyTorch fallback.
5
+
6
+ ONNX Runtime achieves <50ms latency vs ~200ms with raw PyTorch.
7
+ DeBERTa-v3-base fine-tuned on Jigsaw multi-label toxic comment dataset.
8
+
9
+ Label schema (Jigsaw multi-label):
10
+ ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
11
+
12
+ Usage:
13
+ from ml.inference import TransformerInference
14
+ engine = TransformerInference("ml/models/toxiguard-deberta")
15
+ result = engine.predict("you are an idiot")
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import os
21
+ import numpy as np
22
+ from typing import Optional
23
+
24
+ from app.core.logger import logger
25
+
26
+ # When True, the PyTorch fallback is disabled entirely.
27
+ # Set ONNX_ONLY=true in production (Oracle A1 / Docker) to avoid loading
28
+ # the full 2 GB PyTorch package when only model.onnx is present.
29
+ _ONNX_ONLY: bool = os.getenv("ONNX_ONLY", "false").lower() in ("1", "true", "yes")
30
+
31
+ # ──────────────────────────────────────────────────────────────────────────────
32
+ # CONSTANTS
33
+ # ──────────────────────────────────────────────────────────────────────────────
34
+
35
+ LABEL_COLUMNS: list[str] = [
36
+ "toxic",
37
+ "severe_toxic",
38
+ "obscene",
39
+ "threat",
40
+ "insult",
41
+ "identity_hate",
42
+ ]
43
+
44
+ MAX_LENGTH: int = 256
45
+
46
+ # Sigmoid: σ(x) = 1 / (1 + e^-x) — applied to raw logits
47
+ def _sigmoid(x: np.ndarray) -> np.ndarray:
48
+ return 1.0 / (1.0 + np.exp(-x))
49
+
50
+
51
+ # ──────────────────────────────────────────────────────────────────────────────
52
+ # INFERENCE ENGINE
53
+ # ──────────────────────────────────────────────────────────────────────────────
54
+
55
+ class TransformerInference:
56
+ """
57
+ ONNX-optimised inference for DeBERTa-v3-base toxicity classifier.
58
+
59
+ Load priority:
60
+ 1. ONNX model (model.onnx) — fastest, ~40ms/request
61
+ 2. PyTorch model — fallback, ~200ms/request
62
+
63
+ DeBERTa-v3 notes:
64
+ - Uses SentencePiece tokenizer (requires `sentencepiece` package)
65
+ - Does NOT use token_type_ids — excluded from ONNX inputs
66
+ - Multi-label output: sigmoid applied to logits, not softmax
67
+ """
68
+
69
+ def __init__(self, model_dir: str) -> None:
70
+ self.model_dir: str = model_dir
71
+ self.onnx_path: str = os.path.join(model_dir, "model.onnx")
72
+ self.label_columns: list[str] = LABEL_COLUMNS
73
+
74
+ # Declared upfront to avoid AttributeError on partial load failures
75
+ self.session = None # onnxruntime.InferenceSession | None
76
+ self.tokenizer = None # transformers.PreTrainedTokenizerBase | None
77
+ self.pt_model = None # transformers.AutoModelForSequenceClassification | None
78
+ self._onnx_input_names: list[str] = []
79
+
80
+ self._load()
81
+
82
+ # ── LOADING ───────────────────────────────────────────────────────────────
83
+
84
+ def _load(self) -> None:
85
+ """Load ONNX model (or PyTorch fallback) and tokenizer."""
86
+ self._load_tokenizer()
87
+ self._load_model()
88
+
89
+ def _load_tokenizer(self) -> None:
90
+ """Load HuggingFace tokenizer (SentencePiece for DeBERTa-v3)."""
91
+ try:
92
+ from transformers import AutoTokenizer # type: ignore
93
+ self.tokenizer = AutoTokenizer.from_pretrained(
94
+ self.model_dir,
95
+ use_fast=True,
96
+ )
97
+ logger.info(f"[ModelLoad] Tokenizer loaded from {self.model_dir}")
98
+ except Exception as exc:
99
+ logger.error(f"[ModelLoad] Tokenizer load FAILED: {exc}")
100
+ self.tokenizer = None
101
+
102
+ def _load_model(self) -> None:
103
+ """Attempt ONNX load, fall back to PyTorch (unless ONNX_ONLY=true)."""
104
+ if self._try_load_onnx():
105
+ return
106
+ if _ONNX_ONLY:
107
+ logger.error(
108
+ "[ModelLoad] ONNX load failed and ONNX_ONLY=true — "
109
+ "PyTorch fallback is DISABLED. "
110
+ "Ensure model.onnx exists at: %s",
111
+ self.onnx_path,
112
+ )
113
+ return
114
+ self._try_load_pytorch()
115
+
116
+ def _try_load_onnx(self) -> bool:
117
+ """Return True if ONNX model loaded successfully."""
118
+ if not os.path.exists(self.onnx_path):
119
+ logger.info("[ModelLoad] No model.onnx found — will use PyTorch")
120
+ return False
121
+ try:
122
+ import onnxruntime as ort # type: ignore
123
+ self.session = ort.InferenceSession(
124
+ self.onnx_path,
125
+ providers=["CPUExecutionProvider"],
126
+ )
127
+ # Cache the actual input names from the ONNX graph
128
+ self._onnx_input_names = [inp.name for inp in self.session.get_inputs()]
129
+ logger.info(
130
+ f"[ModelLoad] ONNX model loaded — inputs: {self._onnx_input_names}"
131
+ )
132
+ return True
133
+ except Exception as exc:
134
+ logger.warning(f"[ModelLoad] ONNX load failed ({exc}), trying PyTorch")
135
+ self.session = None
136
+ return False
137
+
138
+ def _try_load_pytorch(self) -> None:
139
+ """Load PyTorch model as fallback."""
140
+ try:
141
+ import torch # type: ignore
142
+ from transformers import AutoModelForSequenceClassification # type: ignore
143
+
144
+ self.pt_model = AutoModelForSequenceClassification.from_pretrained(
145
+ self.model_dir,
146
+ num_labels=len(LABEL_COLUMNS),
147
+ )
148
+ self.pt_model.eval()
149
+ logger.info("[ModelLoad] PyTorch model loaded (ONNX fallback)")
150
+ except Exception as exc:
151
+ logger.error(f"[ModelLoad] PyTorch load also FAILED: {exc}")
152
+ self.pt_model = None
153
+
154
+ # ── READINESS ─────────────────────────────────────────────────────────────
155
+
156
+ @property
157
+ def is_ready(self) -> bool:
158
+ """True if at least a tokenizer and one model backend are loaded."""
159
+ return self.tokenizer is not None and (
160
+ self.session is not None or self.pt_model is not None
161
+ )
162
+
163
+ @property
164
+ def backend(self) -> str:
165
+ if self.session:
166
+ return "onnx"
167
+ if self.pt_model:
168
+ return "pytorch"
169
+ return "none"
170
+
171
+ # ── INFERENCE ─────────────────────────────────────────────────────────────
172
+
173
+ def predict(self, text: str, threshold: float = 0.5) -> Optional[dict]:
174
+ """
175
+ Run toxicity classification on a single text.
176
+
177
+ Args:
178
+ text: Input text (raw — tokenizer handles normalisation).
179
+ threshold: Probability cutoff for binary label detection.
180
+
181
+ Returns:
182
+ {
183
+ "toxic": bool,
184
+ "toxicity_probability": float, # max across all labels
185
+ "labels": {"toxic": 0.93, ...}, # per-label sigmoid probs
186
+ "severity": "low" | "medium" | "high",
187
+ "detected_categories": ["toxic", "insult"],
188
+ "source": "transformer_onnx" | "transformer_pytorch",
189
+ }
190
+ or None if model is not ready.
191
+ """
192
+ if not self.is_ready:
193
+ logger.warning("[Inference] Model not ready — returning None")
194
+ return None
195
+
196
+ if not text or not text.strip():
197
+ return None
198
+
199
+ try:
200
+ probs = self._run_inference(text)
201
+ except Exception as exc:
202
+ logger.error(f"[Inference] Runtime error: {exc}")
203
+ return None
204
+
205
+ return self._build_result(probs, threshold)
206
+
207
+ def _run_inference(self, text: str) -> np.ndarray:
208
+ """Tokenise and run model. Returns sigmoid probabilities array."""
209
+ if self.session:
210
+ return self._infer_onnx(text)
211
+ return self._infer_pytorch(text)
212
+
213
+ def _infer_onnx(self, text: str) -> np.ndarray:
214
+ """ONNX Runtime inference path."""
215
+ encoding = self.tokenizer(
216
+ text,
217
+ truncation=True,
218
+ padding="max_length",
219
+ max_length=MAX_LENGTH,
220
+ return_tensors="np",
221
+ )
222
+ # DeBERTa-v3 doesn't use token_type_ids — only pass what ONNX graph expects
223
+ inputs = {
224
+ name: encoding[name].astype(np.int64)
225
+ for name in self._onnx_input_names
226
+ if name in encoding
227
+ }
228
+ logits: np.ndarray = self.session.run(None, inputs)[0]
229
+ return _sigmoid(logits)[0] # shape: (num_labels,)
230
+
231
+ def _infer_pytorch(self, text: str) -> np.ndarray:
232
+ """PyTorch fallback inference path."""
233
+ import torch # type: ignore
234
+
235
+ inputs = self.tokenizer(
236
+ text,
237
+ truncation=True,
238
+ padding="max_length",
239
+ max_length=MAX_LENGTH,
240
+ return_tensors="pt",
241
+ )
242
+ # DeBERTa-v3 doesn't use token_type_ids — remove if present
243
+ inputs.pop("token_type_ids", None)
244
+
245
+ with torch.no_grad():
246
+ logits = self.pt_model(**inputs).logits
247
+ return torch.sigmoid(logits).numpy()[0] # shape: (num_labels,)
248
+
249
+ def _build_result(self, probs: np.ndarray, threshold: float) -> dict:
250
+ """Convert raw sigmoid probabilities into a structured result dict."""
251
+ label_scores = {
252
+ col: round(float(probs[i]), 4)
253
+ for i, col in enumerate(self.label_columns)
254
+ }
255
+ detected = [col for col, score in label_scores.items() if score >= threshold]
256
+
257
+ # Use max across ALL labels as the top-level toxicity probability
258
+ toxicity_prob = float(np.max(probs))
259
+
260
+ if toxicity_prob >= 0.85:
261
+ severity = "high"
262
+ elif toxicity_prob >= 0.60:
263
+ severity = "medium"
264
+ else:
265
+ severity = "low"
266
+
267
+ source = (
268
+ "transformer_onnx" if self.session else "transformer_pytorch"
269
+ )
270
+
271
+ return {
272
+ "toxic": len(detected) > 0,
273
+ "toxicity_probability": round(toxicity_prob, 4),
274
+ "labels": label_scores,
275
+ "severity": severity,
276
+ "detected_categories": detected,
277
+ "source": source,
278
+ }
279
+
280
+ # ── DEBUG ─────────────────────────────────────────────────────────────────
281
+
282
+ def __repr__(self) -> str:
283
+ return (
284
+ f"TransformerInference("
285
+ f"backend={self.backend!r}, "
286
+ f"ready={self.is_ready}, "
287
+ f"labels={len(self.label_columns)})"
288
+ )
ml/models/toxiguard-deberta/added_tokens.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "[MASK]": 128000
3
+ }
ml/models/toxiguard-deberta/config.json ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "DebertaV2ForSequenceClassification"
4
+ ],
5
+ "attention_probs_dropout_prob": 0.1,
6
+ "dtype": "float32",
7
+ "hidden_act": "gelu",
8
+ "hidden_dropout_prob": 0.1,
9
+ "hidden_size": 768,
10
+ "id2label": {
11
+ "0": "LABEL_0",
12
+ "1": "LABEL_1",
13
+ "2": "LABEL_2",
14
+ "3": "LABEL_3",
15
+ "4": "LABEL_4",
16
+ "5": "LABEL_5"
17
+ },
18
+ "initializer_range": 0.02,
19
+ "intermediate_size": 3072,
20
+ "label2id": {
21
+ "LABEL_0": 0,
22
+ "LABEL_1": 1,
23
+ "LABEL_2": 2,
24
+ "LABEL_3": 3,
25
+ "LABEL_4": 4,
26
+ "LABEL_5": 5
27
+ },
28
+ "layer_norm_eps": 1e-07,
29
+ "legacy": true,
30
+ "max_position_embeddings": 512,
31
+ "max_relative_positions": -1,
32
+ "model_type": "deberta-v2",
33
+ "norm_rel_ebd": "layer_norm",
34
+ "num_attention_heads": 12,
35
+ "num_hidden_layers": 12,
36
+ "pad_token_id": 0,
37
+ "pooler_dropout": 0,
38
+ "pooler_hidden_act": "gelu",
39
+ "pooler_hidden_size": 768,
40
+ "pos_att_type": [
41
+ "p2c",
42
+ "c2p"
43
+ ],
44
+ "position_biased_input": false,
45
+ "position_buckets": 256,
46
+ "problem_type": "multi_label_classification",
47
+ "relative_attention": true,
48
+ "share_att_key": true,
49
+ "transformers_version": "4.57.6",
50
+ "type_vocab_size": 0,
51
+ "vocab_size": 128100
52
+ }
ml/models/toxiguard-deberta/model.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b75737dbc7bf0bda553c86ce211a9d90106525c6a8ce2653707904ed555719e9
3
+ size 738617302
ml/models/toxiguard-deberta/special_tokens_map.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "[CLS]",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "cls_token": {
10
+ "content": "[CLS]",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "eos_token": {
17
+ "content": "[SEP]",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "mask_token": {
24
+ "content": "[MASK]",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ },
30
+ "pad_token": {
31
+ "content": "[PAD]",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false
36
+ },
37
+ "sep_token": {
38
+ "content": "[SEP]",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false
43
+ },
44
+ "unk_token": {
45
+ "content": "[UNK]",
46
+ "lstrip": false,
47
+ "normalized": true,
48
+ "rstrip": false,
49
+ "single_word": false
50
+ }
51
+ }
ml/models/toxiguard-deberta/spm.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
3
+ size 2464616
ml/models/toxiguard-deberta/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
ml/models/toxiguard-deberta/tokenizer_config.json ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "[PAD]",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "[CLS]",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "[SEP]",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "[UNK]",
29
+ "lstrip": false,
30
+ "normalized": true,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "128000": {
36
+ "content": "[MASK]",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "bos_token": "[CLS]",
45
+ "clean_up_tokenization_spaces": false,
46
+ "cls_token": "[CLS]",
47
+ "do_lower_case": false,
48
+ "eos_token": "[SEP]",
49
+ "extra_special_tokens": {},
50
+ "mask_token": "[MASK]",
51
+ "model_max_length": 1000000000000000019884624838656,
52
+ "pad_token": "[PAD]",
53
+ "sep_token": "[SEP]",
54
+ "sp_model_kwargs": {},
55
+ "split_by_punct": false,
56
+ "tokenizer_class": "DebertaV2Tokenizer",
57
+ "unk_token": "[UNK]",
58
+ "vocab_type": "spm"
59
+ }
ml/train_transformer.py ADDED
@@ -0,0 +1,326 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ToxiGuard AI — Transformer Model Training Pipeline
3
+ ===================================================
4
+ Fine-tunes DeBERTa-v3-base for multi-label toxic comment classification.
5
+
6
+ Usage:
7
+ python -m ml.train_transformer # full training
8
+ python -m ml.train_transformer --epochs 3 --batch_size 16
9
+ python -m ml.train_transformer --sample_size 20000 # quick test run
10
+ python -m ml.train_transformer --export_onnx # also export to ONNX
11
+
12
+ Requirements:
13
+ pip install transformers datasets torch accelerate scikit-learn sentencepiece
14
+
15
+ Output:
16
+ ml/models/toxiguard-deberta/ — HuggingFace model + tokenizer
17
+ ml/models/training_metrics.json — Evaluation metrics
18
+ """
19
+
20
+ import os
21
+ import json
22
+ import argparse
23
+ import numpy as np
24
+ from datetime import datetime
25
+
26
+ import torch
27
+ from torch.utils.data import Dataset
28
+
29
+ from transformers import (
30
+ AutoTokenizer,
31
+ AutoModelForSequenceClassification,
32
+ TrainingArguments,
33
+ Trainer,
34
+ EarlyStoppingCallback,
35
+ )
36
+
37
+ from sklearn.metrics import (
38
+ f1_score,
39
+ precision_score,
40
+ recall_score,
41
+ roc_auc_score,
42
+ classification_report,
43
+ )
44
+ from sklearn.model_selection import train_test_split
45
+
46
+ import pandas as pd
47
+
48
+ # ── CONFIGURATION ──────────────────────────────────────────────────────────────
49
+
50
+ BASE_DIR = os.path.dirname(os.path.dirname(__file__))
51
+ DATA_DIR = os.path.join(BASE_DIR, "data")
52
+ MODEL_OUTPUT_DIR = os.path.join(os.path.dirname(__file__), "models", "toxiguard-deberta")
53
+
54
+ # Model registry — ranked by accuracy vs speed tradeoff
55
+ MODEL_REGISTRY = {
56
+ "deberta-v3-base": "microsoft/deberta-v3-base", # default — best accuracy
57
+ "deberta-v3-small": "microsoft/deberta-v3-small", # faster, slightly lower accuracy
58
+ "roberta-base": "roberta-base",
59
+ "bert-base": "bert-base-uncased",
60
+ "distilbert": "distilbert-base-uncased",
61
+ }
62
+
63
+ # Jigsaw multi-label columns
64
+ LABEL_COLUMNS = [
65
+ "toxic",
66
+ "severe_toxic",
67
+ "obscene",
68
+ "threat",
69
+ "insult",
70
+ "identity_hate",
71
+ ]
72
+
73
+ MAX_LENGTH = 256
74
+
75
+
76
+ # ── DATASET ────────────────────────────────────────────────────────────────────
77
+
78
+ class ToxicCommentDataset(Dataset):
79
+ """PyTorch Dataset for multi-label toxic comment classification."""
80
+
81
+ def __init__(
82
+ self,
83
+ texts: np.ndarray,
84
+ labels: np.ndarray,
85
+ tokenizer,
86
+ max_length: int = MAX_LENGTH,
87
+ ) -> None:
88
+ self.texts = texts
89
+ self.labels = labels
90
+ self.tokenizer = tokenizer
91
+ self.max_length = max_length
92
+
93
+ def __len__(self) -> int:
94
+ return len(self.texts)
95
+
96
+ def __getitem__(self, idx: int) -> dict:
97
+ text = str(self.texts[idx])
98
+ encoding = self.tokenizer(
99
+ text,
100
+ truncation=True,
101
+ padding="max_length",
102
+ max_length=self.max_length,
103
+ return_tensors="pt",
104
+ )
105
+ item = {
106
+ "input_ids": encoding["input_ids"].squeeze(),
107
+ "attention_mask": encoding["attention_mask"].squeeze(),
108
+ "labels": torch.tensor(self.labels[idx], dtype=torch.float),
109
+ }
110
+ # DeBERTa-v3 does not use token_type_ids — skip if present
111
+ if "token_type_ids" in encoding:
112
+ item["token_type_ids"] = encoding["token_type_ids"].squeeze()
113
+ return item
114
+
115
+
116
+ # ── METRICS ────────────────────────────────────────────────────────────────────
117
+
118
+ def compute_metrics(eval_pred) -> dict:
119
+ """Compute multi-label classification metrics during training."""
120
+ predictions, labels = eval_pred
121
+ probs = torch.sigmoid(torch.tensor(predictions)).numpy()
122
+ preds = (probs >= 0.5).astype(int)
123
+
124
+ f1_macro = f1_score(labels, preds, average="macro", zero_division=0)
125
+ f1_micro = f1_score(labels, preds, average="micro", zero_division=0)
126
+ precision = precision_score(labels, preds, average="macro", zero_division=0)
127
+ recall = recall_score(labels, preds, average="macro", zero_division=0)
128
+
129
+ try:
130
+ auc_roc = roc_auc_score(labels, probs, average="macro")
131
+ except ValueError:
132
+ auc_roc = 0.0
133
+
134
+ per_class_f1 = f1_score(labels, preds, average=None, zero_division=0)
135
+
136
+ metrics = {
137
+ "f1_macro": f1_macro,
138
+ "f1_micro": f1_micro,
139
+ "precision_macro": precision,
140
+ "recall_macro": recall,
141
+ "auc_roc_macro": auc_roc,
142
+ }
143
+ for i, col in enumerate(LABEL_COLUMNS):
144
+ metrics[f"f1_{col}"] = float(per_class_f1[i]) if i < len(per_class_f1) else 0.0
145
+
146
+ return metrics
147
+
148
+
149
+ # ── DATA LOADING ───────────────────────────────────────────────────────────────
150
+
151
+ def load_jigsaw_data(data_dir: str, sample_size: int | None = None):
152
+ """Load Jigsaw Toxic Comment dataset from CSV."""
153
+ train_path = os.path.join(data_dir, "train.csv")
154
+ if not os.path.exists(train_path):
155
+ raise FileNotFoundError(
156
+ f"Training data not found at {train_path}\n"
157
+ f"Download from: https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge"
158
+ )
159
+
160
+ print(f"[Data] Loading from {train_path}...")
161
+ df = pd.read_csv(train_path)
162
+
163
+ if sample_size and sample_size < len(df):
164
+ print(f"[Data] Sampling {sample_size} rows (full dataset = {len(df)})")
165
+ df = df.sample(n=sample_size, random_state=42)
166
+
167
+ texts = df["comment_text"].fillna("").values
168
+ labels = df[LABEL_COLUMNS].values.astype(float)
169
+
170
+ print(f"[Data] Loaded {len(df)} samples")
171
+ for col in LABEL_COLUMNS:
172
+ print(f" {col}: {df[col].sum()} ({df[col].mean()*100:.1f}%)")
173
+
174
+ return texts, labels
175
+
176
+
177
+ # ── TRAINING PIPELINE ──────────────────────────────────────────────────────────
178
+
179
+ def train(args) -> dict:
180
+ """Full DeBERTa fine-tuning pipeline."""
181
+ print("=" * 60)
182
+ print("ToxiGuard AI — DeBERTa Training Pipeline")
183
+ print("=" * 60)
184
+
185
+ model_name = MODEL_REGISTRY.get(args.model, args.model)
186
+ print(f"\n[Model] Base: {model_name}")
187
+ print(f"[Device] {'GPU (CUDA)' if torch.cuda.is_available() else 'CPU'}")
188
+
189
+ # ── TOKENIZER & MODEL ──────────────────────────────────────────────────
190
+ tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
191
+ model = AutoModelForSequenceClassification.from_pretrained(
192
+ model_name,
193
+ num_labels=len(LABEL_COLUMNS),
194
+ problem_type="multi_label_classification",
195
+ ignore_mismatched_sizes=True,
196
+ )
197
+
198
+ # ── DATA ───────────────────────────────────────────────────────────────
199
+ texts, labels = load_jigsaw_data(DATA_DIR, sample_size=args.sample_size)
200
+
201
+ train_texts, val_texts, train_labels, val_labels = train_test_split(
202
+ texts, labels, test_size=0.1, random_state=42, shuffle=True
203
+ )
204
+ print(f"\n[Split] Train: {len(train_texts)} | Val: {len(val_texts)}")
205
+
206
+ train_dataset = ToxicCommentDataset(train_texts, train_labels, tokenizer)
207
+ val_dataset = ToxicCommentDataset(val_texts, val_labels, tokenizer)
208
+
209
+ # ── TRAINING ARGS ──────────────────────────────────────────────────────
210
+ # NOTE: `eval_strategy` replaces deprecated `evaluation_strat` (HF >= 4.41)
211
+ training_args = TrainingArguments(
212
+ output_dir=MODEL_OUTPUT_DIR,
213
+ num_train_epochs=args.epochs,
214
+ per_device_train_batch_size=args.batch_size,
215
+ per_device_eval_batch_size=args.batch_size * 2,
216
+ learning_rate=args.learning_rate,
217
+ weight_decay=0.01,
218
+ warmup_ratio=0.1,
219
+ eval_strategy="epoch", # Fixed: was `evaluation_strat` (deprecated)
220
+ save_strategy="epoch",
221
+ load_best_model_at_end=True,
222
+ metric_for_best_model="f1_macro",
223
+ greater_is_better=True,
224
+ logging_dir=os.path.join(MODEL_OUTPUT_DIR, "logs"),
225
+ logging_steps=50,
226
+ save_total_limit=2,
227
+ save_safetensors=True,
228
+ fp16=torch.cuda.is_available(),
229
+ dataloader_num_workers=2,
230
+ report_to=["mlflow"] if args.mlflow else [],
231
+ run_name=f"toxiguard-{args.model}-{datetime.now().strftime('%Y%m%d-%H%M')}",
232
+ )
233
+
234
+ # ── TRAINER ────────────────────────────────────────────────────────────
235
+ trainer = Trainer(
236
+ model=model,
237
+ args=training_args,
238
+ train_dataset=train_dataset,
239
+ eval_dataset=val_dataset,
240
+ compute_metrics=compute_metrics,
241
+ callbacks=[EarlyStoppingCallback(early_stopping_patience=2)],
242
+ )
243
+
244
+ # ── TRAIN ──────────────────────────────────────────────────────────────
245
+ print(f"\n[Train] Starting — epochs={args.epochs} | batch={args.batch_size} | lr={args.learning_rate}")
246
+ trainer.train()
247
+
248
+ # ── EVALUATE ───────────────────────────────────────────────────────────
249
+ print("\n[Eval] Running final evaluation...")
250
+ eval_results = trainer.evaluate()
251
+
252
+ print("\n" + "=" * 60)
253
+ print("EVALUATION RESULTS")
254
+ print("=" * 60)
255
+ for key, value in sorted(eval_results.items()):
256
+ if isinstance(value, float):
257
+ print(f" {key}: {value:.4f}")
258
+
259
+ # ── SAVE MODEL ─────────────────────────────────────────────────────────
260
+ print(f"\n[Save] Saving to {MODEL_OUTPUT_DIR}...")
261
+ trainer.save_model(MODEL_OUTPUT_DIR)
262
+ tokenizer.save_pretrained(MODEL_OUTPUT_DIR)
263
+
264
+ # Save training metrics JSON
265
+ metrics_path = os.path.join(MODEL_OUTPUT_DIR, "training_metrics.json")
266
+ metrics_data = {
267
+ "model": model_name,
268
+ "trained_at": datetime.now().isoformat(),
269
+ "epochs": args.epochs,
270
+ "batch_size": args.batch_size,
271
+ "learning_rate": args.learning_rate,
272
+ "train_samples": len(train_texts),
273
+ "val_samples": len(val_texts),
274
+ "label_columns": LABEL_COLUMNS,
275
+ "metrics": {
276
+ k: round(v, 4) if isinstance(v, float) else v
277
+ for k, v in eval_results.items()
278
+ },
279
+ }
280
+ with open(metrics_path, "w") as f:
281
+ json.dump(metrics_data, f, indent=2)
282
+ print(f"[Save] Metrics saved to {metrics_path}")
283
+
284
+ # ── PER-CLASS REPORT ────────────────────────────────────────────────────
285
+ print("\n[Report] Per-class Classification Report:")
286
+ val_preds = trainer.predict(val_dataset)
287
+ probs = torch.sigmoid(torch.tensor(val_preds.predictions)).numpy()
288
+ preds = (probs >= 0.5).astype(int)
289
+ print(classification_report(
290
+ val_labels, preds,
291
+ target_names=LABEL_COLUMNS,
292
+ zero_division=0,
293
+ ))
294
+
295
+ # ── ONNX EXPORT (optional) ──────────────────────────────────────────────
296
+ if args.export_onnx:
297
+ print("\n[ONNX] Exporting to ONNX Runtime format...")
298
+ try:
299
+ from ml.export_onnx import export_onnx
300
+ export_onnx(MODEL_OUTPUT_DIR)
301
+ except ImportError as exc:
302
+ print(f"[ONNX] Export skipped — missing packages: {exc}")
303
+ print(" Run: pip install optimum[onnxruntime] onnx")
304
+
305
+ print("\n[Done] Training complete!")
306
+ return eval_results
307
+
308
+ # ── CLI ────────────────────────────────────────────────────────────────────────
309
+
310
+ if __name__ == "__main__":
311
+ parser = argparse.ArgumentParser(
312
+ description="ToxiGuard AI — Fine-tune DeBERTa for toxicity classification"
313
+ )
314
+ parser.add_argument(
315
+ "--model", type=str, default="deberta-v3-base",
316
+ choices=list(MODEL_REGISTRY.keys()),
317
+ help="Base model to fine-tune (default: deberta-v3-base)",
318
+ )
319
+ parser.add_argument("--epochs", type=int, default=3, help="Training epochs")
320
+ parser.add_argument("--batch_size", type=int, default=16, help="Batch size per device")
321
+ parser.add_argument("--learning_rate", type=float, default=2e-5, help="Learning rate")
322
+ parser.add_argument("--sample_size", type=int, default=None, help="Dataset size limit (for testing)")
323
+ parser.add_argument("--mlflow", action="store_true", help="Enable MLflow tracking")
324
+ parser.add_argument("--export_onnx", action="store_true", help="Export to ONNX after training")
325
+
326
+ train(parser.parse_args())
models.py ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # models.py
2
+
3
+ from sqlalchemy import Column, Integer, String, DateTime, Float, Boolean, Text
4
+ from datetime import datetime
5
+
6
+ from database import Base
7
+
8
+
9
+ class User(Base):
10
+ __tablename__ = "users"
11
+
12
+ id = Column(Integer, primary_key=True, index=True)
13
+
14
+ email = Column(String, unique=True, index=True, nullable=False)
15
+
16
+ password_hash = Column(String, nullable=False)
17
+
18
+ created_at = Column(DateTime, default=datetime.utcnow)
19
+
20
+ # SaaS fields
21
+ api_key = Column(String, unique=True, index=True, nullable=True)
22
+
23
+ plan = Column(String, default="free")
24
+
25
+ usage_count = Column(Integer, default=0)
26
+
27
+ # Tracks the last time the user made an authenticated API request.
28
+ # Useful for analytics, stale account cleanup, and pro dashboard UX.
29
+ last_used = Column(DateTime, nullable=True, default=None)
30
+
31
+
32
+ class PredictionLog(Base):
33
+ """
34
+ Stores every /predict response for drift monitoring and analytics.
35
+ Used by DriftMonitor to compute rolling confidence stats.
36
+ """
37
+ __tablename__ = "prediction_logs"
38
+
39
+ id = Column(Integer, primary_key=True, index=True)
40
+ user_id = Column(Integer, nullable=True, index=True)
41
+ text_hash = Column(String, nullable=True) # SHA256 of input (no PII)
42
+ text_length = Column(Integer, nullable=True)
43
+ confidence = Column(Float, nullable=False) # 0.0–1.0 toxicity probability
44
+ toxic = Column(Boolean, nullable=False)
45
+ severity = Column(String, nullable=True) # low/medium/high
46
+ source = Column(String, nullable=True) # transformer_onnx / rules / etc.
47
+ timestamp = Column(DateTime, default=datetime.utcnow, index=True)
48
+
49
+
50
+ class FeedbackSample(Base):
51
+ """
52
+ Active Learning feedback table.
53
+ Users flag incorrect predictions (FP/FN) to build a correction queue.
54
+ """
55
+ __tablename__ = "feedback_samples"
56
+
57
+ id = Column(Integer, primary_key=True, index=True)
58
+ user_id = Column(Integer, nullable=True, index=True)
59
+ input_text = Column(Text, nullable=False) # original text
60
+ predicted_toxic = Column(Boolean, nullable=False) # what model said
61
+ correct_label = Column(Boolean, nullable=False) # what user says is right
62
+ feedback_type = Column(String, nullable=False) # "false_positive" / "false_negative"
63
+ confidence_at_time = Column(Float, nullable=True) # model confidence when flagged
64
+ notes = Column(Text, nullable=True) # optional user comment
65
+ abusive_words = Column(Text, nullable=True) # specific abusive words identified by user
66
+ explanation = Column(Text, nullable=True) # explanation/why it is abusive
67
+ reviewed = Column(Boolean, default=False) # admin reviewed flag
68
+ created_at = Column(DateTime, default=datetime.utcnow, index=True)
requirements-prod.txt ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ToxiGuard AI — Production Dependencies (Hugging Face Spaces & Cloud)
2
+ # ==============================================================================
3
+ # Modern compatible dependency specifications for zero-conflict builds with Gradio 6 / FastAPI.
4
+
5
+ # Web Framework & Server
6
+ fastapi>=0.115.2
7
+ uvicorn[standard]>=0.30.0
8
+ pydantic[email]>=2.8.0
9
+ email-validator>=2.2.0
10
+ pydantic-settings>=2.4.0
11
+
12
+ # Database & Security
13
+ sqlalchemy>=2.0.0
14
+ passlib[bcrypt]>=1.7.4
15
+ bcrypt>=4.0.1
16
+ python-jose[cryptography]>=3.3.0
17
+ cryptography>=42.0.0
18
+ slowapi>=0.1.9
19
+
20
+ # Machine Learning & ONNX Runtime (Inference Engine)
21
+ onnxruntime>=1.19.0
22
+ transformers>=4.48.0
23
+ tokenizers>=0.20.0
24
+ sentencepiece>=0.2.0
25
+ scikit-learn>=1.5.0
26
+ joblib>=1.4.0
27
+ numpy>=1.26.0
28
+ scipy>=1.13.0
29
+
30
+ # NLP & Sentiment Preprocessing
31
+ nltk>=3.9.0
32
+ textblob>=0.18.0
33
+ vaderSentiment>=3.3.2
34
+
35
+ # LLM Fallback & Utils
36
+ openai>=1.50.0
37
+ httpx>=0.27.0
38
+ python-dotenv>=1.0.0
39
+ pandas>=2.2.0
40
+ aiofiles>=24.1.0
41
+ python-multipart>=0.0.18
42
+ loguru>=0.7.2
requirements.txt ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ToxiGuard AI — Production Dependencies (Hugging Face Spaces & Cloud)
2
+ # ==============================================================================
3
+ # Modern compatible dependency specifications for zero-conflict builds with Gradio 6 / FastAPI.
4
+
5
+ # Web Framework & Server
6
+ fastapi>=0.115.2
7
+ uvicorn[standard]>=0.30.0
8
+ pydantic[email]>=2.8.0
9
+ email-validator>=2.2.0
10
+ pydantic-settings>=2.4.0
11
+
12
+ # Database & Security
13
+ sqlalchemy>=2.0.0
14
+ passlib[bcrypt]>=1.7.4
15
+ bcrypt>=4.0.1
16
+ python-jose[cryptography]>=3.3.0
17
+ cryptography>=42.0.0
18
+ slowapi>=0.1.9
19
+
20
+ # Machine Learning & ONNX Runtime (Inference Engine)
21
+ onnxruntime>=1.19.0
22
+ transformers>=4.48.0
23
+ tokenizers>=0.20.0
24
+ sentencepiece>=0.2.0
25
+ scikit-learn>=1.5.0
26
+ joblib>=1.4.0
27
+ numpy>=1.26.0
28
+ scipy>=1.13.0
29
+
30
+ # NLP & Sentiment Preprocessing
31
+ nltk>=3.9.0
32
+ textblob>=0.18.0
33
+ vaderSentiment>=3.3.2
34
+
35
+ # LLM Fallback & Utils
36
+ openai>=1.50.0
37
+ httpx>=0.27.0
38
+ python-dotenv>=1.0.0
39
+ pandas>=2.2.0
40
+ aiofiles>=24.1.0
41
+ python-multipart>=0.0.18
42
+ loguru>=0.7.2