File size: 9,386 Bytes
0fe62e1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
"""
ToxiGuard AI β€” XAI Attribution Engine (Perturbation-Based)
============================================================
Computes per-word importance scores using Leave-One-Out (LOO) perturbation.
Works with ONNX-only model dirs β€” no PyTorch weights required.

Algorithm:
  1. Get baseline confidence score for the full text
  2. For each word, mask it out (replace with [MASK]-equivalent)
  3. Re-run inference on masked text
  4. Attribution score = baseline_confidence - masked_confidence
     β†’ Positive = word increased toxicity (important)
     β†’ Negative = word actually reduced toxicity
  5. Normalise scores to [0, 1]

Why perturbation over attention?
  - Works with ONNX (no PyTorch needed)
  - Model-agnostic β€” works with any classifier
  - More faithful: attention β‰  importance (known research finding, Jain & Wallace 2019)
  - This is the method used in production ML (Google, Meta content moderation)

Usage:
    from ml.explainability import get_explainer
    explainer = get_explainer(model_dir)
    result = explainer.explain("you are an idiot")
    # {"tokens": [{"token": "you", "score": 0.02, ...}, ...], "top_tokens": [...]}
"""

from __future__ import annotations

import re
import numpy as np
from typing import Optional

from app.core.logger import logger


# ──────────────────────────────────────────────────────────────────────────────
# HELPERS
# ──────────────────────────────────────────────────────────────────────────────

def _sigmoid(x: np.ndarray) -> np.ndarray:
    return 1.0 / (1.0 + np.exp(-x))


def _tokenize_words(text: str) -> list[str]:
    """Split text into words preserving order. Handles punctuation."""
    return re.findall(r"\S+", text)


def _mask_word(words: list[str], idx: int, mask: str = "") -> str:
    """Return text with word at idx replaced by mask (or removed)."""
    masked = words[:idx] + ([mask] if mask else []) + words[idx + 1:]
    return " ".join(masked).strip() or "."   # fallback so tokenizer doesn't choke


# ──────────────────────────────────────────────────────────────────────────────
# EXPLAINER
# ──────────────────────────────────────────────────────────────────────────────

class PerturbationExplainer:
    """
    Leave-One-Out attribution using the ONNX inference session directly.
    No PyTorch weights required.
    """

    def __init__(self, model_dir: str) -> None:
        self.model_dir = model_dir
        self.session = None
        self.tokenizer = None
        self._onnx_input_names: list[str] = []
        self._loaded = False
        self._load()

    def _load(self) -> None:
        import os

        if not os.path.isdir(self.model_dir):
            logger.warning(
                f"[XAI] PerturbationExplainer load failed: Directory '{self.model_dir}' not found. "
                "Explainability feature will be disabled."
            )
            self._loaded = False
            return

        onnx_path = os.path.join(self.model_dir, "model.onnx")
        if not os.path.exists(onnx_path):
            logger.warning(
                f"[XAI] PerturbationExplainer load failed: ONNX model file '{onnx_path}' not found. "
                "Explainability feature will be disabled."
            )
            self._loaded = False
            return

        try:
            import onnxruntime as ort
            from transformers import AutoTokenizer

            self.tokenizer = AutoTokenizer.from_pretrained(
                self.model_dir, use_fast=True
            )
            self.session = ort.InferenceSession(
                onnx_path,
                providers=["CPUExecutionProvider"],
            )
            self._onnx_input_names = [
                inp.name for inp in self.session.get_inputs()
            ]
            self._loaded = True
            logger.info("[XAI] PerturbationExplainer loaded (ONNX)")
        except Exception as exc:
            logger.warning(f"[XAI] PerturbationExplainer load failed: {exc}")
            self._loaded = False

    @property
    def is_ready(self) -> bool:
        return self._loaded

    def _infer(self, text: str) -> float:
        """Run ONNX inference and return max toxicity probability."""
        encoding = self.tokenizer(
            text,
            truncation=True,
            padding="max_length",
            max_length=128,
            return_tensors="np",
        )
        inputs = {
            name: encoding[name].astype(np.int64)
            for name in self._onnx_input_names
            if name in encoding
        }
        logits: np.ndarray = self.session.run(None, inputs)[0]
        probs = _sigmoid(logits)[0]          # shape: (num_labels,)
        return float(np.max(probs))

    def explain(self, text: str, max_words: int = 30) -> Optional[dict]:
        """
        LOO attribution β€” score each word by how much removing it
        changes the toxicity confidence.

        Args:
            text:      Raw input text.
            max_words: Limit words to analyze (keeps latency ~1-2s for long texts).

        Returns:
            {
                "tokens": [{"token": str, "score": float, "is_special": bool}],
                "top_tokens": [{"token": str, "score": float}],
                "method": "leave_one_out_perturbation",
                "num_tokens": int,
                "baseline_confidence": float,
            }
        """
        if not self.is_ready:
            return None

        text = text.strip()
        if not text:
            return None

        try:
            words = _tokenize_words(text)

            # Cap at max_words to keep latency manageable
            if len(words) > max_words:
                words = words[:max_words]

            # Step 1: baseline confidence
            baseline = self._infer(text)

            # Step 2: LOO β€” remove each word, measure confidence drop
            raw_scores: list[float] = []
            for i in range(len(words)):
                masked_text = _mask_word(words, i)
                masked_conf = self._infer(masked_text)
                # Positive score = removing this word LOWERED toxicity β†’ word is "toxic"
                raw_scores.append(baseline - masked_conf)

            from utils.abuse_words import get_word_severity

            # Step 3: Blend raw ML perturbation score with rule-based severity weights
            clamped: list[float] = []
            for i, word in enumerate(words):
                # Clean punctuation for rule lookups
                clean_word = re.sub(r"[^\w]", "", word.lower())
                severity = get_word_severity(clean_word)
                rule_boost = {"critical": 1.0, "high": 0.85, "moderate": 0.55}.get(severity, 0.0)

                raw_s = max(0.0, raw_scores[i])
                clamped.append(raw_s + rule_boost)

            max_val = max(clamped) if clamped else 0.0

            if max_val > 0:
                normalised = [round(s / max_val, 4) for s in clamped]
            else:
                normalised = [0.0] * len(clamped)

            # Build token list
            tokens = [
                {
                    "token": word,
                    "score": score,
                    "raw_delta": round(raw_scores[i], 4),
                    "is_special": False,
                }
                for i, (word, score) in enumerate(zip(words, normalised))
            ]

            # Top 5 most influential words
            top_tokens = sorted(
                [t for t in tokens if not t["is_special"]],
                key=lambda x: x["score"],
                reverse=True,
            )[:5]

            return {
                "tokens": tokens,
                "top_tokens": top_tokens,
                "method": "leave_one_out_perturbation",
                "num_tokens": len(tokens),
                "baseline_confidence": round(baseline, 4),
            }

        except Exception as exc:
            logger.error(f"[XAI] explain() failed: {exc}")
            return None


# ──────────────────────────────────────────────────────────────────────────────
# MODULE-LEVEL SINGLETON
# ──────────────────────────────────────────────────────────────────────────────

_explainer_instance: Optional[PerturbationExplainer] = None


def get_explainer(model_dir: str) -> PerturbationExplainer:
    global _explainer_instance
    if _explainer_instance is None:
        _explainer_instance = PerturbationExplainer(model_dir)
    return _explainer_instance