File size: 2,677 Bytes
215f97f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | """
Model evaluation — precision, recall, F1 (macro), and classification report.
Uses the held-out test set with a realistic class distribution (~5-8%
injection) so metrics approximate real-world performance.
"""
import numpy as np
from sklearn.metrics import precision_recall_fscore_support, classification_report
from app.ml.cnn.architecture import LABEL_NAMES
from app.core.logging import get_logger
logger = get_logger(__name__)
def decode_predictions(label_probs: np.ndarray) -> list[str]:
"""Convert softmax probability arrays to label strings.
Args:
label_probs: Array of shape ``(N, 3)`` — softmax output from the
``label`` head of the model.
Returns:
List of label strings (``"safe"``, ``"suspicious"``, ``"injection"``).
"""
indices = np.argmax(label_probs, axis=1)
return [LABEL_NAMES[i] for i in indices]
def evaluate(model, test_texts: np.ndarray, test_labels_onehot: np.ndarray) -> dict:
"""Evaluate the model on the test set.
Runs prediction, decodes labels, and computes macro-averaged
precision, recall, and F1 plus a per-class classification report.
Args:
model: Trained Keras model with dual output heads.
test_texts: Array of shape ``(N, 1)`` — raw text strings.
test_labels_onehot: One-hot encoded true labels, shape ``(N, 3)``.
Returns:
Dict with ``precision``, ``recall``, ``f1``, and ``report`` keys.
"""
# Run prediction — model returns [label_probs, category_probs]
predictions = model.predict(test_texts, verbose=0)
label_probs = predictions[0] # shape (N, 3)
# Decode predictions and true labels
pred_labels = decode_predictions(label_probs)
true_labels = decode_predictions(test_labels_onehot)
# Macro-averaged metrics
precision, recall, f1, _ = precision_recall_fscore_support(
true_labels, pred_labels, average="macro", zero_division=0
)
# Per-class report
# We dynamically determine labels to avoid ValueError if some classes are missing in test set
unique_labels = sorted(list(set(true_labels + pred_labels)))
report = classification_report(
true_labels,
pred_labels,
labels=unique_labels,
output_dict=True,
zero_division=0,
)
metrics = {
"precision": float(precision),
"recall": float(recall),
"f1": float(f1),
"report": report,
}
logger.info(
"Evaluation: precision=%.4f, recall=%.4f, F1=%.4f",
precision,
recall,
f1,
)
return metrics
|