File size: 3,626 Bytes
9313a90 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 | from __future__ import annotations
from pathlib import Path
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.metrics import (
accuracy_score,
confusion_matrix,
f1_score,
precision_score,
recall_score,
roc_auc_score,
roc_curve,
)
def choose_f1_threshold(labels: np.ndarray, probabilities: np.ndarray) -> float:
"""Choose the validation threshold with the best F1 score."""
candidates = np.linspace(0.20, 0.80, 121)
scores = np.asarray(
[f1_score(labels, probabilities >= value, zero_division=0) for value in candidates]
)
# Several thresholds often tie on a small validation set. Choosing the one
# closest to the conventional 0.5 avoids an arbitrary low-threshold bias.
tied = np.flatnonzero(np.isclose(scores, scores.max()))
best_index = tied[np.argmin(np.abs(candidates[tied] - 0.5))]
return float(candidates[best_index])
def classification_metrics(
labels: np.ndarray, probabilities: np.ndarray, threshold: float
) -> dict[str, float | int]:
predictions = (probabilities >= threshold).astype(int)
tn, fp, fn, tp = confusion_matrix(labels, predictions, labels=[0, 1]).ravel()
return {
"accuracy": float(accuracy_score(labels, predictions)),
"precision": float(precision_score(labels, predictions, zero_division=0)),
"recall": float(recall_score(labels, predictions, zero_division=0)),
"specificity": float(tn / max(tn + fp, 1)),
"f1": float(f1_score(labels, predictions, zero_division=0)),
"roc_auc": float(roc_auc_score(labels, probabilities))
if len(np.unique(labels)) == 2
else float("nan"),
"threshold": float(threshold),
"tn": int(tn),
"fp": int(fp),
"fn": int(fn),
"tp": int(tp),
"n_samples": int(len(labels)),
}
def save_evaluation_plots(
labels: np.ndarray,
probabilities: np.ndarray,
threshold: float,
output_dir: str | Path,
title: str,
) -> None:
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
predictions = (probabilities >= threshold).astype(int)
matrix = confusion_matrix(labels, predictions, labels=[0, 1])
plt.figure(figsize=(5, 4))
sns.heatmap(
matrix,
annot=True,
fmt="d",
cmap="Blues",
xticklabels=["Binh thuong", "Te nga"],
yticklabels=["Binh thuong", "Te nga"],
)
plt.xlabel("Du doan")
plt.ylabel("Thuc te")
plt.title(f"Ma tran nham lan - {title}")
plt.tight_layout()
plt.savefig(output_dir / "confusion_matrix.png", dpi=180)
plt.close()
if len(np.unique(labels)) == 2:
fpr, tpr, _ = roc_curve(labels, probabilities)
auc = roc_auc_score(labels, probabilities)
plt.figure(figsize=(5, 4))
plt.plot(fpr, tpr, label=f"AUC = {auc:.3f}")
plt.plot([0, 1], [0, 1], "--", color="gray")
plt.xlabel("False Positive Rate")
plt.ylabel("True Positive Rate")
plt.title(f"ROC - {title}")
plt.legend()
plt.tight_layout()
plt.savefig(output_dir / "roc_curve.png", dpi=180)
plt.close()
def save_predictions(
path: str | Path,
labels: np.ndarray,
probabilities: np.ndarray,
sources: np.ndarray,
threshold: float,
) -> None:
pd.DataFrame(
{
"source": sources,
"label": labels.astype(int),
"probability_fall": probabilities,
"prediction": (probabilities >= threshold).astype(int),
}
).to_csv(path, index=False)
|