File size: 3,631 Bytes
88b1f73 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 | """Fixed-prediction evaluation entry points for the published Test_TSB script."""
import numpy as np
import torch
from sklearn.metrics import precision_recall_fscore_support
from .basic_metrics import basic_metricor, generate_curve, improved_cardinality_fn
def get_metrics(score, labels, slidingWindow=100, pred=None, version="opt", thre=250):
"""Evaluate scores without selecting thresholds from ground-truth labels.
Respect supplied predictions. Otherwise use mean(score) + 3 * std(score),
matching Test_TSB.py. Remove labels == -1 before calibration and evaluation.
Undefined ranking/event metrics are NaN, rather than a fabricated zero.
"""
score = np.asarray(score, dtype=float)
labels = np.asarray(labels)
if score.ndim != 1 or labels.shape != score.shape:
raise ValueError("Scores and labels must be one-dimensional with equal lengths.")
if pred is not None:
pred = np.asarray(pred)
if pred.shape != score.shape:
raise ValueError("Predictions must have the same shape as scores.")
valid = labels != -1
score, labels = score[valid], labels[valid]
if not len(score) or not np.isfinite(score).all():
raise ValueError("At least one finite, unpadded score is required.")
if not np.isin(labels, [0, 1]).all():
raise ValueError("Labels must be 0, 1, or -1 for padding.")
labels = labels.astype(int)
if pred is None:
pred = score > (score.mean() + 3 * score.std())
else:
pred = pred[valid]
if not np.isin(pred, [0, 1]).all():
raise ValueError("Predictions must be binary.")
pred = pred.astype(int, copy=True)
if int(slidingWindow) != slidingWindow or slidingWindow < 1:
raise ValueError("slidingWindow must be a positive integer.")
grader = basic_metricor()
precision, recall, f1, _ = precision_recall_fscore_support(
labels, pred, average="binary", zero_division=0
)
adjusted = grader.adjustment(labels, pred.copy())
pa_p, pa_r, pa_f, _ = precision_recall_fscore_support(
labels, adjusted, average="binary", zero_division=0
)
t_p, t_r = grader.ts_precision_and_recall(
torch.as_tensor(labels), torch.as_tensor(pred),
recall_cardinality_fn=improved_cardinality_fn,
weighted_precision=True,
)
t_f = 2 * t_p * t_r / (t_p + t_r) if t_p + t_r else 0.0
auc_roc = auc_pr = vus_roc = vus_pr = float("nan")
if len(np.unique(labels)) == 2:
auc_roc = grader.metric_ROC(labels, score)
auc_pr = grader.metric_PR(labels, score)
*_, vus_roc, vus_pr = generate_curve(
labels, score, int(slidingWindow), version=version, thre=thre
)
aff_f = aff_p = aff_r = float("nan")
if labels.any():
if pred.any():
aff_f, aff_p, aff_r = grader.metric_Affiliation(labels, score, preds=pred)
else:
aff_f, aff_p, aff_r = 0.0, float("nan"), 0.0
return {key: float(value) for key, value in {
"AUC-PR": auc_pr, "AUC-ROC": auc_roc,
"VUS-PR": vus_pr, "VUS-ROC": vus_roc,
"Standard-F1": f1, "Standard-Precision": precision, "Standard-Recall": recall,
"PA-F1": pa_f, "PA-Precision": pa_p, "PA-Recall": pa_r,
"Affiliation-F": aff_f, "Affiliation-P": aff_p, "Affiliation-R": aff_r,
"F1_T": t_f, "Precision_T": t_p, "Recall_T": t_r,
}.items()}
def get_metrics_pred(score, labels, pred, slidingWindow=100):
"""Evaluate a fixed prediction vector through the same public protocol."""
return get_metrics(score, labels, slidingWindow=slidingWindow, pred=pred)
|