src/wisp_release/scoring.py

"""Frozen paper score definitions, including full-vocabulary worst-class F1.

Copied from the paper artifact's wristharbench/scoring.py; no baseline code is
needed. Macro F1 averages classes observed in truth or prediction; worst-class
F1 considers the full dataset vocabulary, including absent classes.
"""

from __future__ import annotations

from typing import Any


def from_confusion(confusion: Any) -> dict[str, Any]:
    import numpy as np

    cm = np.asarray(confusion, dtype=float)
    if cm.ndim != 2 or cm.shape[0] != cm.shape[1] or not np.isfinite(cm).all() or (cm < 0).any() or cm.sum() <= 0:
        raise ValueError("Expected a nonempty square finite nonnegative confusion matrix")
    truth, predicted = cm.sum(1), cm.sum(0)
    tp = np.diag(cm)
    denominator = truth + predicted
    per_class = np.divide(2 * tp, denominator, out=np.zeros(len(tp)), where=denominator > 0)
    recall = np.divide(tp, truth, out=np.zeros(len(tp)), where=truth > 0)
    return {"macro_f1": float(per_class[denominator > 0].mean()),
            "accuracy": float(tp.sum() / cm.sum()),
            "balanced_accuracy": float(recall[truth > 0].mean()),
            "worst_class_f1": float(per_class.min())}


def score(y_true: Any, y_pred: Any, n_classes: int) -> dict[str, Any]:
    import numpy as np
    from sklearn.metrics import confusion_matrix

    truth, prediction = np.asarray(y_true), np.asarray(y_pred)
    if n_classes < 1 or truth.ndim != 1 or truth.shape != prediction.shape or not len(truth):
        raise ValueError("Expected aligned nonempty labels and a positive global class count")
    if not np.issubdtype(truth.dtype, np.integer) or not np.issubdtype(prediction.dtype, np.integer):
        raise ValueError("Labels must be encoded integers")
    if min(truth.min(), prediction.min()) < 0 or max(truth.max(), prediction.max()) >= n_classes:
        raise ValueError("Labels fall outside the global class vocabulary")
    cm = confusion_matrix(truth, prediction, labels=np.arange(n_classes))
    return {**from_confusion(cm), "confusion_matrix": cm.tolist()}