PULSE-code / pulse /stats.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
3.65 kB
"""Aggregation statistics used for every table.
Convention (stated in every caption):
* per fold, average the metric over seeds;
* report mean +- SD over the 5 fold means, SD with ddof=1;
* row-vs-row comparisons on the common test set (recordings present in every
compared row), with a two-sided paired t-test over the 5 fold means
(n=5) and over fold x seed pairs (n=15), plus the number of folds with a
positive difference.
"""
from __future__ import annotations
import math
from typing import Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
import numpy as np
DDOF = 1
def mean_sd(values: Sequence[float], ddof: int = DDOF) -> Tuple[float, float]:
a = np.asarray(list(values), dtype=np.float64)
if a.size == 0:
return float("nan"), float("nan")
sd = float(a.std(ddof=ddof)) if a.size > ddof else float("nan")
return float(a.mean()), sd
def fold_means(values: Mapping[Tuple[int, int], float]) -> Dict[int, float]:
"""``{(fold, seed): v}`` -> ``{fold: mean over seeds}``."""
by: Dict[int, List[float]] = {}
for (fold, _seed), v in values.items():
by.setdefault(fold, []).append(v)
return {f: float(np.mean(v)) for f, v in sorted(by.items())}
def paired_t(a: Sequence[float], b: Sequence[float]) -> Dict[str, float]:
"""Two-sided paired t-test of ``a - b``."""
x = np.asarray(a, dtype=np.float64) - np.asarray(b, dtype=np.float64)
n = int(x.size)
out = {"n": n, "mean_diff": float(x.mean()) if n else float("nan"),
"t": float("nan"), "p": float("nan"), "df": n - 1}
if n < 2:
return out
sd = x.std(ddof=1)
if sd == 0:
out["t"] = math.copysign(math.inf, out["mean_diff"]) if out["mean_diff"] else float("nan")
out["p"] = 0.0 if out["mean_diff"] else 1.0
return out
t = x.mean() / (sd / math.sqrt(n))
from scipy import stats as _st
out["t"] = float(t)
out["p"] = float(2 * _st.t.sf(abs(t), df=n - 1))
return out
def positive_count(diffs: Iterable[float]) -> int:
return int(sum(1 for d in diffs if d > 0))
def compare_rows(row: Mapping[Tuple[int, int], float], ref: Mapping[Tuple[int, int], float]
) -> Dict[str, object]:
"""Paired comparison of two rows given ``{(fold, seed): metric}`` on the same recordings."""
fr, ff = fold_means(row), fold_means(ref)
folds = sorted(set(fr) & set(ff))
if set(fr) != set(ff):
raise ValueError(f"rows cover different folds: {sorted(fr)} vs {sorted(ff)}")
d5 = [fr[f] - ff[f] for f in folds]
res: Dict[str, object] = {
"folds": folds,
"fold_diffs": d5,
"delta_mean": float(np.mean(d5)),
"delta_sd": mean_sd(d5)[1],
"positive_folds": positive_count(d5),
"t5": paired_t([fr[f] for f in folds], [ff[f] for f in folds]),
}
keys = sorted(set(row) & set(ref))
if set(row) == set(ref) and len(keys) > len(folds):
res["t15"] = paired_t([row[k] for k in keys], [ref[k] for k in keys])
res["n_pairs"] = len(keys)
else:
res["t15"] = None
res["n_pairs"] = len(keys)
return res
def format_pm(mean: float, sd: Optional[float], digits: int = 3) -> str:
if mean != mean: # nan
return "--"
if sd is None or sd != sd:
return f"{mean:.{digits}f}"
return f"{mean:.{digits}f} $\\pm$ {sd:.{digits}f}"
def format_p(p: Optional[float]) -> str:
if p is None or p != p:
return "--"
if p < 1e-4:
exp = int(math.floor(math.log10(p)))
mant = p / 10 ** exp
return f"${mant:.1f}\\times 10^{{{exp}}}$"
return f"{p:.3f}" if p >= 0.001 else f"{p:.4f}"