"""Aggregation statistics used for every table. Convention (stated in every caption): * per fold, average the metric over seeds; * report mean +- SD over the 5 fold means, SD with ddof=1; * row-vs-row comparisons on the common test set (recordings present in every compared row), with a two-sided paired t-test over the 5 fold means (n=5) and over fold x seed pairs (n=15), plus the number of folds with a positive difference. """ from __future__ import annotations import math from typing import Dict, Iterable, List, Mapping, Optional, Sequence, Tuple import numpy as np DDOF = 1 def mean_sd(values: Sequence[float], ddof: int = DDOF) -> Tuple[float, float]: a = np.asarray(list(values), dtype=np.float64) if a.size == 0: return float("nan"), float("nan") sd = float(a.std(ddof=ddof)) if a.size > ddof else float("nan") return float(a.mean()), sd def fold_means(values: Mapping[Tuple[int, int], float]) -> Dict[int, float]: """``{(fold, seed): v}`` -> ``{fold: mean over seeds}``.""" by: Dict[int, List[float]] = {} for (fold, _seed), v in values.items(): by.setdefault(fold, []).append(v) return {f: float(np.mean(v)) for f, v in sorted(by.items())} def paired_t(a: Sequence[float], b: Sequence[float]) -> Dict[str, float]: """Two-sided paired t-test of ``a - b``.""" x = np.asarray(a, dtype=np.float64) - np.asarray(b, dtype=np.float64) n = int(x.size) out = {"n": n, "mean_diff": float(x.mean()) if n else float("nan"), "t": float("nan"), "p": float("nan"), "df": n - 1} if n < 2: return out sd = x.std(ddof=1) if sd == 0: out["t"] = math.copysign(math.inf, out["mean_diff"]) if out["mean_diff"] else float("nan") out["p"] = 0.0 if out["mean_diff"] else 1.0 return out t = x.mean() / (sd / math.sqrt(n)) from scipy import stats as _st out["t"] = float(t) out["p"] = float(2 * _st.t.sf(abs(t), df=n - 1)) return out def positive_count(diffs: Iterable[float]) -> int: return int(sum(1 for d in diffs if d > 0)) def compare_rows(row: Mapping[Tuple[int, int], float], ref: Mapping[Tuple[int, int], float] ) -> Dict[str, object]: """Paired comparison of two rows given ``{(fold, seed): metric}`` on the same recordings.""" fr, ff = fold_means(row), fold_means(ref) folds = sorted(set(fr) & set(ff)) if set(fr) != set(ff): raise ValueError(f"rows cover different folds: {sorted(fr)} vs {sorted(ff)}") d5 = [fr[f] - ff[f] for f in folds] res: Dict[str, object] = { "folds": folds, "fold_diffs": d5, "delta_mean": float(np.mean(d5)), "delta_sd": mean_sd(d5)[1], "positive_folds": positive_count(d5), "t5": paired_t([fr[f] for f in folds], [ff[f] for f in folds]), } keys = sorted(set(row) & set(ref)) if set(row) == set(ref) and len(keys) > len(folds): res["t15"] = paired_t([row[k] for k in keys], [ref[k] for k in keys]) res["n_pairs"] = len(keys) else: res["t15"] = None res["n_pairs"] = len(keys) return res def format_pm(mean: float, sd: Optional[float], digits: int = 3) -> str: if mean != mean: # nan return "--" if sd is None or sd != sd: return f"{mean:.{digits}f}" return f"{mean:.{digits}f} $\\pm$ {sd:.{digits}f}" def format_p(p: Optional[float]) -> str: if p is None or p != p: return "--" if p < 1e-4: exp = int(math.floor(math.log10(p))) mant = p / 10 ** exp return f"${mant:.1f}\\times 10^{{{exp}}}$" return f"{p:.3f}" if p >= 0.001 else f"{p:.4f}"