File size: 3,652 Bytes
dffbc1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
"""Aggregation statistics used for every table.

Convention (stated in every caption):

* per fold, average the metric over seeds;
* report mean +- SD over the 5 fold means, SD with ddof=1;
* row-vs-row comparisons on the common test set (recordings present in every
  compared row), with a two-sided paired t-test over the 5 fold means
  (n=5) and over fold x seed pairs (n=15), plus the number of folds with a
  positive difference.
"""
from __future__ import annotations

import math
from typing import Dict, Iterable, List, Mapping, Optional, Sequence, Tuple

import numpy as np

DDOF = 1


def mean_sd(values: Sequence[float], ddof: int = DDOF) -> Tuple[float, float]:
    a = np.asarray(list(values), dtype=np.float64)
    if a.size == 0:
        return float("nan"), float("nan")
    sd = float(a.std(ddof=ddof)) if a.size > ddof else float("nan")
    return float(a.mean()), sd


def fold_means(values: Mapping[Tuple[int, int], float]) -> Dict[int, float]:
    """``{(fold, seed): v}`` -> ``{fold: mean over seeds}``."""
    by: Dict[int, List[float]] = {}
    for (fold, _seed), v in values.items():
        by.setdefault(fold, []).append(v)
    return {f: float(np.mean(v)) for f, v in sorted(by.items())}


def paired_t(a: Sequence[float], b: Sequence[float]) -> Dict[str, float]:
    """Two-sided paired t-test of ``a - b``."""
    x = np.asarray(a, dtype=np.float64) - np.asarray(b, dtype=np.float64)
    n = int(x.size)
    out = {"n": n, "mean_diff": float(x.mean()) if n else float("nan"),
           "t": float("nan"), "p": float("nan"), "df": n - 1}
    if n < 2:
        return out
    sd = x.std(ddof=1)
    if sd == 0:
        out["t"] = math.copysign(math.inf, out["mean_diff"]) if out["mean_diff"] else float("nan")
        out["p"] = 0.0 if out["mean_diff"] else 1.0
        return out
    t = x.mean() / (sd / math.sqrt(n))
    from scipy import stats as _st
    out["t"] = float(t)
    out["p"] = float(2 * _st.t.sf(abs(t), df=n - 1))
    return out


def positive_count(diffs: Iterable[float]) -> int:
    return int(sum(1 for d in diffs if d > 0))


def compare_rows(row: Mapping[Tuple[int, int], float], ref: Mapping[Tuple[int, int], float]
                 ) -> Dict[str, object]:
    """Paired comparison of two rows given ``{(fold, seed): metric}`` on the same recordings."""
    fr, ff = fold_means(row), fold_means(ref)
    folds = sorted(set(fr) & set(ff))
    if set(fr) != set(ff):
        raise ValueError(f"rows cover different folds: {sorted(fr)} vs {sorted(ff)}")
    d5 = [fr[f] - ff[f] for f in folds]
    res: Dict[str, object] = {
        "folds": folds,
        "fold_diffs": d5,
        "delta_mean": float(np.mean(d5)),
        "delta_sd": mean_sd(d5)[1],
        "positive_folds": positive_count(d5),
        "t5": paired_t([fr[f] for f in folds], [ff[f] for f in folds]),
    }
    keys = sorted(set(row) & set(ref))
    if set(row) == set(ref) and len(keys) > len(folds):
        res["t15"] = paired_t([row[k] for k in keys], [ref[k] for k in keys])
        res["n_pairs"] = len(keys)
    else:
        res["t15"] = None
        res["n_pairs"] = len(keys)
    return res


def format_pm(mean: float, sd: Optional[float], digits: int = 3) -> str:
    if mean != mean:  # nan
        return "--"
    if sd is None or sd != sd:
        return f"{mean:.{digits}f}"
    return f"{mean:.{digits}f} $\\pm$ {sd:.{digits}f}"


def format_p(p: Optional[float]) -> str:
    if p is None or p != p:
        return "--"
    if p < 1e-4:
        exp = int(math.floor(math.log10(p)))
        mant = p / 10 ** exp
        return f"${mant:.1f}\\times 10^{{{exp}}}$"
    return f"{p:.3f}" if p >= 0.001 else f"{p:.4f}"