Download pulse/stats.py from velvet-pine-22/PULSE-code: direct link, hf CLI and curl.
- Browser
- Download file 3.65 kB
-
https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/pulse/stats.py
- Command line
-
hf download hf://velvet-pine-22/PULSE-code/pulse/stats.py
-
curl -L -o stats.py https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/pulse/stats.py
3.65 kB
| """Aggregation statistics used for every table. | |
| Convention (stated in every caption): | |
| * per fold, average the metric over seeds; | |
| * report mean +- SD over the 5 fold means, SD with ddof=1; | |
| * row-vs-row comparisons on the common test set (recordings present in every | |
| compared row), with a two-sided paired t-test over the 5 fold means | |
| (n=5) and over fold x seed pairs (n=15), plus the number of folds with a | |
| positive difference. | |
| """ | |
| from __future__ import annotations | |
| import math | |
| from typing import Dict, Iterable, List, Mapping, Optional, Sequence, Tuple | |
| import numpy as np | |
| DDOF = 1 | |
| def mean_sd(values: Sequence[float], ddof: int = DDOF) -> Tuple[float, float]: | |
| a = np.asarray(list(values), dtype=np.float64) | |
| if a.size == 0: | |
| return float("nan"), float("nan") | |
| sd = float(a.std(ddof=ddof)) if a.size > ddof else float("nan") | |
| return float(a.mean()), sd | |
| def fold_means(values: Mapping[Tuple[int, int], float]) -> Dict[int, float]: | |
| """``{(fold, seed): v}`` -> ``{fold: mean over seeds}``.""" | |
| by: Dict[int, List[float]] = {} | |
| for (fold, _seed), v in values.items(): | |
| by.setdefault(fold, []).append(v) | |
| return {f: float(np.mean(v)) for f, v in sorted(by.items())} | |
| def paired_t(a: Sequence[float], b: Sequence[float]) -> Dict[str, float]: | |
| """Two-sided paired t-test of ``a - b``.""" | |
| x = np.asarray(a, dtype=np.float64) - np.asarray(b, dtype=np.float64) | |
| n = int(x.size) | |
| out = {"n": n, "mean_diff": float(x.mean()) if n else float("nan"), | |
| "t": float("nan"), "p": float("nan"), "df": n - 1} | |
| if n < 2: | |
| return out | |
| sd = x.std(ddof=1) | |
| if sd == 0: | |
| out["t"] = math.copysign(math.inf, out["mean_diff"]) if out["mean_diff"] else float("nan") | |
| out["p"] = 0.0 if out["mean_diff"] else 1.0 | |
| return out | |
| t = x.mean() / (sd / math.sqrt(n)) | |
| from scipy import stats as _st | |
| out["t"] = float(t) | |
| out["p"] = float(2 * _st.t.sf(abs(t), df=n - 1)) | |
| return out | |
| def positive_count(diffs: Iterable[float]) -> int: | |
| return int(sum(1 for d in diffs if d > 0)) | |
| def compare_rows(row: Mapping[Tuple[int, int], float], ref: Mapping[Tuple[int, int], float] | |
| ) -> Dict[str, object]: | |
| """Paired comparison of two rows given ``{(fold, seed): metric}`` on the same recordings.""" | |
| fr, ff = fold_means(row), fold_means(ref) | |
| folds = sorted(set(fr) & set(ff)) | |
| if set(fr) != set(ff): | |
| raise ValueError(f"rows cover different folds: {sorted(fr)} vs {sorted(ff)}") | |
| d5 = [fr[f] - ff[f] for f in folds] | |
| res: Dict[str, object] = { | |
| "folds": folds, | |
| "fold_diffs": d5, | |
| "delta_mean": float(np.mean(d5)), | |
| "delta_sd": mean_sd(d5)[1], | |
| "positive_folds": positive_count(d5), | |
| "t5": paired_t([fr[f] for f in folds], [ff[f] for f in folds]), | |
| } | |
| keys = sorted(set(row) & set(ref)) | |
| if set(row) == set(ref) and len(keys) > len(folds): | |
| res["t15"] = paired_t([row[k] for k in keys], [ref[k] for k in keys]) | |
| res["n_pairs"] = len(keys) | |
| else: | |
| res["t15"] = None | |
| res["n_pairs"] = len(keys) | |
| return res | |
| def format_pm(mean: float, sd: Optional[float], digits: int = 3) -> str: | |
| if mean != mean: # nan | |
| return "--" | |
| if sd is None or sd != sd: | |
| return f"{mean:.{digits}f}" | |
| return f"{mean:.{digits}f} $\\pm$ {sd:.{digits}f}" | |
| def format_p(p: Optional[float]) -> str: | |
| if p is None or p != p: | |
| return "--" | |
| if p < 1e-4: | |
| exp = int(math.floor(math.log10(p))) | |
| mant = p / 10 ** exp | |
| return f"${mant:.1f}\\times 10^{{{exp}}}$" | |
| return f"{p:.3f}" if p >= 0.001 else f"{p:.4f}" | |