| """Statistical confidence intervals and significance testing.""" |
|
|
| from typing import Any, Dict, List, Tuple |
|
|
| import numpy as np |
| from scipy import stats |
|
|
|
|
| def bootstrap_confidence_interval( |
| results: List[bool], |
| confidence_level: float = 0.95, |
| n_bootstrap: int = 1000, |
| seed: int = 42, |
| ) -> Tuple[float, float, float]: |
| if not results: |
| return 0.0, 0.0, 0.0 |
|
|
| arr = np.array(results, dtype=np.float64) |
| original_mean = float(np.mean(arr)) |
|
|
| rng = np.random.default_rng(seed) |
| bootstrap_means = [] |
| n = len(results) |
|
|
| for _ in range(n_bootstrap): |
| sample = rng.choice(arr, size=n, replace=True) |
| bootstrap_means.append(np.mean(sample)) |
|
|
| alpha = (1.0 - confidence_level) / 2.0 |
| low = float(np.percentile(bootstrap_means, alpha * 100)) |
| high = float(np.percentile(bootstrap_means, (1.0 - alpha) * 100)) |
|
|
| return original_mean, low, high |
|
|
|
|
| def mcnemar_test(results_a: List[bool], results_b: List[bool]) -> Dict[str, Any]: |
| if len(results_a) != len(results_b): |
| raise ValueError( |
| f"Results lists must have same length: {len(results_a)} vs {len(results_b)}" |
| ) |
|
|
| b = 0 |
| c = 0 |
| both_correct = 0 |
| both_wrong = 0 |
|
|
| for a, b_val in zip(results_a, results_b): |
| if a and not b_val: |
| b += 1 |
| elif not a and b_val: |
| c += 1 |
| elif a and b_val: |
| both_correct += 1 |
| else: |
| both_wrong += 1 |
|
|
| total_discordant = b + c |
| if total_discordant == 0: |
| return { |
| "statistic": 0.0, |
| "p_value": 1.0, |
| "significant": False, |
| "b_a_correct_b_wrong": b, |
| "c_a_wrong_b_correct": c, |
| } |
|
|
| if total_discordant < 25: |
| p_val = float(2 * stats.binom.cdf(min(b, c), total_discordant, 0.5)) |
| p_val = min(1.0, p_val) |
| stat = float(abs(b - c)) |
| else: |
| stat = float((abs(b - c) - 1.0) ** 2 / total_discordant) |
| p_val = float(1.0 - stats.chi2.cdf(stat, df=1)) |
|
|
| return { |
| "statistic": stat, |
| "p_value": p_val, |
| "significant": p_val < 0.05, |
| "b_a_correct_b_wrong": b, |
| "c_a_wrong_b_correct": c, |
| "both_correct": both_correct, |
| "both_wrong": both_wrong, |
| } |
|
|
