| { |
| "n_images": 8125, |
| "accuracy_a": 0.9911384615384615, |
| "accuracy_b": 0.9907692307692307, |
| "mcnemar": { |
| "both_correct": 8004, |
| "only_a_correct": 49, |
| "only_b_correct": 46, |
| "both_wrong": 26, |
| "statistic": 0.042105263157894736, |
| "p_value": 0.8374192519835879, |
| "method": "chi-squared (continuity corrected)" |
| }, |
| "bootstrap": { |
| "observed_difference": -0.00036923076923078835, |
| "ci_low": -0.0027076923076923, |
| "ci_high": 0.001969230769230834, |
| "n_resamples": 10000, |
| "confidence": 0.95, |
| "p_value": 0.8012 |
| }, |
| "per_class": { |
| "mean_difference": 0.0037079681416117925, |
| "t_statistic": 1.23458202602619, |
| "p_value": 0.22476960764208556, |
| "df": 37, |
| "ci_low": -0.0023775384859782067, |
| "ci_high": 0.009793474769201792, |
| "cohens_d": 0.2002756507644646, |
| "effect_size_label": "small" |
| }, |
| "power": { |
| "effect_size": 0.2002756507644646, |
| "n": 38, |
| "alpha": 0.05, |
| "power": 0.22522047555120145, |
| "n_for_80_percent_power": 198 |
| }, |
| "notes": [ |
| "NOTE: the per-class test is underpowered \u2014 a null result here is inconclusive, not evidence of equivalence." |
| ], |
| "baseline_file": "artifacts/preds_baseline.npz", |
| "challenger_file": "artifacts/preds_challenger.npz" |
| } |
