File size: 2,422 Bytes
d61821a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | from __future__ import annotations
import importlib.util
import math
from pathlib import Path
import unittest
SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "analyze_study2.py"
SPEC = importlib.util.spec_from_file_location("study2_analysis", SCRIPT)
assert SPEC is not None and SPEC.loader is not None
analysis = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(analysis)
class Study2AnalysisTests(unittest.TestCase):
def test_exact_mcnemar_uses_only_discordant_pairs(self) -> None:
n10, n01, p_value = analysis.exact_mcnemar([1, 1, 1, 0], [0, 0, 1, 1])
self.assertEqual((n10, n01), (2, 1))
self.assertEqual(p_value, 1.0)
def test_holm_is_monotone_in_raw_p_order(self) -> None:
raw = [0.04, 0.001, 0.02, 0.5]
adjusted = analysis.holm_adjust(raw)
ordered = sorted(range(len(raw)), key=raw.__getitem__)
values = [adjusted[index] for index in ordered]
self.assertEqual(values, sorted(values))
self.assertTrue(all(0 <= value <= 1 for value in values))
def test_perfect_reliability_statistics(self) -> None:
ratings = [[0, 0, 0], [1, 1, 1], [0, 0, 0], [1, 1, 1]]
counts = [[3, 0], [0, 3], [3, 0], [0, 3]]
self.assertAlmostEqual(analysis.fleiss_kappa(counts), 1.0)
self.assertAlmostEqual(analysis.krippendorff_alpha_nominal(ratings), 1.0)
self.assertAlmostEqual(analysis.icc_one_way(ratings), 1.0)
def test_nonagreement_can_produce_negative_reliability(self) -> None:
ratings = [[0, 0, 1], [1, 1, 0], [0, 1, 0], [1, 0, 1]]
counts = [[2, 1], [1, 2], [2, 1], [1, 2]]
self.assertLess(analysis.fleiss_kappa(counts), 0.0)
self.assertLess(analysis.krippendorff_alpha_nominal(ratings), 0.0)
def test_chance_corrected_agreement_is_undefined_without_variance(self) -> None:
ratings = [[0, 0, 0], [0, 0, 0]]
counts = [[3, 0], [3, 0]]
self.assertTrue(math.isnan(analysis.fleiss_kappa(counts)))
self.assertTrue(math.isnan(analysis.krippendorff_alpha_nominal(ratings)))
self.assertTrue(math.isnan(analysis.icc_one_way(ratings)))
def test_secondary_family_is_model_stratified(self) -> None:
self.assertEqual(len(analysis.SECONDARY_CONTRASTS), 13)
self.assertEqual({spec[1] for spec in analysis.SECONDARY_CONTRASTS}, {"M002", "M003"})
if __name__ == "__main__":
unittest.main()
|