| from __future__ import annotations |
|
|
| import importlib.util |
| import math |
| from pathlib import Path |
| import unittest |
|
|
|
|
| SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "analyze_study2.py" |
| SPEC = importlib.util.spec_from_file_location("study2_analysis", SCRIPT) |
| assert SPEC is not None and SPEC.loader is not None |
| analysis = importlib.util.module_from_spec(SPEC) |
| SPEC.loader.exec_module(analysis) |
|
|
|
|
| class Study2AnalysisTests(unittest.TestCase): |
| def test_exact_mcnemar_uses_only_discordant_pairs(self) -> None: |
| n10, n01, p_value = analysis.exact_mcnemar([1, 1, 1, 0], [0, 0, 1, 1]) |
| self.assertEqual((n10, n01), (2, 1)) |
| self.assertEqual(p_value, 1.0) |
|
|
| def test_holm_is_monotone_in_raw_p_order(self) -> None: |
| raw = [0.04, 0.001, 0.02, 0.5] |
| adjusted = analysis.holm_adjust(raw) |
| ordered = sorted(range(len(raw)), key=raw.__getitem__) |
| values = [adjusted[index] for index in ordered] |
| self.assertEqual(values, sorted(values)) |
| self.assertTrue(all(0 <= value <= 1 for value in values)) |
|
|
| def test_perfect_reliability_statistics(self) -> None: |
| ratings = [[0, 0, 0], [1, 1, 1], [0, 0, 0], [1, 1, 1]] |
| counts = [[3, 0], [0, 3], [3, 0], [0, 3]] |
| self.assertAlmostEqual(analysis.fleiss_kappa(counts), 1.0) |
| self.assertAlmostEqual(analysis.krippendorff_alpha_nominal(ratings), 1.0) |
| self.assertAlmostEqual(analysis.icc_one_way(ratings), 1.0) |
|
|
| def test_nonagreement_can_produce_negative_reliability(self) -> None: |
| ratings = [[0, 0, 1], [1, 1, 0], [0, 1, 0], [1, 0, 1]] |
| counts = [[2, 1], [1, 2], [2, 1], [1, 2]] |
| self.assertLess(analysis.fleiss_kappa(counts), 0.0) |
| self.assertLess(analysis.krippendorff_alpha_nominal(ratings), 0.0) |
|
|
| def test_chance_corrected_agreement_is_undefined_without_variance(self) -> None: |
| ratings = [[0, 0, 0], [0, 0, 0]] |
| counts = [[3, 0], [3, 0]] |
| self.assertTrue(math.isnan(analysis.fleiss_kappa(counts))) |
| self.assertTrue(math.isnan(analysis.krippendorff_alpha_nominal(ratings))) |
| self.assertTrue(math.isnan(analysis.icc_one_way(ratings))) |
|
|
| def test_secondary_family_is_model_stratified(self) -> None: |
| self.assertEqual(len(analysis.SECONDARY_CONTRASTS), 13) |
| self.assertEqual({spec[1] for spec in analysis.SECONDARY_CONTRASTS}, {"M002", "M003"}) |
|
|
|
|
| if __name__ == "__main__": |
| unittest.main() |
|
|
