File size: 4,165 Bytes
d61821a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 | from __future__ import annotations
import importlib.util
from pathlib import Path
import unittest
ROOT = Path(__file__).resolve().parents[1]
SPEC = importlib.util.spec_from_file_location(
"analyze_study3", ROOT / "scripts" / "analyze_study3.py"
)
assert SPEC is not None and SPEC.loader is not None
ANALYSIS = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(ANALYSIS)
class Study3AnalysisTests(unittest.TestCase):
def test_exact_mcnemar_matches_known_two_sided_probability(self) -> None:
left = [1] * 8 + [0] * 2
right = [0] * 8 + [1] * 2
n10, n01, probability = ANALYSIS.exact_mcnemar(left, right)
self.assertEqual((n10, n01), (8, 2))
self.assertAlmostEqual(probability, 0.109375)
def test_exact_mcnemar_returns_one_without_discordance(self) -> None:
self.assertEqual(
ANALYSIS.exact_mcnemar([0, 1, 1], [0, 1, 1]), (0, 0, 1.0)
)
def test_holm_adjustment_is_monotone_in_rank(self) -> None:
raw = [0.04, 0.01, 0.20, 0.03]
adjusted = ANALYSIS.holm_adjust(raw)
ordered = sorted(zip(raw, adjusted))
self.assertEqual(adjusted, [0.09, 0.04, 0.20, 0.09])
self.assertEqual(
[value for _, value in ordered],
sorted(value for _, value in ordered),
)
def test_compatibility_gate_uses_frozen_outcome_blind_order(self) -> None:
summaries = []
for model in ANALYSIS.MODELS:
summaries.extend(
[
{
"model_id": model,
"interface_id": "P001",
"accepted_edit_count": 11,
"attempt_acceptance_rate": 1.0,
"accepted_edit_rate": 0.90,
"applicable_rate": 1.0,
"mean_total_tokens": 1.0,
},
{
"model_id": model,
"interface_id": "P002",
"accepted_edit_count": 12,
"attempt_acceptance_rate": 0.50,
"accepted_edit_rate": 0.20,
"applicable_rate": 0.70,
"mean_total_tokens": 200.0,
},
{
"model_id": model,
"interface_id": "P003",
"accepted_edit_count": 12,
"attempt_acceptance_rate": 0.50,
"accepted_edit_rate": 0.20,
"applicable_rate": 0.70,
"mean_total_tokens": 100.0,
},
]
)
gate = ANALYSIS.compatibility_gate(summaries)
for model in ANALYSIS.MODELS:
self.assertEqual(gate["models"][model]["selected_interface"], "P003")
self.assertFalse(gate["criteria"]["hidden_test_resolution_used"])
def test_contrast_preserves_task_pairing(self) -> None:
rows = []
for index in range(60):
task = f"T{index:03d}"
rows.extend(
[
{
"task_id": task,
"model_id": "M003",
"interface_id": "P002",
"accepted_edit_cell": int(index < 20),
},
{
"task_id": task,
"model_id": "M003",
"interface_id": "P001",
"accepted_edit_cell": int(index < 5),
},
]
)
result = ANALYSIS._paired(
rows,
"synthetic",
"accepted_edit_cell",
"M003",
"P002",
"P001",
"test",
)
self.assertEqual(result["tasks"], 60)
self.assertEqual(result["discordant_left_only"], 15)
self.assertEqual(result["discordant_right_only"], 0)
self.assertAlmostEqual(result["paired_risk_difference"], 0.25)
if __name__ == "__main__":
unittest.main()
|