| from __future__ import annotations |
|
|
| import importlib.util |
| from pathlib import Path |
| import unittest |
|
|
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| SPEC = importlib.util.spec_from_file_location( |
| "analyze_study3", ROOT / "scripts" / "analyze_study3.py" |
| ) |
| assert SPEC is not None and SPEC.loader is not None |
| ANALYSIS = importlib.util.module_from_spec(SPEC) |
| SPEC.loader.exec_module(ANALYSIS) |
|
|
|
|
| class Study3AnalysisTests(unittest.TestCase): |
| def test_exact_mcnemar_matches_known_two_sided_probability(self) -> None: |
| left = [1] * 8 + [0] * 2 |
| right = [0] * 8 + [1] * 2 |
| n10, n01, probability = ANALYSIS.exact_mcnemar(left, right) |
| self.assertEqual((n10, n01), (8, 2)) |
| self.assertAlmostEqual(probability, 0.109375) |
|
|
| def test_exact_mcnemar_returns_one_without_discordance(self) -> None: |
| self.assertEqual( |
| ANALYSIS.exact_mcnemar([0, 1, 1], [0, 1, 1]), (0, 0, 1.0) |
| ) |
|
|
| def test_holm_adjustment_is_monotone_in_rank(self) -> None: |
| raw = [0.04, 0.01, 0.20, 0.03] |
| adjusted = ANALYSIS.holm_adjust(raw) |
| ordered = sorted(zip(raw, adjusted)) |
| self.assertEqual(adjusted, [0.09, 0.04, 0.20, 0.09]) |
| self.assertEqual( |
| [value for _, value in ordered], |
| sorted(value for _, value in ordered), |
| ) |
|
|
| def test_compatibility_gate_uses_frozen_outcome_blind_order(self) -> None: |
| summaries = [] |
| for model in ANALYSIS.MODELS: |
| summaries.extend( |
| [ |
| { |
| "model_id": model, |
| "interface_id": "P001", |
| "accepted_edit_count": 11, |
| "attempt_acceptance_rate": 1.0, |
| "accepted_edit_rate": 0.90, |
| "applicable_rate": 1.0, |
| "mean_total_tokens": 1.0, |
| }, |
| { |
| "model_id": model, |
| "interface_id": "P002", |
| "accepted_edit_count": 12, |
| "attempt_acceptance_rate": 0.50, |
| "accepted_edit_rate": 0.20, |
| "applicable_rate": 0.70, |
| "mean_total_tokens": 200.0, |
| }, |
| { |
| "model_id": model, |
| "interface_id": "P003", |
| "accepted_edit_count": 12, |
| "attempt_acceptance_rate": 0.50, |
| "accepted_edit_rate": 0.20, |
| "applicable_rate": 0.70, |
| "mean_total_tokens": 100.0, |
| }, |
| ] |
| ) |
| gate = ANALYSIS.compatibility_gate(summaries) |
| for model in ANALYSIS.MODELS: |
| self.assertEqual(gate["models"][model]["selected_interface"], "P003") |
| self.assertFalse(gate["criteria"]["hidden_test_resolution_used"]) |
|
|
| def test_contrast_preserves_task_pairing(self) -> None: |
| rows = [] |
| for index in range(60): |
| task = f"T{index:03d}" |
| rows.extend( |
| [ |
| { |
| "task_id": task, |
| "model_id": "M003", |
| "interface_id": "P002", |
| "accepted_edit_cell": int(index < 20), |
| }, |
| { |
| "task_id": task, |
| "model_id": "M003", |
| "interface_id": "P001", |
| "accepted_edit_cell": int(index < 5), |
| }, |
| ] |
| ) |
| result = ANALYSIS._paired( |
| rows, |
| "synthetic", |
| "accepted_edit_cell", |
| "M003", |
| "P002", |
| "P001", |
| "test", |
| ) |
| self.assertEqual(result["tasks"], 60) |
| self.assertEqual(result["discordant_left_only"], 15) |
| self.assertEqual(result["discordant_right_only"], 0) |
| self.assertAlmostEqual(result["paired_risk_difference"], 0.25) |
|
|
|
|
| if __name__ == "__main__": |
| unittest.main() |
|
|
