from __future__ import annotations

import importlib.util
from pathlib import Path
import unittest


ROOT = Path(__file__).resolve().parents[1]
SPEC = importlib.util.spec_from_file_location(
    "analyze_study3", ROOT / "scripts" / "analyze_study3.py"
)
assert SPEC is not None and SPEC.loader is not None
ANALYSIS = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(ANALYSIS)


class Study3AnalysisTests(unittest.TestCase):
    def test_exact_mcnemar_matches_known_two_sided_probability(self) -> None:
        left = [1] * 8 + [0] * 2
        right = [0] * 8 + [1] * 2
        n10, n01, probability = ANALYSIS.exact_mcnemar(left, right)
        self.assertEqual((n10, n01), (8, 2))
        self.assertAlmostEqual(probability, 0.109375)

    def test_exact_mcnemar_returns_one_without_discordance(self) -> None:
        self.assertEqual(
            ANALYSIS.exact_mcnemar([0, 1, 1], [0, 1, 1]), (0, 0, 1.0)
        )

    def test_holm_adjustment_is_monotone_in_rank(self) -> None:
        raw = [0.04, 0.01, 0.20, 0.03]
        adjusted = ANALYSIS.holm_adjust(raw)
        ordered = sorted(zip(raw, adjusted))
        self.assertEqual(adjusted, [0.09, 0.04, 0.20, 0.09])
        self.assertEqual(
            [value for _, value in ordered],
            sorted(value for _, value in ordered),
        )

    def test_compatibility_gate_uses_frozen_outcome_blind_order(self) -> None:
        summaries = []
        for model in ANALYSIS.MODELS:
            summaries.extend(
                [
                    {
                        "model_id": model,
                        "interface_id": "P001",
                        "accepted_edit_count": 11,
                        "attempt_acceptance_rate": 1.0,
                        "accepted_edit_rate": 0.90,
                        "applicable_rate": 1.0,
                        "mean_total_tokens": 1.0,
                    },
                    {
                        "model_id": model,
                        "interface_id": "P002",
                        "accepted_edit_count": 12,
                        "attempt_acceptance_rate": 0.50,
                        "accepted_edit_rate": 0.20,
                        "applicable_rate": 0.70,
                        "mean_total_tokens": 200.0,
                    },
                    {
                        "model_id": model,
                        "interface_id": "P003",
                        "accepted_edit_count": 12,
                        "attempt_acceptance_rate": 0.50,
                        "accepted_edit_rate": 0.20,
                        "applicable_rate": 0.70,
                        "mean_total_tokens": 100.0,
                    },
                ]
            )
        gate = ANALYSIS.compatibility_gate(summaries)
        for model in ANALYSIS.MODELS:
            self.assertEqual(gate["models"][model]["selected_interface"], "P003")
        self.assertFalse(gate["criteria"]["hidden_test_resolution_used"])

    def test_contrast_preserves_task_pairing(self) -> None:
        rows = []
        for index in range(60):
            task = f"T{index:03d}"
            rows.extend(
                [
                    {
                        "task_id": task,
                        "model_id": "M003",
                        "interface_id": "P002",
                        "accepted_edit_cell": int(index < 20),
                    },
                    {
                        "task_id": task,
                        "model_id": "M003",
                        "interface_id": "P001",
                        "accepted_edit_cell": int(index < 5),
                    },
                ]
            )
        result = ANALYSIS._paired(
            rows,
            "synthetic",
            "accepted_edit_cell",
            "M003",
            "P002",
            "P001",
            "test",
        )
        self.assertEqual(result["tasks"], 60)
        self.assertEqual(result["discordant_left_only"], 15)
        self.assertEqual(result["discordant_right_only"], 0)
        self.assertAlmostEqual(result["paired_risk_difference"], 0.25)


if __name__ == "__main__":
    unittest.main()