import unittest from evaluation.metrics import ( average_precision, binary_metrics, select_f1_threshold, select_threshold_at_max_fpr, ) from evaluation.protocols import ( SupportEntry, build_comparison_mask, support_hash, support_summary, ) class EvaluationContractTests(unittest.TestCase): def test_comparison_mask_excludes_t0_uncertain_and_invalid_slots(self): mask = build_comparison_mask( [1, 0, 2, 1], token_indices=[0, 1, 2, 3], score_valid_mask=[True, True, True, False], ) self.assertEqual(mask, [False, True, False, False]) def test_support_hash_is_order_independent_and_content_sensitive(self): entries = [ SupportEntry(1, "b", 2, 22, 1), SupportEntry(0, "a", 1, 11, 0), ] self.assertEqual(support_hash(entries), support_hash(reversed(entries))) self.assertNotEqual( support_hash(entries), support_hash([entries[0], SupportEntry(0, "a", 1, 99, 0)]), ) self.assertEqual(support_summary(entries)["n_hallucinated"], 1) def test_average_precision_and_auc_are_exact_for_perfect_ranking(self): result = binary_metrics([0.1, 0.2, 0.8, 0.9], [0, 0, 1, 1], threshold=0.8) self.assertEqual(result.average_precision, 1.0) self.assertEqual(result.auroc, 1.0) self.assertEqual(result.f1, 1.0) def test_f1_threshold_tie_breaks_toward_higher_threshold(self): threshold = select_f1_threshold([0.1, 0.5, 0.5, 0.9], [0, 0, 1, 1]) self.assertEqual(threshold, 0.5) def test_one_percent_fpr_threshold_never_uses_test_distribution(self): threshold = select_threshold_at_max_fpr( [0.1, 0.2, 0.3, 0.8, 0.9], [0, 0, 0, 1, 1], max_fpr=0.01, ) self.assertEqual(threshold, 0.8) def test_reportable_metrics_fail_on_degenerate_or_nonfinite_inputs(self): for scores, labels in [([], []), ([0.1], [0]), ([float("nan"), 0.2], [0, 1])]: with self.subTest(scores=scores, labels=labels): with self.assertRaises(ValueError): average_precision(scores, labels) if __name__ == "__main__": unittest.main()