Download tests/test_evaluation_contract.py from Anonymous1611/Lattice_Spike: direct link, hf CLI and curl.
- Browser
- Download file 2.27 kB
-
https://huggingface.co/Anonymous1611/Lattice_Spike/resolve/main/tests/test_evaluation_contract.py
- Command line
-
hf download hf://Anonymous1611/Lattice_Spike/tests/test_evaluation_contract.py
-
curl -L -o test_evaluation_contract.py https://huggingface.co/Anonymous1611/Lattice_Spike/resolve/main/tests/test_evaluation_contract.py
2.27 kB
| import unittest | |
| from evaluation.metrics import ( | |
| average_precision, | |
| binary_metrics, | |
| select_f1_threshold, | |
| select_threshold_at_max_fpr, | |
| ) | |
| from evaluation.protocols import ( | |
| SupportEntry, | |
| build_comparison_mask, | |
| support_hash, | |
| support_summary, | |
| ) | |
| class EvaluationContractTests(unittest.TestCase): | |
| def test_comparison_mask_excludes_t0_uncertain_and_invalid_slots(self): | |
| mask = build_comparison_mask( | |
| [1, 0, 2, 1], | |
| token_indices=[0, 1, 2, 3], | |
| score_valid_mask=[True, True, True, False], | |
| ) | |
| self.assertEqual(mask, [False, True, False, False]) | |
| def test_support_hash_is_order_independent_and_content_sensitive(self): | |
| entries = [ | |
| SupportEntry(1, "b", 2, 22, 1), | |
| SupportEntry(0, "a", 1, 11, 0), | |
| ] | |
| self.assertEqual(support_hash(entries), support_hash(reversed(entries))) | |
| self.assertNotEqual( | |
| support_hash(entries), | |
| support_hash([entries[0], SupportEntry(0, "a", 1, 99, 0)]), | |
| ) | |
| self.assertEqual(support_summary(entries)["n_hallucinated"], 1) | |
| def test_average_precision_and_auc_are_exact_for_perfect_ranking(self): | |
| result = binary_metrics([0.1, 0.2, 0.8, 0.9], [0, 0, 1, 1], threshold=0.8) | |
| self.assertEqual(result.average_precision, 1.0) | |
| self.assertEqual(result.auroc, 1.0) | |
| self.assertEqual(result.f1, 1.0) | |
| def test_f1_threshold_tie_breaks_toward_higher_threshold(self): | |
| threshold = select_f1_threshold([0.1, 0.5, 0.5, 0.9], [0, 0, 1, 1]) | |
| self.assertEqual(threshold, 0.5) | |
| def test_one_percent_fpr_threshold_never_uses_test_distribution(self): | |
| threshold = select_threshold_at_max_fpr( | |
| [0.1, 0.2, 0.3, 0.8, 0.9], | |
| [0, 0, 0, 1, 1], | |
| max_fpr=0.01, | |
| ) | |
| self.assertEqual(threshold, 0.8) | |
| def test_reportable_metrics_fail_on_degenerate_or_nonfinite_inputs(self): | |
| for scores, labels in [([], []), ([0.1], [0]), ([float("nan"), 0.2], [0, 1])]: | |
| with self.subTest(scores=scores, labels=labels): | |
| with self.assertRaises(ValueError): | |
| average_precision(scores, labels) | |
| if __name__ == "__main__": | |
| unittest.main() | |