Lattice_Spike / tests /test_evaluation_contract.py
Md Saiful Islam Sajol
Upload Lattice_Spike project with full LFS weights
e2c9d42 verified
Raw History Blame Contribute Delete
2.27 kB
import unittest
from evaluation.metrics import (
average_precision,
binary_metrics,
select_f1_threshold,
select_threshold_at_max_fpr,
)
from evaluation.protocols import (
SupportEntry,
build_comparison_mask,
support_hash,
support_summary,
)
class EvaluationContractTests(unittest.TestCase):
def test_comparison_mask_excludes_t0_uncertain_and_invalid_slots(self):
mask = build_comparison_mask(
[1, 0, 2, 1],
token_indices=[0, 1, 2, 3],
score_valid_mask=[True, True, True, False],
)
self.assertEqual(mask, [False, True, False, False])
def test_support_hash_is_order_independent_and_content_sensitive(self):
entries = [
SupportEntry(1, "b", 2, 22, 1),
SupportEntry(0, "a", 1, 11, 0),
]
self.assertEqual(support_hash(entries), support_hash(reversed(entries)))
self.assertNotEqual(
support_hash(entries),
support_hash([entries[0], SupportEntry(0, "a", 1, 99, 0)]),
)
self.assertEqual(support_summary(entries)["n_hallucinated"], 1)
def test_average_precision_and_auc_are_exact_for_perfect_ranking(self):
result = binary_metrics([0.1, 0.2, 0.8, 0.9], [0, 0, 1, 1], threshold=0.8)
self.assertEqual(result.average_precision, 1.0)
self.assertEqual(result.auroc, 1.0)
self.assertEqual(result.f1, 1.0)
def test_f1_threshold_tie_breaks_toward_higher_threshold(self):
threshold = select_f1_threshold([0.1, 0.5, 0.5, 0.9], [0, 0, 1, 1])
self.assertEqual(threshold, 0.5)
def test_one_percent_fpr_threshold_never_uses_test_distribution(self):
threshold = select_threshold_at_max_fpr(
[0.1, 0.2, 0.3, 0.8, 0.9],
[0, 0, 0, 1, 1],
max_fpr=0.01,
)
self.assertEqual(threshold, 0.8)
def test_reportable_metrics_fail_on_degenerate_or_nonfinite_inputs(self):
for scores, labels in [([], []), ([0.1], [0]), ([float("nan"), 0.2], [0, 1])]:
with self.subTest(scores=scores, labels=labels):
with self.assertRaises(ValueError):
average_precision(scores, labels)
if __name__ == "__main__":
unittest.main()