File size: 6,553 Bytes
49ad2ef | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 | import unittest
from vons.mind2web import Mind2WebExample, evaluate_mind2web, recall_at_k, task_bootstrap_intervals
class Mind2WebEvaluationTests(unittest.TestCase):
def test_recall_and_selection_are_separate(self) -> None:
rows = [
Mind2WebExample("a", ("x", "target"), "target"),
Mind2WebExample("b", ("target", "y"), "target"),
]
metrics = evaluate_mind2web(
rows,
{"a": ("x", "target"), "b": ("y",)},
{"a": "x", "b": "target"},
)
self.assertEqual(metrics.candidate_recall, 0.5)
self.assertEqual(metrics.selection_accuracy_given_recall, 0.0)
self.assertEqual(metrics.complete_case_selection_accuracy_given_recall, 0.0)
self.assertEqual(metrics.evaluated_recalled_rows, 1)
self.assertNotIn("browser_task_success", metrics.to_mapping())
def test_mapping_keeps_positive_ids_out_of_retrieved_candidates(self) -> None:
row = Mind2WebExample.from_mapping(
{"id": "a", "candidate_ids": ["x"], "target_id": "target"}
)
self.assertEqual(row.candidate_ids, ("x",))
self.assertEqual(row.positive_ids, ("target",))
retrieved = Mind2WebExample.from_mapping(
{"id": "a", "candidate_ids": ["target"], "target_id": "target"}
)
self.assertEqual(retrieved.candidate_ids, ("target",))
def test_mapping_supports_multiple_positives_and_metadata(self) -> None:
row = Mind2WebExample.from_mapping(
{
"id": "a",
"candidate_ids": ["x", "y"],
"positive_ids": ["target", "target-alias"],
"task_id": "task-1",
"action_id": "action-2",
"split": "cross-website",
"website": "example.test",
"domain": "retail",
}
)
self.assertEqual(row.positive_ids, ("target", "target-alias"))
self.assertEqual(row.target_id, "target")
self.assertEqual(row.split, "cross-website")
def test_mapping_supports_explicit_no_positive_and_empty_candidates(self) -> None:
row = Mind2WebExample.from_mapping({"id": "none", "candidates": [], "no_positive": True})
self.assertTrue(row.no_positive)
self.assertEqual(row.positive_ids, ())
def test_mapping_rejects_string_false_and_null_candidate_ids(self) -> None:
with self.assertRaisesRegex(TypeError, "no_positive"):
Mind2WebExample.from_mapping({"id": "none", "candidates": [], "no_positive": "false"})
with self.assertRaisesRegex(TypeError, "ids"):
Mind2WebExample.from_mapping({"id": "bad", "candidates": [{"id": None}], "positive_ids": ["target"]})
def test_missing_prediction_is_excluded_from_conditional_accuracy(self) -> None:
rows = [
Mind2WebExample.from_mapping(
{"id": "a", "candidate_ids": ["x"], "positive_ids": ["target"]}
),
Mind2WebExample.from_mapping(
{"id": "b", "candidate_ids": ["x"], "positive_ids": ["target"]}
),
]
metrics = evaluate_mind2web(
rows,
{"a": ("target",), "b": ("target",)},
{"a": None, "b": "target"},
)
self.assertEqual(metrics.candidate_recall, 1.0)
self.assertEqual(metrics.selection_accuracy_given_recall, 0.5)
self.assertEqual(metrics.complete_case_selection_accuracy_given_recall, 1.0)
self.assertEqual(metrics.missing_predictions, 1)
self.assertEqual(metrics.evaluated_recalled_rows, 1)
def test_invalid_selection_is_evaluated_but_not_correct(self) -> None:
row = Mind2WebExample.from_mapping(
{"id": "a", "candidate_ids": ["x"], "positive_ids": ["target"]}
)
metrics = evaluate_mind2web({row}, {"a": ("target", "x")}, {"a": "not-retrieved"})
self.assertEqual(metrics.selection_accuracy_given_recall, 0.0)
self.assertEqual(metrics.invalid_selections, 1)
def test_undefined_metrics_are_null_and_no_positive_is_separate(self) -> None:
row = Mind2WebExample.from_mapping({"id": "none", "no_positive": True})
metrics = evaluate_mind2web([row], {"none": ()}, {"none": None})
self.assertIsNone(metrics.candidate_recall)
self.assertIsNone(metrics.selection_accuracy_given_recall)
self.assertEqual(metrics.no_positive_rows, 1)
self.assertIsNone(metrics.to_mapping()["candidate_recall"])
def test_recall_at_k_uses_retriever_order(self) -> None:
row = Mind2WebExample.from_mapping(
{"id": "a", "candidate_ids": [], "positive_ids": ["target"]}
)
self.assertEqual(recall_at_k([row], {"a": ("x", "target")}, 1), 0.0)
self.assertEqual(recall_at_k([row], {"a": ("x", "target")}, 2), 1.0)
def test_task_macro_and_step_micro_metrics_are_separate(self) -> None:
rows = [
Mind2WebExample("a1", ("target",), "target", task_id="task-a"),
Mind2WebExample("a2", ("x",), "target", task_id="task-a"),
Mind2WebExample("b1", ("target",), "target", task_id="task-b"),
]
metrics = evaluate_mind2web(
rows,
{"a1": ("target",), "a2": ("x",), "b1": ("target",)},
{"a1": "target", "b1": "target"},
)
self.assertEqual(metrics.candidate_recall, 2 / 3)
self.assertEqual(metrics.candidate_recall_task_macro, 0.75)
self.assertEqual(metrics.selection_accuracy_given_recall_task_macro, 1.0)
self.assertEqual(metrics.to_mapping()["candidate_recall_step_micro"], 2 / 3)
self.assertEqual(metrics.task_group_count, 2)
def test_task_bootstrap_is_deterministic_and_task_weighted(self) -> None:
rows = [
Mind2WebExample("a1", ("target",), "target", task_id="task-a"),
Mind2WebExample("a2", ("x",), "target", task_id="task-a"),
Mind2WebExample("b1", ("target",), "target", task_id="task-b"),
]
args = (
rows,
{"a1": ("target",), "a2": ("x",), "b1": ("target",)},
{"a1": "target", "b1": "target"},
)
first = task_bootstrap_intervals(*args, draws=20, seed=11)
second = task_bootstrap_intervals(*args, draws=20, seed=11)
self.assertEqual(first, second)
self.assertEqual(first["task_group_count"], 2)
self.assertEqual(len(first["candidate_recall_task_macro_ci95"]), 2)
if __name__ == "__main__":
unittest.main()
|