Spaces:
Running
Running
File size: 4,224 Bytes
3b2bae1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 | import contextlib
import io
import unittest
from contextlib import ExitStack
from unittest.mock import patch
import numpy as np
import pandas as pd
import eval_exp
from chinatravel.evaluation import commonsense_constraint as evaluation
from chinatravel.symbol_verification import commonsense_constraint as commonsense
from chinatravel.symbol_verification.concept_func import activity_position
def _preference_plan():
activities = [
{"type": "attraction", "transports": []}
for _ in range(4)
]
activities.extend(
{"type": meal_type, "transports": []}
for meal_type in ("breakfast", "lunch", "dinner")
)
return {
"people_number": 1,
"start_city": "北京",
"target_city": "南京",
"itinerary": [{"day": 1, "activities": activities}],
}
class DefaultPreferenceScoringTests(unittest.TestCase):
def test_invalid_samples_contribute_zero(self):
plan = _preference_plan()
scores = eval_exp._default_preference_scores(
["valid", "invalid"],
{"valid": plan, "invalid": plan},
["valid"],
)
np.testing.assert_allclose(scores, np.array([0.5, 0.0, 0.5]))
class CommonsenseExceptionTests(unittest.TestCase):
def test_validator_exception_is_recorded_as_failure(self):
zero_result = pd.DataFrame({"Mock Validation": [0]})
validator_names = (
"Is_activity_grounded",
"Is_intercity_transport_correct",
"Is_attractions_correct",
"Is_hotels_correct",
"Is_restaurants_correct",
"Is_transport_correct",
"Is_time_correct",
"Is_space_correct",
)
with ExitStack() as stack:
stack.enter_context(patch.object(evaluation, "_set_tool_lang"))
for name in validator_names:
stack.enter_context(
patch.object(
evaluation,
name,
return_value=(zero_result, []),
)
)
stack.enter_context(
patch.object(
evaluation,
"Is_time_correct",
side_effect=RuntimeError("test failure"),
)
)
stderr = io.StringIO()
with contextlib.redirect_stderr(stderr):
_, _, result_agg, pass_ids = (
evaluation.evaluate_commonsense_constraints(
["query-id"],
{
"query-id": {
"start_city": "北京",
"target_city": "南京",
}
},
{"query-id": {"itinerary": []}},
lang="zh",
)
)
self.assertEqual(
result_agg.loc[0, "Commonsense Evaluator Exception"],
1,
)
self.assertEqual(pass_ids, [])
self.assertIn("query-id", stderr.getvalue())
class EntityGroundingTests(unittest.TestCase):
def test_middle_intercity_transport_is_rejected(self):
plan = {
"itinerary": [
{
"activities": [
{"type": "attraction", "position": "A"},
{
"type": "train",
"TrainID": "G1",
"start": "A Station",
"end": "B Station",
},
{"type": "attraction", "position": "B"},
]
}
]
}
table, errors = commonsense.Is_activity_grounded({}, plan)
self.assertEqual(table.iloc[0, 0], 1)
self.assertTrue(any("first or last" in error for error in errors))
def test_intercity_transport_has_no_activity_position(self):
self.assertEqual(
activity_position({"type": "train", "position": "Injected POI"}),
"",
)
if __name__ == "__main__":
unittest.main()
|