snl-intern / tests /test_refinement.py
xPushkal's picture
SHL Assessment Recommender — deploy snapshot
ff17b25
Raw History Blame Contribute Delete
4.09 kB
"""Refinement: add/remove/replace/keep-only/multiple actions + constraint application."""
from app.agent.turn import run_turn
def _java_base():
base = run_turn([{"role": "user", "content": "hiring a java developer"}])
types = [t.test_type for t in base.recommendations]
# 9 role-specific Java tests + the flagship personality slot (battery coverage).
assert len(base.recommendations) == 10 and types.count("K") >= 8
return base
def _hist(base, msg):
return [
{"role": "user", "content": "hiring a java developer"},
{"role": "assistant", "content": base.reply},
{"role": "user", "content": msg},
]
def test_add_personality_to_full_shortlist_evicts_and_retains():
base = _java_base()
r = run_turn(_hist(base, "actually, add a personality test"))
types = [t.test_type for t in r.recommendations]
assert len(r.recommendations) == 10 # stayed full (evicted a K)
assert "P" in types # the addition was retained
assert "Updated shortlist" in r.reply
def test_replace_removes_and_adds():
base = _java_base()
victim = base.recommendations[0].name
r = run_turn(_hist(base, f"replace {victim} with a personality test"))
names = [x.name for x in r.recommendations]
assert victim not in names
assert "P" in [t.test_type for t in r.recommendations]
def test_keep_only_restricts_shortlist():
base = _java_base()
a, b = base.recommendations[0].name, base.recommendations[1].name
r = run_turn(_hist(base, f"keep only {a} and {b}"))
names = [x.name for x in r.recommendations]
assert names == [a, b]
def test_multiple_actions_remove_and_add():
base = _java_base()
first = base.recommendations[0].name
r = run_turn(_hist(base, "remove the first one and add a personality test"))
names = [x.name for x in r.recommendations]
assert first not in names
assert "P" in [t.test_type for t in r.recommendations]
def test_add_multiple_products_retains_both():
# Java base = 10 items, none AWS/Docker -> forces genuine eviction on each add,
# so this catches a later add displacing an earlier one.
base = _java_base()
r = run_turn(_hist(base, "add AWS and Docker"))
names = [x.name for x in r.recommendations]
assert any("AWS" in n for n in names), names
assert any("Docker" in n for n in names), names
assert len(names) == 10
def test_multiple_actions_add_two_and_remove_one():
base = run_turn([{"role": "user", "content": "hiring a cloud engineer"}])
first = base.recommendations[0].name
r = run_turn([
{"role": "user", "content": "hiring a cloud engineer"},
{"role": "assistant", "content": base.reply},
{"role": "user", "content": "add AWS and Docker; drop the first one"},
])
names = [x.name for x in r.recommendations]
assert any("AWS" in n for n in names) and any("Docker" in n for n in names)
assert first not in names
def test_new_duration_constraint_applies_to_shortlist():
from app.data.catalog import load_catalog
cat = load_catalog()
base = _java_base()
r = run_turn(_hist(base, "make them under 15 minutes"))
assert r.recommendations, "should retain the short-enough items"
for rec in r.recommendations:
mins = cat.get_by_url(rec.url)["duration_minutes"]
assert mins is not None and mins <= 15
def test_final_list_finalizes_conversation():
base = _java_base()
a, b = base.recommendations[0].name, base.recommendations[1].name
r = run_turn(_hist(base, f"Final list: {a} and {b}."))
assert r.end_of_conversation is True
assert [x.name for x in r.recommendations] == [a, b]
def test_keep_as_is_finalizes_conversation():
base = _java_base()
r = run_turn(_hist(base, "Understood. Keep the shortlist as-is."))
assert r.end_of_conversation is True and len(r.recommendations) == 10
def test_mid_conversation_refine_does_not_finalize():
base = _java_base()
r = run_turn(_hist(base, "add a personality test"))
assert r.end_of_conversation is False