File size: 3,462 Bytes
31c8075 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 | """Tests for the typed Validator plan module."""
import pytest
from arena.models import Criteria, ValidationCheck
from arena.validator_plan import (
CommandValidationStep,
RubricValidationStep,
StagehandValidationStep,
ValidatorContext,
ValidatorPlan,
validation_checks_from_user_tests,
)
def test_validator_plan_from_user_tests_separates_command_and_rubric_steps() -> None:
plan = ValidatorPlan.from_user_tests(["pytest -q", "ruff check"])
assert len(plan.command_steps) == 2
assert all(isinstance(step, CommandValidationStep) for step in plan.command_steps)
assert plan.stagehand_steps == ()
assert isinstance(plan.rubric_step, RubricValidationStep)
assert [step.check.command for step in plan.command_steps] == ["pytest -q", "ruff check"]
def test_validator_plan_from_checks_routes_stagehand_to_typed_step() -> None:
checks = [
ValidationCheck(id="cmd-1", name="tests", command="pytest -q", source="user"),
ValidationCheck(id="vis-1", name="hero loads", command="https://app.example", source="stagehand"),
]
plan = ValidatorPlan.from_checks(checks)
assert [step.check.id for step in plan.command_steps] == ["cmd-1"]
assert [step.check.id for step in plan.stagehand_steps] == ["vis-1"]
assert isinstance(plan.stagehand_steps[0], StagehandValidationStep)
def test_validator_plan_can_disable_rubric_step() -> None:
plan = ValidatorPlan.from_user_tests(["pytest -q"], rubric_enabled=False)
assert plan.rubric_step is None
def test_validator_plan_carries_prompt_and_criteria_context() -> None:
context = ValidatorContext(prompt="Build a CLI", criteria=(Criteria(text="pytest must pass"),))
plan = ValidatorPlan.from_user_tests(["pytest -q"], context=context)
assert plan.context.prompt == "Build a CLI"
assert len(plan.context.criteria) == 1
assert plan.context.criteria[0].text == "pytest must pass"
def test_validator_plan_adds_gradio_launch_check_for_demo_context() -> None:
context = ValidatorContext(
prompt="Build a Gradio demo app",
criteria=(Criteria(text="Works as a Gradio app"),),
)
plan = ValidatorPlan.from_user_tests(["python -m py_compile app.py"], context=context)
assert [step.check.id for step in plan.command_steps] == [
"user-test-1",
"generated-gradio-smoke",
]
assert "127.0.0.1:{port}" in plan.command_steps[1].check.command
def test_validator_plan_checks_property_returns_ordered_checks() -> None:
plan = ValidatorPlan.from_checks([
ValidationCheck(id="c-1", name="c1", command="cmd1"),
ValidationCheck(id="s-1", name="s1", command="https://x", source="stagehand"),
ValidationCheck(id="c-2", name="c2", command="cmd2"),
])
assert [check.id for check in plan.checks] == ["c-1", "c-2", "s-1"]
def test_validation_checks_from_user_tests_assigns_stable_ids() -> None:
checks = validation_checks_from_user_tests(["echo a", "echo b"])
assert [check.id for check in checks] == ["user-test-1", "user-test-2"]
assert all(check.source == "user" for check in checks)
def test_validator_plan_default_context_is_empty() -> None:
plan = ValidatorPlan.from_user_tests(["pytest -q"])
assert plan.context == ValidatorContext()
def test_validator_plan_is_immutable() -> None:
plan = ValidatorPlan.from_user_tests(["pytest -q"])
with pytest.raises(Exception):
plan.command_steps = ()
|