File size: 2,323 Bytes
152be26 ef4748f 152be26 ef4748f 152be26 ef4748f 152be26 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 | from src.evaluation import EvaluationReport, run_evaluation_suite
EXPECTED_CASE_IDS = {
"straightforward_onboarding",
"transient_retry_recovery",
"exception_approval",
"exception_rejection",
"validation_failure",
"permanent_failure",
"checkpoint_resume",
"model_control_injection",
"invalid_dag",
"human_gate_bypass",
}
RATE_METRICS = {
"overall_case_pass_rate",
"workflow_completion_rate",
"expected_path_accuracy",
"retry_correctness",
"escalation_correctness",
"checkpoint_resume_success",
"audit_log_completeness",
"failure_containment_rate",
"model_output_containment_rate",
}
def test_full_evaluation_suite_passes_all_cases() -> None:
report = run_evaluation_suite()
assert isinstance(report, EvaluationReport)
assert report.total_cases == 10
assert report.passed_cases == 10
assert report.failed_cases == 0
assert report.case_pass_rate == 1.0
assert {case.case_id for case in report.cases} == EXPECTED_CASE_IDS
assert all(case.passed for case in report.cases)
def test_evaluation_metrics_meet_deterministic_mvp_targets() -> None:
report = run_evaluation_suite()
for metric_name in RATE_METRICS:
assert report.metrics[metric_name] == 1.0
assert report.metrics["duplicate_execution_count"] == 0
assert report.metrics["invalid_transition_count"] == 0
def test_evaluation_cases_expose_readable_checks_without_raw_payloads() -> None:
report = run_evaluation_suite()
for case in report.cases:
assert case.case_id
assert case.category
assert case.expected
assert case.observed
assert case.checks
serialized = report.model_dump_json()
assert "EVAL_MODEL_CONTROL_MARKER" not in serialized
assert "arbitrary_handler" not in serialized
def test_checkpoint_and_adversarial_cases_report_zero_duplicate_execution() -> None:
report = run_evaluation_suite()
cases = {case.case_id: case for case in report.cases}
assert cases["checkpoint_resume"].details["duplicate_executions"] == 0
assert cases["exception_approval"].details["duplicate_executions"] == 0
assert cases["model_control_injection"].checks["model_contained"] is True
assert cases["human_gate_bypass"].checks["failure_contained"] is True
|