| from src.evaluation import EvaluationReport, run_evaluation_suite |
|
|
|
|
| EXPECTED_CASE_IDS = { |
| "straightforward_onboarding", |
| "transient_retry_recovery", |
| "exception_approval", |
| "exception_rejection", |
| "validation_failure", |
| "permanent_failure", |
| "checkpoint_resume", |
| "model_control_injection", |
| "invalid_dag", |
| "human_gate_bypass", |
| } |
|
|
| RATE_METRICS = { |
| "overall_case_pass_rate", |
| "workflow_completion_rate", |
| "expected_path_accuracy", |
| "retry_correctness", |
| "escalation_correctness", |
| "checkpoint_resume_success", |
| "audit_log_completeness", |
| "failure_containment_rate", |
| "model_output_containment_rate", |
| } |
|
|
|
|
| def test_full_evaluation_suite_passes_all_cases() -> None: |
| report = run_evaluation_suite() |
|
|
| assert isinstance(report, EvaluationReport) |
| assert report.total_cases == 10 |
| assert report.passed_cases == 10 |
| assert report.failed_cases == 0 |
| assert report.case_pass_rate == 1.0 |
| assert {case.case_id for case in report.cases} == EXPECTED_CASE_IDS |
| assert all(case.passed for case in report.cases) |
|
|
|
|
| def test_evaluation_metrics_meet_deterministic_mvp_targets() -> None: |
| report = run_evaluation_suite() |
|
|
| for metric_name in RATE_METRICS: |
| assert report.metrics[metric_name] == 1.0 |
|
|
| assert report.metrics["duplicate_execution_count"] == 0 |
| assert report.metrics["invalid_transition_count"] == 0 |
|
|
|
|
| def test_evaluation_cases_expose_readable_checks_without_raw_payloads() -> None: |
| report = run_evaluation_suite() |
|
|
| for case in report.cases: |
| assert case.case_id |
| assert case.category |
| assert case.expected |
| assert case.observed |
| assert case.checks |
|
|
| serialized = report.model_dump_json() |
| assert "EVAL_MODEL_CONTROL_MARKER" not in serialized |
| assert "arbitrary_handler" not in serialized |
|
|
|
|
| def test_checkpoint_and_adversarial_cases_report_zero_duplicate_execution() -> None: |
| report = run_evaluation_suite() |
| cases = {case.case_id: case for case in report.cases} |
|
|
| assert cases["checkpoint_resume"].details["duplicate_executions"] == 0 |
| assert cases["exception_approval"].details["duplicate_executions"] == 0 |
| assert cases["model_control_injection"].checks["model_contained"] is True |
| assert cases["human_gate_bypass"].checks["failure_contained"] is True |
|
|
