tests/test_validation.py

from inferscale.validation import validate_cases


def test_validation_report_compares_external_metrics():
    base = {
        "model": "Qwen2.5-3B", "accelerator": "L4", "quantization": "int8",
        "duration_s": 4, "request_rate_rps": 1, "prompt_tokens_mean": 128, "output_tokens_mean": 8,
    }
    # Values are deliberately arbitrary: this tests the validation machinery,
    # not empirical accuracy of the analytical reference profile.
    report = validate_cases([{
        "name": "fixture",
        "config": base,
        "measured": {"p95_ttft_ms": 100.0, "goodput_rps": 0.8},
    }])
    assert report["case_count"] == 1
    assert report["observation_count"] == 2
    assert report["mape_pct"] >= 0
    assert {row["metric"] for row in report["rows"]} == {"p95_ttft_ms", "goodput_rps"}