tests/test_evaluator.py

from __future__ import annotations

import asyncio
import base64
from pathlib import Path

import httpx

from autocad_bench.evaluation.scoring import HttpDWGEvaluator
from autocad_bench.evaluation.scoring.models import LayoutViewportFeature, PaperLayoutSummary
from tests.scoring_helpers import valid_drawing


def test_http_evaluator_contract_and_authentication() -> None:
    source = b"AC1032-candidate"
    drawing = valid_drawing(
        source,
        evaluator_version="autocad-2026.1-v1",
        drawing_type="3d",
    )
    requests: list[httpx.Request] = []

    def handler(request: httpx.Request) -> httpx.Response:
        requests.append(request)
        assert request.headers["authorization"] == "Bearer secret"
        if request.url.path == "/health":
            return httpx.Response(
                200,
                json={"status": "ok", "evaluator_version": "autocad-2026.1-v1"},
            )
        assert request.url.path == "/v1/inspect-render"
        assert "multipart/form-data" in request.headers["content-type"]
        assert source in request.content
        assert b'name="drawing_type"' in request.content
        assert b"3d" in request.content
        return httpx.Response(
            200,
            json={
                "metadata": drawing.metadata.model_dump(mode="json"),
                "render_png_base64": base64.b64encode(drawing.render_png).decode(),
            },
        )

    evaluator = HttpDWGEvaluator(
        "https://evaluator.test",
        token="secret",
        evaluator_version="autocad-2026.1-v1",
        transport=httpx.MockTransport(handler),
    )

    health = asyncio.run(evaluator.health())
    evaluated = asyncio.run(
        evaluator.inspect_and_render(
            source, source_name="attempt.dwg", drawing_type="3d"
        )
    )

    assert health["status"] == "ok"
    assert evaluated == drawing
    assert len(requests) == 2


def test_layout_metadata_records_viewport_inventory() -> None:
    viewport = LayoutViewportFeature(
        number=2,
        center={"x": 100, "y": 75},
        width=80,
        height=60,
        view_direction={"x": 1, "y": -1, "z": 1},
        view_target={"x": 0, "y": 0, "z": 0},
        custom_scale=0.5,
        locked=True,
    )
    layout = PaperLayoutSummary(
        name="Layout2",
        tab_order=2,
        entity_count=3,
        non_viewport_entity_count=2,
        floating_viewport_count=1,
        dimension_count=1,
        text_annotation_count=1,
        entity_types={"VIEWPORT": 1, "DIMENSION": 1, "MTEXT": 1},
        viewports=(viewport,),
        has_presentation=True,
    )

    assert layout.viewports[0].view_direction.z == 1
    assert layout.floating_viewport_count == 1


def test_windows_evaluator_pins_3d_rendering_to_layout1() -> None:
    source = (
        Path(__file__).parents[1]
        / "sandbox"
        / "windows"
        / "evaluator"
        / "AutoCADBench.Plugin.cs"
    ).read_text()

    assert "InspectPaperLayouts(" in source
    assert 'schema_version = "4"' in source
    assert "SelectRenderLayout(metadata);" in source
    assert "GetLayoutId(layoutName)" in source
    assert "layoutManager.CurrentLayout" not in source
    assert "model_space_3d_entity_count" in source
    assert "model_space_3d_entity_types" in source
    assert "model_space_extents_3d" in source
    assert "model_space_extents_all" in source
    assert "model_space_3d_entities" in source
    assert "MillimetersPerDrawingUnit(database.Insunits)" in source
    assert "unit_scale_to_mm" in source
    assert "measurement *= unitScale" in source
    assert "ThreeDimensionalDxfNames.Contains(dxfName)" in source
    assert '"REGION"' not in source.split("ThreeDimensionalDxfNames", 1)[1].split("};", 1)[0]
    assert "entity is RasterImage" in source
    assert 'metadata.drawing_type == "2d"' in source
    assert '"Layout1"' in source
    assert '"task_contract_3d_layout1"' in source
    assert '"required_3d_layout1_empty"' in source
    assert '"required_3d_layout1_missing"' in source
    assert "PresentationScore" not in source
    assert '"highest_populated_paper_layout"' not in source
    assert "new UTF8Encoding(false)" in source
    assert "serializer.Serialize(metadata), Utf8NoBom" in source


def test_windows_evaluator_bounds_and_retries_stalled_autocad_launches() -> None:
    evaluator_root = (
        Path(__file__).parents[1] / "sandbox" / "windows" / "evaluator"
    )
    runner = (evaluator_root / "run-evaluation.ps1").read_text()
    interactive = (evaluator_root / "run-interactive-evaluation.ps1").read_text()

    assert '"autocad-2019-r23.0.46-v5"' in runner
    assert '"autocad-2019-r23.0.46-v5"' in interactive
    assert '"6698199549b4aa88b23cb616fdbce7e4ada7b7ae562b612cc82c465e9b2edf53"' in runner
    assert '"6698199549b4aa88b23cb616fdbce7e4ada7b7ae562b612cc82c465e9b2edf53"' in interactive
    assert "[int]$MaxLaunchAttempts = 3" in runner
    assert "[int]$LaunchTimeoutSeconds = 120" in runner
    assert "$process.WaitForExit($LaunchTimeoutSeconds * 1000)" in runner
    assert "taskkill.exe /PID $Process.Id /T /F" in runner
    assert "Stop-EvaluatorProcessTree $process" in runner
    assert "-Wait" not in runner
    assert '"broker\\acadiso.dwt"' in runner
    assert '"PROXYNOTICE"' in runner
    assert '"_.OPEN"' in runner
    assert '"/t", (\'"\' + $template + \'"\')' in runner
    assert '"-TimeoutSeconds " + $TimeoutSeconds' in interactive
    assert "-MaxLaunchAttempts $launchAttemptLimit" in interactive
    assert "-LaunchTimeoutSeconds $launchTimeoutSeconds" in interactive
    assert "launch_attempts = $result.LaunchAttempts" in interactive