File size: 5,670 Bytes
1866d85 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 | from __future__ import annotations
import asyncio
import base64
from pathlib import Path
import httpx
from autocad_bench.evaluation.scoring import HttpDWGEvaluator
from autocad_bench.evaluation.scoring.models import LayoutViewportFeature, PaperLayoutSummary
from tests.scoring_helpers import valid_drawing
def test_http_evaluator_contract_and_authentication() -> None:
source = b"AC1032-candidate"
drawing = valid_drawing(
source,
evaluator_version="autocad-2026.1-v1",
drawing_type="3d",
)
requests: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
assert request.headers["authorization"] == "Bearer secret"
if request.url.path == "/health":
return httpx.Response(
200,
json={"status": "ok", "evaluator_version": "autocad-2026.1-v1"},
)
assert request.url.path == "/v1/inspect-render"
assert "multipart/form-data" in request.headers["content-type"]
assert source in request.content
assert b'name="drawing_type"' in request.content
assert b"3d" in request.content
return httpx.Response(
200,
json={
"metadata": drawing.metadata.model_dump(mode="json"),
"render_png_base64": base64.b64encode(drawing.render_png).decode(),
},
)
evaluator = HttpDWGEvaluator(
"https://evaluator.test",
token="secret",
evaluator_version="autocad-2026.1-v1",
transport=httpx.MockTransport(handler),
)
health = asyncio.run(evaluator.health())
evaluated = asyncio.run(
evaluator.inspect_and_render(
source, source_name="attempt.dwg", drawing_type="3d"
)
)
assert health["status"] == "ok"
assert evaluated == drawing
assert len(requests) == 2
def test_layout_metadata_records_viewport_inventory() -> None:
viewport = LayoutViewportFeature(
number=2,
center={"x": 100, "y": 75},
width=80,
height=60,
view_direction={"x": 1, "y": -1, "z": 1},
view_target={"x": 0, "y": 0, "z": 0},
custom_scale=0.5,
locked=True,
)
layout = PaperLayoutSummary(
name="Layout2",
tab_order=2,
entity_count=3,
non_viewport_entity_count=2,
floating_viewport_count=1,
dimension_count=1,
text_annotation_count=1,
entity_types={"VIEWPORT": 1, "DIMENSION": 1, "MTEXT": 1},
viewports=(viewport,),
has_presentation=True,
)
assert layout.viewports[0].view_direction.z == 1
assert layout.floating_viewport_count == 1
def test_windows_evaluator_pins_3d_rendering_to_layout1() -> None:
source = (
Path(__file__).parents[1]
/ "sandbox"
/ "windows"
/ "evaluator"
/ "AutoCADBench.Plugin.cs"
).read_text()
assert "InspectPaperLayouts(" in source
assert 'schema_version = "4"' in source
assert "SelectRenderLayout(metadata);" in source
assert "GetLayoutId(layoutName)" in source
assert "layoutManager.CurrentLayout" not in source
assert "model_space_3d_entity_count" in source
assert "model_space_3d_entity_types" in source
assert "model_space_extents_3d" in source
assert "model_space_extents_all" in source
assert "model_space_3d_entities" in source
assert "MillimetersPerDrawingUnit(database.Insunits)" in source
assert "unit_scale_to_mm" in source
assert "measurement *= unitScale" in source
assert "ThreeDimensionalDxfNames.Contains(dxfName)" in source
assert '"REGION"' not in source.split("ThreeDimensionalDxfNames", 1)[1].split("};", 1)[0]
assert "entity is RasterImage" in source
assert 'metadata.drawing_type == "2d"' in source
assert '"Layout1"' in source
assert '"task_contract_3d_layout1"' in source
assert '"required_3d_layout1_empty"' in source
assert '"required_3d_layout1_missing"' in source
assert "PresentationScore" not in source
assert '"highest_populated_paper_layout"' not in source
assert "new UTF8Encoding(false)" in source
assert "serializer.Serialize(metadata), Utf8NoBom" in source
def test_windows_evaluator_bounds_and_retries_stalled_autocad_launches() -> None:
evaluator_root = (
Path(__file__).parents[1] / "sandbox" / "windows" / "evaluator"
)
runner = (evaluator_root / "run-evaluation.ps1").read_text()
interactive = (evaluator_root / "run-interactive-evaluation.ps1").read_text()
assert '"autocad-2019-r23.0.46-v5"' in runner
assert '"autocad-2019-r23.0.46-v5"' in interactive
assert '"6698199549b4aa88b23cb616fdbce7e4ada7b7ae562b612cc82c465e9b2edf53"' in runner
assert '"6698199549b4aa88b23cb616fdbce7e4ada7b7ae562b612cc82c465e9b2edf53"' in interactive
assert "[int]$MaxLaunchAttempts = 3" in runner
assert "[int]$LaunchTimeoutSeconds = 120" in runner
assert "$process.WaitForExit($LaunchTimeoutSeconds * 1000)" in runner
assert "taskkill.exe /PID $Process.Id /T /F" in runner
assert "Stop-EvaluatorProcessTree $process" in runner
assert "-Wait" not in runner
assert '"broker\\acadiso.dwt"' in runner
assert '"PROXYNOTICE"' in runner
assert '"_.OPEN"' in runner
assert '"/t", (\'"\' + $template + \'"\')' in runner
assert '"-TimeoutSeconds " + $TimeoutSeconds' in interactive
assert "-MaxLaunchAttempts $launchAttemptLimit" in interactive
assert "-LaunchTimeoutSeconds $launchTimeoutSeconds" in interactive
assert "launch_attempts = $result.LaunchAttempts" in interactive
|