File size: 3,039 Bytes
1866d85 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | from __future__ import annotations
import asyncio
import json
import pytest
from PIL import Image
from autocad_bench.tasks.manifest import load_manifest
from autocad_bench.evaluation.scoring import (
GoldCacheError,
GoldCacheInvalidError,
GoldCacheStore,
StaticDWGEvaluator,
build_gold_cache,
)
from tests.scoring_helpers import valid_drawing
def test_gold_cache_round_trip_and_tamper_detection(tmp_path) -> None:
drawing = valid_drawing(b"AC1032-gold")
store = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path)
entry_dir = store.write("task-001", drawing)
assert store.load("task-001") == drawing
(entry_dir / "render.png").write_bytes(b"corrupt")
with pytest.raises(GoldCacheInvalidError):
store.load("task-001")
def test_gold_cache_rejects_3d_gold_not_presented_on_layout1(tmp_path) -> None:
drawing = valid_drawing(b"AC1032-3d-gold", drawing_type="3d")
invalid_metadata = drawing.metadata.model_copy(
update={
"rendered_layout": "Layout2",
"paper_layouts": tuple(
layout.model_copy(update={"name": "Layout2"})
for layout in drawing.metadata.paper_layouts
),
}
)
store = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path)
with pytest.raises(GoldCacheError, match="must render paper-space Layout1"):
store.write(
"task-022",
drawing.model_copy(update={"metadata": invalid_metadata}),
)
def test_builds_and_verifies_all_five_development_entries(tmp_path) -> None:
fixtures = {}
dev_entries = [entry for entry in load_manifest() if entry.split == "dev"]
for entry in dev_entries:
gold_bytes = entry.resolve_gold_path().read_bytes()
# Reference PNG use is test-only. Production caches must come from AutoCAD.
render_png = entry.resolve_image_path().read_bytes()
with Image.open(entry.resolve_image_path()) as image:
width, height = image.size
drawing = valid_drawing(
gold_bytes,
render_png=render_png,
extents={
"min_x": 0,
"min_y": 0,
"max_x": width,
"max_y": height,
"units": "mm",
},
drawing_type=entry.drawing_type,
)
fixtures[drawing.metadata.source_sha256] = drawing
evaluator = StaticDWGEvaluator(fixtures, evaluator_version="static-test-v1")
store = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path)
built = asyncio.run(build_gold_cache(evaluator, store, split="dev"))
assert built == tuple(entry.task_id for entry in dev_entries)
assert store.verify(list(built)) == built
for task_id in built:
metadata = json.loads(
(tmp_path / "static-test-v1" / task_id / "metadata.json").read_text()
)
assert metadata["task_id"] == task_id
assert metadata["evaluator_version"] == "static-test-v1"
|