tests/test_gold_cache.py

from __future__ import annotations

import asyncio
import json

import pytest
from PIL import Image

from autocad_bench.tasks.manifest import load_manifest
from autocad_bench.evaluation.scoring import (
    GoldCacheError,
    GoldCacheInvalidError,
    GoldCacheStore,
    StaticDWGEvaluator,
    build_gold_cache,
)
from tests.scoring_helpers import valid_drawing


def test_gold_cache_round_trip_and_tamper_detection(tmp_path) -> None:
    drawing = valid_drawing(b"AC1032-gold")
    store = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path)
    entry_dir = store.write("task-001", drawing)

    assert store.load("task-001") == drawing

    (entry_dir / "render.png").write_bytes(b"corrupt")
    with pytest.raises(GoldCacheInvalidError):
        store.load("task-001")


def test_gold_cache_rejects_3d_gold_not_presented_on_layout1(tmp_path) -> None:
    drawing = valid_drawing(b"AC1032-3d-gold", drawing_type="3d")
    invalid_metadata = drawing.metadata.model_copy(
        update={
            "rendered_layout": "Layout2",
            "paper_layouts": tuple(
                layout.model_copy(update={"name": "Layout2"})
                for layout in drawing.metadata.paper_layouts
            ),
        }
    )
    store = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path)

    with pytest.raises(GoldCacheError, match="must render paper-space Layout1"):
        store.write(
            "task-022",
            drawing.model_copy(update={"metadata": invalid_metadata}),
        )


def test_builds_and_verifies_all_five_development_entries(tmp_path) -> None:
    fixtures = {}
    dev_entries = [entry for entry in load_manifest() if entry.split == "dev"]
    for entry in dev_entries:
        gold_bytes = entry.resolve_gold_path().read_bytes()
        # Reference PNG use is test-only. Production caches must come from AutoCAD.
        render_png = entry.resolve_image_path().read_bytes()
        with Image.open(entry.resolve_image_path()) as image:
            width, height = image.size
        drawing = valid_drawing(
            gold_bytes,
            render_png=render_png,
            extents={
                "min_x": 0,
                "min_y": 0,
                "max_x": width,
                "max_y": height,
                "units": "mm",
            },
            drawing_type=entry.drawing_type,
        )
        fixtures[drawing.metadata.source_sha256] = drawing

    evaluator = StaticDWGEvaluator(fixtures, evaluator_version="static-test-v1")
    store = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path)

    built = asyncio.run(build_gold_cache(evaluator, store, split="dev"))

    assert built == tuple(entry.task_id for entry in dev_entries)
    assert store.verify(list(built)) == built
    for task_id in built:
        metadata = json.loads(
            (tmp_path / "static-test-v1" / task_id / "metadata.json").read_text()
        )
        assert metadata["task_id"] == task_id
        assert metadata["evaluator_version"] == "static-test-v1"