from __future__ import annotations import asyncio import pytest from autocad_bench.evaluation.scoring import ( DimensionFeature, DrawingExtents, EvaluationFailure, GeometryPathFeature, GoldCacheStore, LayoutViewportFeature, PaperLayoutSummary, Point2D, ScorerConfig, SpatialExtents, StaticDWGEvaluator, TextAnnotationFeature, score_final_dwg, ) from autocad_bench.evaluation.scoring.render_compare import compare_evaluated_drawings from tests.scoring_helpers import invalid_drawing, line_png, valid_drawing def _path(*points: tuple[float, float], closed: bool = False) -> GeometryPathFeature: return GeometryPathFeature( entity_type="LWPOLYLINE", closed=closed, points=tuple(Point2D(x=x, y=y) for x, y in points), ) def _semantic_drawing( source: bytes, *, paths: tuple[GeometryPathFeature, ...], dimensions: tuple[DimensionFeature, ...] = (), render_png: bytes | None = None, ): drawing = valid_drawing(source, render_png=render_png) metadata = drawing.metadata.model_copy( update={"geometry_paths": paths, "dimensions": dimensions} ) return drawing.model_copy(update={"metadata": metadata}) def _dimension(value: float, x: float, y: float) -> DimensionFeature: return DimensionFeature( kind="RotatedDimension", measurement=value, text="", text_position=Point2D(x=x, y=y), ) def _annotation(text: str, x: float, y: float) -> TextAnnotationFeature: return TextAnnotationFeature( entity_type="MText", text=text, position=Point2D(x=x, y=y), height=2.5, ) def _viewport( number: int, *, center: tuple[float, float], direction: tuple[float, float, float], ) -> LayoutViewportFeature: return LayoutViewportFeature( number=number, center={"x": center[0], "y": center[1]}, width=40, height=30, view_direction={"x": direction[0], "y": direction[1], "z": direction[2]}, view_target={"x": 0, "y": 0, "z": 0}, ) def _three_d_drawing( source: bytes, *, extents: SpatialExtents, viewports: tuple[LayoutViewportFeature, ...], dimensions: tuple[DimensionFeature, ...] = (), annotations: tuple[TextAnnotationFeature, ...] = (), render_png: bytes | None = None, schema_version: str = "4", three_d_entities: int = 1, ): drawing = valid_drawing(source, drawing_type="3d", render_png=render_png) layout = PaperLayoutSummary( name="Layout1", tab_order=1, entity_count=len(viewports) + len(dimensions) + len(annotations), non_viewport_entity_count=len(dimensions) + len(annotations), floating_viewport_count=len(viewports), dimension_count=len(dimensions), text_annotation_count=len(annotations), entity_types={ **({"VIEWPORT": len(viewports)} if viewports else {}), **({"DIMENSION": len(dimensions)} if dimensions else {}), **({"MTEXT": len(annotations)} if annotations else {}), }, viewports=viewports, dimensions=dimensions, text_annotations=annotations, has_presentation=True, ) metadata = drawing.metadata.model_copy( update={ "schema_version": schema_version, "model_space_3d_entity_count": three_d_entities, "model_space_3d_entity_types": ( {"3DSOLID": three_d_entities} if three_d_entities else {} ), "model_space_extents_3d": extents, "paper_layouts": (layout,), } ) return drawing.model_copy(update={"metadata": metadata}) def test_identical_final_render_and_extents_score_one() -> None: gold = valid_drawing(b"gold") candidate = valid_drawing(b"candidate") result = compare_evaluated_drawings( task_id="task-001", candidate=candidate, gold=gold ).result assert result.valid assert result.score == pytest.approx(1.0) assert result.linework_similarity == pytest.approx(1.0) assert result.extent_similarity == pytest.approx(1.0) def test_pixel_tolerance_accepts_small_render_shift() -> None: base_lines = ( ((20, 20), (108, 20)), ((108, 20), (108, 108)), ((108, 108), (20, 108)), ((20, 108), (20, 20)), ) shifted = tuple( ((start[0] + 1, start[1] + 1), (end[0] + 1, end[1] + 1)) for start, end in base_lines ) gold = valid_drawing(b"gold", render_png=line_png(base_lines)) candidate = valid_drawing(b"candidate", render_png=line_png(shifted)) result = compare_evaluated_drawings( task_id="task-001", candidate=candidate, gold=gold ).result assert result.linework_similarity == pytest.approx(1.0) assert result.score == pytest.approx(1.0) def test_missing_geometry_reduces_linework_score() -> None: gold = valid_drawing(b"gold") candidate = valid_drawing( b"candidate", render_png=line_png( ( ((20, 20), (108, 20)), ((108, 20), (108, 108)), ((108, 108), (20, 108)), ) ), ) result = compare_evaluated_drawings( task_id="task-001", candidate=candidate, gold=gold ).result assert 0.0 < result.linework_similarity < 1.0 assert result.score < 1.0 def test_wrong_world_scale_is_penalized_even_with_same_normalized_render() -> None: gold = valid_drawing(b"gold") candidate = valid_drawing( b"candidate", extents=DrawingExtents( min_x=50, min_y=20, max_x=250, max_y=220, units="mm" ), ) result = compare_evaluated_drawings( task_id="task-001", candidate=candidate, gold=gold ).result assert result.linework_similarity == pytest.approx(1.0) assert result.extent_similarity == pytest.approx(0.5) assert result.score == pytest.approx(0.9) @pytest.mark.parametrize( "failure", [ EvaluationFailure.UNREADABLE_DWG, EvaluationFailure.EMPTY_MODEL_SPACE, EvaluationFailure.PROHIBITED_ENTITY, EvaluationFailure.EXTERNAL_REFERENCE, EvaluationFailure.RENDER_FAILED, ], ) def test_invalid_evaluator_result_forces_zero( failure: EvaluationFailure, tmp_path ) -> None: gold_bytes = b"AC1032-gold" candidate_bytes = b"AC1032-candidate" gold = valid_drawing(gold_bytes) candidate = invalid_drawing(candidate_bytes, failure) cache = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path) cache.write("task-001", gold) evaluator = StaticDWGEvaluator( {candidate.metadata.source_sha256: candidate}, evaluator_version="static-test-v1", ) bundle = asyncio.run( score_final_dwg( task_id="task-001", candidate_dwg=candidate_bytes, evaluator=evaluator, gold_cache=cache, ) ) assert bundle.result.score == 0.0 assert not bundle.result.valid assert bundle.result.failure_reason is failure def test_missing_and_invalid_gold_cache_are_distinct(tmp_path) -> None: candidate_bytes = b"AC1032-candidate" candidate = valid_drawing(candidate_bytes) evaluator = StaticDWGEvaluator( {candidate.metadata.source_sha256: candidate}, evaluator_version="static-test-v1", ) cache = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path) missing = asyncio.run( score_final_dwg( task_id="task-001", candidate_dwg=candidate_bytes, evaluator=evaluator, gold_cache=cache, ) ).result assert missing.failure_reason is EvaluationFailure.GOLD_CACHE_MISSING entry_dir = cache.write("task-001", valid_drawing(b"AC1032-gold")) (entry_dir / "render.png").write_bytes(b"corrupt") invalid = asyncio.run( score_final_dwg( task_id="task-001", candidate_dwg=candidate_bytes, evaluator=evaluator, gold_cache=cache, ) ).result assert invalid.failure_reason is EvaluationFailure.GOLD_CACHE_INVALID def test_evaluator_unavailable_forces_zero(tmp_path) -> None: candidate_bytes = b"AC1032-candidate" cache = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path) cache.write("task-001", valid_drawing(b"AC1032-gold")) evaluator = StaticDWGEvaluator({}, evaluator_version="static-test-v1") result = asyncio.run( score_final_dwg( task_id="task-001", candidate_dwg=candidate_bytes, evaluator=evaluator, gold_cache=cache, ) ).result assert result.score == 0.0 assert result.failure_reason is EvaluationFailure.EVALUATOR_UNAVAILABLE @pytest.mark.parametrize( ("candidate", "reason"), [ (None, EvaluationFailure.MISSING_CANDIDATE), (b"not-a-dwg", EvaluationFailure.INVALID_DWG_SIGNATURE), ], ) def test_candidate_input_gates(candidate: bytes | None, reason: EvaluationFailure, tmp_path) -> None: evaluator = StaticDWGEvaluator({}, evaluator_version="static-test-v1") cache = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path) result = asyncio.run( score_final_dwg( task_id="task-001", candidate_dwg=candidate, evaluator=evaluator, gold_cache=cache, config=ScorerConfig(), ) ).result assert result.score == 0.0 assert result.failure_reason is reason def test_score_json_is_deterministic() -> None: gold = valid_drawing(b"gold") candidate = valid_drawing(b"candidate") first = compare_evaluated_drawings( task_id="task-001", candidate=candidate, gold=gold ).result.model_dump_json() second = compare_evaluated_drawings( task_id="task-001", candidate=candidate, gold=gold ).result.model_dump_json() assert first == second def test_semantic_geometry_ignores_translation_and_entity_segmentation() -> None: gold = _semantic_drawing( b"gold", paths=( _path((0, 0), (100, 0)), _path((100, 0), (100, 100)), _path((100, 100), (0, 100)), _path((0, 100), (0, 0)), ), ) candidate = _semantic_drawing( b"candidate", paths=( _path( (250, -40), (350, -40), (350, 60), (250, 60), closed=True, ), ), ) result = compare_evaluated_drawings( task_id="task-001", candidate=candidate, gold=gold ).result assert result.scoring_method == "vector-semantic-v4" assert result.geometry_similarity == pytest.approx(1.0, abs=0.01) assert result.geometry_extent_similarity == pytest.approx(1.0) assert result.score == pytest.approx(1.0, abs=0.01) def test_semantic_geometry_penalizes_wrong_physical_scale() -> None: gold = _semantic_drawing( b"gold", paths=(_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),), ) candidate = _semantic_drawing( b"candidate", paths=(_path((0, 0), (200, 0), (200, 200), (0, 200), closed=True),), ) result = compare_evaluated_drawings( task_id="task-001", candidate=candidate, gold=gold ).result assert result.geometry_extent_similarity == pytest.approx(0.5) assert result.geometry_similarity < 0.6 assert result.score < 0.8 def test_dimension_score_rewards_correct_values_and_penalizes_one_missing() -> None: values = (50, 30, 10, 20, 42, 144, 46, 9.78, 18.34, 44, 18) gold_dimensions = tuple( _dimension(value, index * 10.0, 130.0) for index, value in enumerate(values) ) candidate_dimensions = tuple( _dimension(value, 200.0 + index * 10.0, 80.0) for index, value in enumerate(values[:-1]) ) gold = _semantic_drawing( b"gold", paths=(_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),), dimensions=gold_dimensions, ) candidate = _semantic_drawing( b"candidate", paths=( _path((200, -50), (300, -50), (300, 50), (200, 50), closed=True), ), dimensions=candidate_dimensions, ) result = compare_evaluated_drawings( task_id="task-001", candidate=candidate, gold=gold ).result assert result.matched_dimensions == 10 assert result.candidate_dimensions == 10 assert result.gold_dimensions == 11 assert result.dimension_similarity == pytest.approx(20 / 21) assert result.dimension_measurement_similarity == pytest.approx(20 / 21) assert result.dimension_placement_similarity == pytest.approx(20 / 21) assert not result.annotation_applicable assert result.annotation_similarity == 0.0 def test_visible_dimension_text_override_is_the_scored_value() -> None: geometry = (_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),) gold_dimension = _dimension(50, 50, 110) candidate_dimension = _dimension(80, 50, 110).model_copy( update={"text": "50"} ) gold = _semantic_drawing( b"gold", paths=geometry, dimensions=(gold_dimension,) ) candidate = _semantic_drawing( b"candidate", paths=geometry, dimensions=(candidate_dimension,) ) result = compare_evaluated_drawings( task_id="task-001", candidate=candidate, gold=gold ).result assert result.dimension_measurement_similarity == pytest.approx(1.0) assert result.matched_dimensions == 1 def test_manual_dimension_text_is_equivalent_to_a_native_dimension() -> None: geometry = (_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),) gold = _semantic_drawing( b"gold", paths=geometry, dimensions=(_dimension(50, 50, 110),) ) candidate = _semantic_drawing(b"candidate", paths=geometry) candidate = candidate.model_copy( update={ "metadata": candidate.metadata.model_copy( update={"text_annotations": (_annotation("50", 50, 110),)} ) } ) result = compare_evaluated_drawings( task_id="task-001", candidate=candidate, gold=gold ).result assert result.dimension_measurement_similarity == pytest.approx(1.0) assert result.matched_dimensions == 1 assert result.candidate_dimensions == 1 assert result.candidate_annotations == 0 def test_multiplicity_prefixed_manual_radius_is_parsed() -> None: geometry = (_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),) radial = _dimension(1.4, 50, 110).model_copy( update={"kind": "RadialDimension", "text": "2x<>"} ) gold = _semantic_drawing(b"gold", paths=geometry, dimensions=(radial,)) candidate = _semantic_drawing(b"candidate", paths=geometry) candidate = candidate.model_copy( update={ "metadata": candidate.metadata.model_copy( update={"text_annotations": (_annotation("2xR1.4", 50, 110),)} ) } ) result = compare_evaluated_drawings( task_id="task-001", candidate=candidate, gold=gold ).result assert result.dimension_measurement_similarity == pytest.approx(1.0) assert result.matched_dimensions == 1 def test_leader_paths_are_annotation_not_part_geometry() -> None: geometry = (_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),) leader = _path((10, 10), (180, 180)).model_copy( update={"entity_type": "LEADER"} ) gold = _semantic_drawing(b"gold", paths=geometry) candidate = _semantic_drawing(b"candidate", paths=geometry + (leader,)) result = compare_evaluated_drawings( task_id="task-001", candidate=candidate, gold=gold ).result assert result.geometry_similarity == pytest.approx(1.0, abs=0.01) def test_isolated_centerlines_are_not_base_geometry() -> None: geometry = (_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),) centerlines = ( _path((-20, 50), (120, 50)).model_copy(update={"entity_type": "LINE"}), _path((50, -20), (50, 120)).model_copy(update={"entity_type": "LINE"}), ) gold = _semantic_drawing(b"gold", paths=geometry) candidate = _semantic_drawing(b"candidate", paths=geometry + centerlines) result = compare_evaluated_drawings( task_id="task-001", candidate=candidate, gold=gold ).result assert result.geometry_similarity == pytest.approx(1.0, abs=0.01) def test_explicit_no_dimension_required_note_allows_uniform_scale_alignment() -> None: gold = _semantic_drawing( b"gold", paths=(_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),), ) candidate = _semantic_drawing( b"candidate", paths=(_path((0, 0), (200, 0), (200, 200), (0, 200), closed=True),), ) note = (_annotation("No dimension required for the drawing", 20, -20),) gold = gold.model_copy( update={"metadata": gold.metadata.model_copy(update={"text_annotations": note})} ) candidate = candidate.model_copy( update={ "metadata": candidate.metadata.model_copy(update={"text_annotations": note}) } ) result = compare_evaluated_drawings( task_id="task-021", candidate=candidate, gold=gold ).result assert result.geometry_similarity == pytest.approx(1.0, abs=0.01) assert result.score == pytest.approx(1.0, abs=0.01) def test_radically_different_raster_style_has_only_five_percent_weight() -> None: geometry = (_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),) gold = _semantic_drawing(b"gold", paths=geometry) black_render = line_png( tuple(((0, y), (127, y)) for y in range(128)), ) candidate = _semantic_drawing( b"candidate", paths=geometry, render_png=black_render ) result = compare_evaluated_drawings( task_id="task-001", candidate=candidate, gold=gold ).result assert result.geometry_similarity == pytest.approx(1.0, abs=0.01) assert result.visual_similarity < 0.6 assert result.score >= 0.90 def test_absent_dimensions_and_annotations_are_not_free_credit() -> None: gold = _semantic_drawing( b"gold", paths=(_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),), ) candidate = _semantic_drawing( b"candidate", paths=(_path((0, 0), (200, 0), (200, 200), (0, 200), closed=True),), ) result = compare_evaluated_drawings( task_id="task-021", candidate=candidate, gold=gold ).result assert not result.dimension_applicable assert not result.annotation_applicable assert result.dimension_similarity == 0.0 assert result.annotation_similarity == 0.0 weighted = ( 0.6 * result.geometry_similarity + 0.05 * result.visual_similarity ) / 0.65 geometry_gate = 0.35 + (0.65 * result.geometry_similarity) expected = min(weighted, geometry_gate) assert result.score == pytest.approx(expected) def test_standalone_text_annotations_are_scored_separately() -> None: geometry = (_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),) gold = _semantic_drawing(b"gold", paths=geometry) candidate = _semantic_drawing(b"candidate", paths=geometry) gold = gold.model_copy( update={ "metadata": gold.metadata.model_copy( update={"text_annotations": (_annotation("DETAIL A", 20, 20),)} ) } ) candidate = candidate.model_copy( update={ "metadata": candidate.metadata.model_copy( update={"text_annotations": (_annotation("WRONG NOTE", 20, 20),)} ) } ) result = compare_evaluated_drawings( task_id="task-010", candidate=candidate, gold=gold ).result assert result.annotation_applicable assert result.annotation_similarity == 0.0 assert result.matched_annotations == 0 assert result.score < 1.0 def test_3d_scorer_uses_clean_spatial_bounds_and_layout1_features() -> None: gold_extents = SpatialExtents( min_x=-8.5, min_y=-8.5, min_z=0, max_x=8.5, max_y=8.5, max_z=12, ) candidate_extents = SpatialExtents( min_x=100, min_y=200, min_z=50, max_x=117, max_y=217, max_z=62, ) views = ( _viewport(2, center=(40, 40), direction=(0, 0, 10)), _viewport(3, center=(100, 40), direction=(0, -3, 0)), _viewport(4, center=(40, 100), direction=(1, -1, 1)), ) dimensions = (_dimension(17, 10, 10), _dimension(12, 20, 10)) annotations = (_annotation("DETAIL A", 10, 20),) gold = _three_d_drawing( b"gold", extents=gold_extents, viewports=views, dimensions=dimensions, annotations=annotations, ) candidate = _three_d_drawing( b"candidate", extents=candidate_extents, viewports=views, dimensions=dimensions, annotations=annotations, ) result = compare_evaluated_drawings( task_id="task-030", candidate=candidate, gold=gold ).result assert result.scoring_method == "layout-spatial-3d-v2" assert result.geometry_extent_applicable assert result.geometry_extent_similarity == pytest.approx(1.0) assert result.geometry_similarity == pytest.approx(1.0) assert result.dimension_similarity == pytest.approx(1.0) assert result.annotation_similarity == pytest.approx(1.0) assert result.presentation_similarity == pytest.approx(1.0) assert result.score == pytest.approx(1.0) def test_3d_visual_comparison_normalizes_sheet_framing() -> None: extents = SpatialExtents( min_x=0, min_y=0, min_z=0, max_x=100, max_y=80, max_z=40, ) views = ( _viewport(2, center=(40, 40), direction=(0, 0, 1)), _viewport(3, center=(100, 40), direction=(1, -1, 1)), ) gold_render = line_png( ( ((20, 20), (108, 20)), ((108, 20), (108, 108)), ((108, 108), (20, 108)), ((20, 108), (20, 20)), ) ) candidate_render = line_png( ( ((40, 40), (90, 40)), ((90, 40), (90, 90)), ((90, 90), (40, 90)), ((40, 90), (40, 40)), ) ) gold = _three_d_drawing( b"gold", extents=extents, viewports=views, render_png=gold_render ) candidate = _three_d_drawing( b"candidate", extents=extents, viewports=views, render_png=candidate_render, ) result = compare_evaluated_drawings( task_id="task-022", candidate=candidate, gold=gold ).result assert result.visual_similarity == pytest.approx(1.0) assert result.score == pytest.approx(1.0) def test_3d_presentation_cannot_rescue_wrong_geometry() -> None: extents = SpatialExtents( min_x=0, min_y=0, min_z=0, max_x=100, max_y=80, max_z=40, ) views = ( _viewport(2, center=(40, 40), direction=(0, 0, 1)), _viewport(3, center=(100, 40), direction=(1, -1, 1)), ) gold = _three_d_drawing( b"gold", extents=extents, viewports=views, render_png=line_png( ( ((20, 20), (108, 20)), ((108, 20), (108, 108)), ((108, 108), (20, 108)), ((20, 108), (20, 20)), ) ), ) candidate = _three_d_drawing( b"candidate", extents=extents, viewports=views, render_png=line_png((((20, 20), (108, 108)),)), ) result = compare_evaluated_drawings( task_id="task-022", candidate=candidate, gold=gold ).result assert result.presentation_similarity == pytest.approx(1.0) assert result.score <= 0.35 + (0.65 * result.geometry_similarity) assert result.score < 0.7 def test_3d_schema_v3_polluted_bounds_are_non_applicable() -> None: polluted = SpatialExtents( min_x=-10000, min_y=-10000, min_z=0, max_x=17, max_y=17, max_z=12, ) clean = SpatialExtents( min_x=-8.5, min_y=-8.5, min_z=0, max_x=8.5, max_y=8.5, max_z=12, ) views = (_viewport(2, center=(40, 40), direction=(0, 0, 1)),) gold = _three_d_drawing( b"gold", extents=polluted, viewports=views, schema_version="3" ) candidate = _three_d_drawing( b"candidate", extents=clean, viewports=views, schema_version="3" ) result = compare_evaluated_drawings( task_id="task-026", candidate=candidate, gold=gold ).result assert not result.geometry_extent_applicable assert result.geometry_extent_similarity == 0.0 assert result.geometry_similarity == pytest.approx(1.0) assert result.score == pytest.approx(1.0) def test_3d_copied_views_without_true_geometry_cannot_earn_geometry_credit() -> None: extents = SpatialExtents( min_x=0, min_y=0, min_z=0, max_x=100, max_y=80, max_z=40, ) views = (_viewport(2, center=(40, 40), direction=(0, 0, 1)),) gold = _three_d_drawing(b"gold", extents=extents, viewports=views) candidate = _three_d_drawing( b"candidate", extents=extents, viewports=views, three_d_entities=0, ) result = compare_evaluated_drawings( task_id="task-049", candidate=candidate, gold=gold ).result assert result.visual_similarity == pytest.approx(1.0) assert result.geometry_similarity == 0.0 assert result.score < 0.5 def test_3d_native_drawing_view_support_viewports_are_deduplicated() -> None: extents = SpatialExtents( min_x=0, min_y=0, min_z=0, max_x=100, max_y=80, max_z=40, ) gold_views = ( _viewport(2, center=(40, 40), direction=(0, 0, 1)), _viewport(3, center=(40, 40), direction=(0, -1, 0)), _viewport(4, center=(100, 40), direction=(0, 0, 1)), _viewport(5, center=(100, 40), direction=(1, -1, 1)), ) candidate_views = ( _viewport(2, center=(20, 20), direction=(0, -2, 0)), _viewport(3, center=(80, 20), direction=(10, -10, 10)), ) gold = _three_d_drawing(b"gold", extents=extents, viewports=gold_views) candidate = _three_d_drawing( b"candidate", extents=extents, viewports=candidate_views ) result = compare_evaluated_drawings( task_id="task-026", candidate=candidate, gold=gold ).result assert result.gold_views == 2 assert result.candidate_views == 2 assert result.view_direction_similarity == pytest.approx(1.0)