Download tests/test_scoring.py from windows2t2/cad-bench: direct link, hf CLI and curl.
- Browser
- Download file 26.9 kB
-
https://huggingface.co/windows2t2/cad-bench/resolve/main/tests/test_scoring.py
- Command line
-
hf download hf://windows2t2/cad-bench/tests/test_scoring.py
-
curl -L -o test_scoring.py https://huggingface.co/windows2t2/cad-bench/resolve/main/tests/test_scoring.py
26.9 kB
| from __future__ import annotations | |
| import asyncio | |
| import pytest | |
| from autocad_bench.evaluation.scoring import ( | |
| DimensionFeature, | |
| DrawingExtents, | |
| EvaluationFailure, | |
| GeometryPathFeature, | |
| GoldCacheStore, | |
| LayoutViewportFeature, | |
| PaperLayoutSummary, | |
| Point2D, | |
| ScorerConfig, | |
| SpatialExtents, | |
| StaticDWGEvaluator, | |
| TextAnnotationFeature, | |
| score_final_dwg, | |
| ) | |
| from autocad_bench.evaluation.scoring.render_compare import compare_evaluated_drawings | |
| from tests.scoring_helpers import invalid_drawing, line_png, valid_drawing | |
| def _path(*points: tuple[float, float], closed: bool = False) -> GeometryPathFeature: | |
| return GeometryPathFeature( | |
| entity_type="LWPOLYLINE", | |
| closed=closed, | |
| points=tuple(Point2D(x=x, y=y) for x, y in points), | |
| ) | |
| def _semantic_drawing( | |
| source: bytes, | |
| *, | |
| paths: tuple[GeometryPathFeature, ...], | |
| dimensions: tuple[DimensionFeature, ...] = (), | |
| render_png: bytes | None = None, | |
| ): | |
| drawing = valid_drawing(source, render_png=render_png) | |
| metadata = drawing.metadata.model_copy( | |
| update={"geometry_paths": paths, "dimensions": dimensions} | |
| ) | |
| return drawing.model_copy(update={"metadata": metadata}) | |
| def _dimension(value: float, x: float, y: float) -> DimensionFeature: | |
| return DimensionFeature( | |
| kind="RotatedDimension", | |
| measurement=value, | |
| text="", | |
| text_position=Point2D(x=x, y=y), | |
| ) | |
| def _annotation(text: str, x: float, y: float) -> TextAnnotationFeature: | |
| return TextAnnotationFeature( | |
| entity_type="MText", | |
| text=text, | |
| position=Point2D(x=x, y=y), | |
| height=2.5, | |
| ) | |
| def _viewport( | |
| number: int, | |
| *, | |
| center: tuple[float, float], | |
| direction: tuple[float, float, float], | |
| ) -> LayoutViewportFeature: | |
| return LayoutViewportFeature( | |
| number=number, | |
| center={"x": center[0], "y": center[1]}, | |
| width=40, | |
| height=30, | |
| view_direction={"x": direction[0], "y": direction[1], "z": direction[2]}, | |
| view_target={"x": 0, "y": 0, "z": 0}, | |
| ) | |
| def _three_d_drawing( | |
| source: bytes, | |
| *, | |
| extents: SpatialExtents, | |
| viewports: tuple[LayoutViewportFeature, ...], | |
| dimensions: tuple[DimensionFeature, ...] = (), | |
| annotations: tuple[TextAnnotationFeature, ...] = (), | |
| render_png: bytes | None = None, | |
| schema_version: str = "4", | |
| three_d_entities: int = 1, | |
| ): | |
| drawing = valid_drawing(source, drawing_type="3d", render_png=render_png) | |
| layout = PaperLayoutSummary( | |
| name="Layout1", | |
| tab_order=1, | |
| entity_count=len(viewports) + len(dimensions) + len(annotations), | |
| non_viewport_entity_count=len(dimensions) + len(annotations), | |
| floating_viewport_count=len(viewports), | |
| dimension_count=len(dimensions), | |
| text_annotation_count=len(annotations), | |
| entity_types={ | |
| **({"VIEWPORT": len(viewports)} if viewports else {}), | |
| **({"DIMENSION": len(dimensions)} if dimensions else {}), | |
| **({"MTEXT": len(annotations)} if annotations else {}), | |
| }, | |
| viewports=viewports, | |
| dimensions=dimensions, | |
| text_annotations=annotations, | |
| has_presentation=True, | |
| ) | |
| metadata = drawing.metadata.model_copy( | |
| update={ | |
| "schema_version": schema_version, | |
| "model_space_3d_entity_count": three_d_entities, | |
| "model_space_3d_entity_types": ( | |
| {"3DSOLID": three_d_entities} if three_d_entities else {} | |
| ), | |
| "model_space_extents_3d": extents, | |
| "paper_layouts": (layout,), | |
| } | |
| ) | |
| return drawing.model_copy(update={"metadata": metadata}) | |
| def test_identical_final_render_and_extents_score_one() -> None: | |
| gold = valid_drawing(b"gold") | |
| candidate = valid_drawing(b"candidate") | |
| result = compare_evaluated_drawings( | |
| task_id="task-001", candidate=candidate, gold=gold | |
| ).result | |
| assert result.valid | |
| assert result.score == pytest.approx(1.0) | |
| assert result.linework_similarity == pytest.approx(1.0) | |
| assert result.extent_similarity == pytest.approx(1.0) | |
| def test_pixel_tolerance_accepts_small_render_shift() -> None: | |
| base_lines = ( | |
| ((20, 20), (108, 20)), | |
| ((108, 20), (108, 108)), | |
| ((108, 108), (20, 108)), | |
| ((20, 108), (20, 20)), | |
| ) | |
| shifted = tuple( | |
| ((start[0] + 1, start[1] + 1), (end[0] + 1, end[1] + 1)) | |
| for start, end in base_lines | |
| ) | |
| gold = valid_drawing(b"gold", render_png=line_png(base_lines)) | |
| candidate = valid_drawing(b"candidate", render_png=line_png(shifted)) | |
| result = compare_evaluated_drawings( | |
| task_id="task-001", candidate=candidate, gold=gold | |
| ).result | |
| assert result.linework_similarity == pytest.approx(1.0) | |
| assert result.score == pytest.approx(1.0) | |
| def test_missing_geometry_reduces_linework_score() -> None: | |
| gold = valid_drawing(b"gold") | |
| candidate = valid_drawing( | |
| b"candidate", | |
| render_png=line_png( | |
| ( | |
| ((20, 20), (108, 20)), | |
| ((108, 20), (108, 108)), | |
| ((108, 108), (20, 108)), | |
| ) | |
| ), | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-001", candidate=candidate, gold=gold | |
| ).result | |
| assert 0.0 < result.linework_similarity < 1.0 | |
| assert result.score < 1.0 | |
| def test_wrong_world_scale_is_penalized_even_with_same_normalized_render() -> None: | |
| gold = valid_drawing(b"gold") | |
| candidate = valid_drawing( | |
| b"candidate", | |
| extents=DrawingExtents( | |
| min_x=50, min_y=20, max_x=250, max_y=220, units="mm" | |
| ), | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-001", candidate=candidate, gold=gold | |
| ).result | |
| assert result.linework_similarity == pytest.approx(1.0) | |
| assert result.extent_similarity == pytest.approx(0.5) | |
| assert result.score == pytest.approx(0.9) | |
| def test_invalid_evaluator_result_forces_zero( | |
| failure: EvaluationFailure, tmp_path | |
| ) -> None: | |
| gold_bytes = b"AC1032-gold" | |
| candidate_bytes = b"AC1032-candidate" | |
| gold = valid_drawing(gold_bytes) | |
| candidate = invalid_drawing(candidate_bytes, failure) | |
| cache = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path) | |
| cache.write("task-001", gold) | |
| evaluator = StaticDWGEvaluator( | |
| {candidate.metadata.source_sha256: candidate}, | |
| evaluator_version="static-test-v1", | |
| ) | |
| bundle = asyncio.run( | |
| score_final_dwg( | |
| task_id="task-001", | |
| candidate_dwg=candidate_bytes, | |
| evaluator=evaluator, | |
| gold_cache=cache, | |
| ) | |
| ) | |
| assert bundle.result.score == 0.0 | |
| assert not bundle.result.valid | |
| assert bundle.result.failure_reason is failure | |
| def test_missing_and_invalid_gold_cache_are_distinct(tmp_path) -> None: | |
| candidate_bytes = b"AC1032-candidate" | |
| candidate = valid_drawing(candidate_bytes) | |
| evaluator = StaticDWGEvaluator( | |
| {candidate.metadata.source_sha256: candidate}, | |
| evaluator_version="static-test-v1", | |
| ) | |
| cache = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path) | |
| missing = asyncio.run( | |
| score_final_dwg( | |
| task_id="task-001", | |
| candidate_dwg=candidate_bytes, | |
| evaluator=evaluator, | |
| gold_cache=cache, | |
| ) | |
| ).result | |
| assert missing.failure_reason is EvaluationFailure.GOLD_CACHE_MISSING | |
| entry_dir = cache.write("task-001", valid_drawing(b"AC1032-gold")) | |
| (entry_dir / "render.png").write_bytes(b"corrupt") | |
| invalid = asyncio.run( | |
| score_final_dwg( | |
| task_id="task-001", | |
| candidate_dwg=candidate_bytes, | |
| evaluator=evaluator, | |
| gold_cache=cache, | |
| ) | |
| ).result | |
| assert invalid.failure_reason is EvaluationFailure.GOLD_CACHE_INVALID | |
| def test_evaluator_unavailable_forces_zero(tmp_path) -> None: | |
| candidate_bytes = b"AC1032-candidate" | |
| cache = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path) | |
| cache.write("task-001", valid_drawing(b"AC1032-gold")) | |
| evaluator = StaticDWGEvaluator({}, evaluator_version="static-test-v1") | |
| result = asyncio.run( | |
| score_final_dwg( | |
| task_id="task-001", | |
| candidate_dwg=candidate_bytes, | |
| evaluator=evaluator, | |
| gold_cache=cache, | |
| ) | |
| ).result | |
| assert result.score == 0.0 | |
| assert result.failure_reason is EvaluationFailure.EVALUATOR_UNAVAILABLE | |
| def test_candidate_input_gates(candidate: bytes | None, reason: EvaluationFailure, tmp_path) -> None: | |
| evaluator = StaticDWGEvaluator({}, evaluator_version="static-test-v1") | |
| cache = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path) | |
| result = asyncio.run( | |
| score_final_dwg( | |
| task_id="task-001", | |
| candidate_dwg=candidate, | |
| evaluator=evaluator, | |
| gold_cache=cache, | |
| config=ScorerConfig(), | |
| ) | |
| ).result | |
| assert result.score == 0.0 | |
| assert result.failure_reason is reason | |
| def test_score_json_is_deterministic() -> None: | |
| gold = valid_drawing(b"gold") | |
| candidate = valid_drawing(b"candidate") | |
| first = compare_evaluated_drawings( | |
| task_id="task-001", candidate=candidate, gold=gold | |
| ).result.model_dump_json() | |
| second = compare_evaluated_drawings( | |
| task_id="task-001", candidate=candidate, gold=gold | |
| ).result.model_dump_json() | |
| assert first == second | |
| def test_semantic_geometry_ignores_translation_and_entity_segmentation() -> None: | |
| gold = _semantic_drawing( | |
| b"gold", | |
| paths=( | |
| _path((0, 0), (100, 0)), | |
| _path((100, 0), (100, 100)), | |
| _path((100, 100), (0, 100)), | |
| _path((0, 100), (0, 0)), | |
| ), | |
| ) | |
| candidate = _semantic_drawing( | |
| b"candidate", | |
| paths=( | |
| _path( | |
| (250, -40), | |
| (350, -40), | |
| (350, 60), | |
| (250, 60), | |
| closed=True, | |
| ), | |
| ), | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-001", candidate=candidate, gold=gold | |
| ).result | |
| assert result.scoring_method == "vector-semantic-v4" | |
| assert result.geometry_similarity == pytest.approx(1.0, abs=0.01) | |
| assert result.geometry_extent_similarity == pytest.approx(1.0) | |
| assert result.score == pytest.approx(1.0, abs=0.01) | |
| def test_semantic_geometry_penalizes_wrong_physical_scale() -> None: | |
| gold = _semantic_drawing( | |
| b"gold", | |
| paths=(_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),), | |
| ) | |
| candidate = _semantic_drawing( | |
| b"candidate", | |
| paths=(_path((0, 0), (200, 0), (200, 200), (0, 200), closed=True),), | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-001", candidate=candidate, gold=gold | |
| ).result | |
| assert result.geometry_extent_similarity == pytest.approx(0.5) | |
| assert result.geometry_similarity < 0.6 | |
| assert result.score < 0.8 | |
| def test_dimension_score_rewards_correct_values_and_penalizes_one_missing() -> None: | |
| values = (50, 30, 10, 20, 42, 144, 46, 9.78, 18.34, 44, 18) | |
| gold_dimensions = tuple( | |
| _dimension(value, index * 10.0, 130.0) for index, value in enumerate(values) | |
| ) | |
| candidate_dimensions = tuple( | |
| _dimension(value, 200.0 + index * 10.0, 80.0) | |
| for index, value in enumerate(values[:-1]) | |
| ) | |
| gold = _semantic_drawing( | |
| b"gold", | |
| paths=(_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),), | |
| dimensions=gold_dimensions, | |
| ) | |
| candidate = _semantic_drawing( | |
| b"candidate", | |
| paths=( | |
| _path((200, -50), (300, -50), (300, 50), (200, 50), closed=True), | |
| ), | |
| dimensions=candidate_dimensions, | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-001", candidate=candidate, gold=gold | |
| ).result | |
| assert result.matched_dimensions == 10 | |
| assert result.candidate_dimensions == 10 | |
| assert result.gold_dimensions == 11 | |
| assert result.dimension_similarity == pytest.approx(20 / 21) | |
| assert result.dimension_measurement_similarity == pytest.approx(20 / 21) | |
| assert result.dimension_placement_similarity == pytest.approx(20 / 21) | |
| assert not result.annotation_applicable | |
| assert result.annotation_similarity == 0.0 | |
| def test_visible_dimension_text_override_is_the_scored_value() -> None: | |
| geometry = (_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),) | |
| gold_dimension = _dimension(50, 50, 110) | |
| candidate_dimension = _dimension(80, 50, 110).model_copy( | |
| update={"text": "50"} | |
| ) | |
| gold = _semantic_drawing( | |
| b"gold", paths=geometry, dimensions=(gold_dimension,) | |
| ) | |
| candidate = _semantic_drawing( | |
| b"candidate", paths=geometry, dimensions=(candidate_dimension,) | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-001", candidate=candidate, gold=gold | |
| ).result | |
| assert result.dimension_measurement_similarity == pytest.approx(1.0) | |
| assert result.matched_dimensions == 1 | |
| def test_manual_dimension_text_is_equivalent_to_a_native_dimension() -> None: | |
| geometry = (_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),) | |
| gold = _semantic_drawing( | |
| b"gold", paths=geometry, dimensions=(_dimension(50, 50, 110),) | |
| ) | |
| candidate = _semantic_drawing(b"candidate", paths=geometry) | |
| candidate = candidate.model_copy( | |
| update={ | |
| "metadata": candidate.metadata.model_copy( | |
| update={"text_annotations": (_annotation("50", 50, 110),)} | |
| ) | |
| } | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-001", candidate=candidate, gold=gold | |
| ).result | |
| assert result.dimension_measurement_similarity == pytest.approx(1.0) | |
| assert result.matched_dimensions == 1 | |
| assert result.candidate_dimensions == 1 | |
| assert result.candidate_annotations == 0 | |
| def test_multiplicity_prefixed_manual_radius_is_parsed() -> None: | |
| geometry = (_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),) | |
| radial = _dimension(1.4, 50, 110).model_copy( | |
| update={"kind": "RadialDimension", "text": "2x<>"} | |
| ) | |
| gold = _semantic_drawing(b"gold", paths=geometry, dimensions=(radial,)) | |
| candidate = _semantic_drawing(b"candidate", paths=geometry) | |
| candidate = candidate.model_copy( | |
| update={ | |
| "metadata": candidate.metadata.model_copy( | |
| update={"text_annotations": (_annotation("2xR1.4", 50, 110),)} | |
| ) | |
| } | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-001", candidate=candidate, gold=gold | |
| ).result | |
| assert result.dimension_measurement_similarity == pytest.approx(1.0) | |
| assert result.matched_dimensions == 1 | |
| def test_leader_paths_are_annotation_not_part_geometry() -> None: | |
| geometry = (_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),) | |
| leader = _path((10, 10), (180, 180)).model_copy( | |
| update={"entity_type": "LEADER"} | |
| ) | |
| gold = _semantic_drawing(b"gold", paths=geometry) | |
| candidate = _semantic_drawing(b"candidate", paths=geometry + (leader,)) | |
| result = compare_evaluated_drawings( | |
| task_id="task-001", candidate=candidate, gold=gold | |
| ).result | |
| assert result.geometry_similarity == pytest.approx(1.0, abs=0.01) | |
| def test_isolated_centerlines_are_not_base_geometry() -> None: | |
| geometry = (_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),) | |
| centerlines = ( | |
| _path((-20, 50), (120, 50)).model_copy(update={"entity_type": "LINE"}), | |
| _path((50, -20), (50, 120)).model_copy(update={"entity_type": "LINE"}), | |
| ) | |
| gold = _semantic_drawing(b"gold", paths=geometry) | |
| candidate = _semantic_drawing(b"candidate", paths=geometry + centerlines) | |
| result = compare_evaluated_drawings( | |
| task_id="task-001", candidate=candidate, gold=gold | |
| ).result | |
| assert result.geometry_similarity == pytest.approx(1.0, abs=0.01) | |
| def test_explicit_no_dimension_required_note_allows_uniform_scale_alignment() -> None: | |
| gold = _semantic_drawing( | |
| b"gold", | |
| paths=(_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),), | |
| ) | |
| candidate = _semantic_drawing( | |
| b"candidate", | |
| paths=(_path((0, 0), (200, 0), (200, 200), (0, 200), closed=True),), | |
| ) | |
| note = (_annotation("No dimension required for the drawing", 20, -20),) | |
| gold = gold.model_copy( | |
| update={"metadata": gold.metadata.model_copy(update={"text_annotations": note})} | |
| ) | |
| candidate = candidate.model_copy( | |
| update={ | |
| "metadata": candidate.metadata.model_copy(update={"text_annotations": note}) | |
| } | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-021", candidate=candidate, gold=gold | |
| ).result | |
| assert result.geometry_similarity == pytest.approx(1.0, abs=0.01) | |
| assert result.score == pytest.approx(1.0, abs=0.01) | |
| def test_radically_different_raster_style_has_only_five_percent_weight() -> None: | |
| geometry = (_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),) | |
| gold = _semantic_drawing(b"gold", paths=geometry) | |
| black_render = line_png( | |
| tuple(((0, y), (127, y)) for y in range(128)), | |
| ) | |
| candidate = _semantic_drawing( | |
| b"candidate", paths=geometry, render_png=black_render | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-001", candidate=candidate, gold=gold | |
| ).result | |
| assert result.geometry_similarity == pytest.approx(1.0, abs=0.01) | |
| assert result.visual_similarity < 0.6 | |
| assert result.score >= 0.90 | |
| def test_absent_dimensions_and_annotations_are_not_free_credit() -> None: | |
| gold = _semantic_drawing( | |
| b"gold", | |
| paths=(_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),), | |
| ) | |
| candidate = _semantic_drawing( | |
| b"candidate", | |
| paths=(_path((0, 0), (200, 0), (200, 200), (0, 200), closed=True),), | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-021", candidate=candidate, gold=gold | |
| ).result | |
| assert not result.dimension_applicable | |
| assert not result.annotation_applicable | |
| assert result.dimension_similarity == 0.0 | |
| assert result.annotation_similarity == 0.0 | |
| weighted = ( | |
| 0.6 * result.geometry_similarity + 0.05 * result.visual_similarity | |
| ) / 0.65 | |
| geometry_gate = 0.35 + (0.65 * result.geometry_similarity) | |
| expected = min(weighted, geometry_gate) | |
| assert result.score == pytest.approx(expected) | |
| def test_standalone_text_annotations_are_scored_separately() -> None: | |
| geometry = (_path((0, 0), (100, 0), (100, 100), (0, 100), closed=True),) | |
| gold = _semantic_drawing(b"gold", paths=geometry) | |
| candidate = _semantic_drawing(b"candidate", paths=geometry) | |
| gold = gold.model_copy( | |
| update={ | |
| "metadata": gold.metadata.model_copy( | |
| update={"text_annotations": (_annotation("DETAIL A", 20, 20),)} | |
| ) | |
| } | |
| ) | |
| candidate = candidate.model_copy( | |
| update={ | |
| "metadata": candidate.metadata.model_copy( | |
| update={"text_annotations": (_annotation("WRONG NOTE", 20, 20),)} | |
| ) | |
| } | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-010", candidate=candidate, gold=gold | |
| ).result | |
| assert result.annotation_applicable | |
| assert result.annotation_similarity == 0.0 | |
| assert result.matched_annotations == 0 | |
| assert result.score < 1.0 | |
| def test_3d_scorer_uses_clean_spatial_bounds_and_layout1_features() -> None: | |
| gold_extents = SpatialExtents( | |
| min_x=-8.5, | |
| min_y=-8.5, | |
| min_z=0, | |
| max_x=8.5, | |
| max_y=8.5, | |
| max_z=12, | |
| ) | |
| candidate_extents = SpatialExtents( | |
| min_x=100, | |
| min_y=200, | |
| min_z=50, | |
| max_x=117, | |
| max_y=217, | |
| max_z=62, | |
| ) | |
| views = ( | |
| _viewport(2, center=(40, 40), direction=(0, 0, 10)), | |
| _viewport(3, center=(100, 40), direction=(0, -3, 0)), | |
| _viewport(4, center=(40, 100), direction=(1, -1, 1)), | |
| ) | |
| dimensions = (_dimension(17, 10, 10), _dimension(12, 20, 10)) | |
| annotations = (_annotation("DETAIL A", 10, 20),) | |
| gold = _three_d_drawing( | |
| b"gold", | |
| extents=gold_extents, | |
| viewports=views, | |
| dimensions=dimensions, | |
| annotations=annotations, | |
| ) | |
| candidate = _three_d_drawing( | |
| b"candidate", | |
| extents=candidate_extents, | |
| viewports=views, | |
| dimensions=dimensions, | |
| annotations=annotations, | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-030", candidate=candidate, gold=gold | |
| ).result | |
| assert result.scoring_method == "layout-spatial-3d-v2" | |
| assert result.geometry_extent_applicable | |
| assert result.geometry_extent_similarity == pytest.approx(1.0) | |
| assert result.geometry_similarity == pytest.approx(1.0) | |
| assert result.dimension_similarity == pytest.approx(1.0) | |
| assert result.annotation_similarity == pytest.approx(1.0) | |
| assert result.presentation_similarity == pytest.approx(1.0) | |
| assert result.score == pytest.approx(1.0) | |
| def test_3d_visual_comparison_normalizes_sheet_framing() -> None: | |
| extents = SpatialExtents( | |
| min_x=0, | |
| min_y=0, | |
| min_z=0, | |
| max_x=100, | |
| max_y=80, | |
| max_z=40, | |
| ) | |
| views = ( | |
| _viewport(2, center=(40, 40), direction=(0, 0, 1)), | |
| _viewport(3, center=(100, 40), direction=(1, -1, 1)), | |
| ) | |
| gold_render = line_png( | |
| ( | |
| ((20, 20), (108, 20)), | |
| ((108, 20), (108, 108)), | |
| ((108, 108), (20, 108)), | |
| ((20, 108), (20, 20)), | |
| ) | |
| ) | |
| candidate_render = line_png( | |
| ( | |
| ((40, 40), (90, 40)), | |
| ((90, 40), (90, 90)), | |
| ((90, 90), (40, 90)), | |
| ((40, 90), (40, 40)), | |
| ) | |
| ) | |
| gold = _three_d_drawing( | |
| b"gold", extents=extents, viewports=views, render_png=gold_render | |
| ) | |
| candidate = _three_d_drawing( | |
| b"candidate", | |
| extents=extents, | |
| viewports=views, | |
| render_png=candidate_render, | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-022", candidate=candidate, gold=gold | |
| ).result | |
| assert result.visual_similarity == pytest.approx(1.0) | |
| assert result.score == pytest.approx(1.0) | |
| def test_3d_presentation_cannot_rescue_wrong_geometry() -> None: | |
| extents = SpatialExtents( | |
| min_x=0, | |
| min_y=0, | |
| min_z=0, | |
| max_x=100, | |
| max_y=80, | |
| max_z=40, | |
| ) | |
| views = ( | |
| _viewport(2, center=(40, 40), direction=(0, 0, 1)), | |
| _viewport(3, center=(100, 40), direction=(1, -1, 1)), | |
| ) | |
| gold = _three_d_drawing( | |
| b"gold", | |
| extents=extents, | |
| viewports=views, | |
| render_png=line_png( | |
| ( | |
| ((20, 20), (108, 20)), | |
| ((108, 20), (108, 108)), | |
| ((108, 108), (20, 108)), | |
| ((20, 108), (20, 20)), | |
| ) | |
| ), | |
| ) | |
| candidate = _three_d_drawing( | |
| b"candidate", | |
| extents=extents, | |
| viewports=views, | |
| render_png=line_png((((20, 20), (108, 108)),)), | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-022", candidate=candidate, gold=gold | |
| ).result | |
| assert result.presentation_similarity == pytest.approx(1.0) | |
| assert result.score <= 0.35 + (0.65 * result.geometry_similarity) | |
| assert result.score < 0.7 | |
| def test_3d_schema_v3_polluted_bounds_are_non_applicable() -> None: | |
| polluted = SpatialExtents( | |
| min_x=-10000, | |
| min_y=-10000, | |
| min_z=0, | |
| max_x=17, | |
| max_y=17, | |
| max_z=12, | |
| ) | |
| clean = SpatialExtents( | |
| min_x=-8.5, | |
| min_y=-8.5, | |
| min_z=0, | |
| max_x=8.5, | |
| max_y=8.5, | |
| max_z=12, | |
| ) | |
| views = (_viewport(2, center=(40, 40), direction=(0, 0, 1)),) | |
| gold = _three_d_drawing( | |
| b"gold", extents=polluted, viewports=views, schema_version="3" | |
| ) | |
| candidate = _three_d_drawing( | |
| b"candidate", extents=clean, viewports=views, schema_version="3" | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-026", candidate=candidate, gold=gold | |
| ).result | |
| assert not result.geometry_extent_applicable | |
| assert result.geometry_extent_similarity == 0.0 | |
| assert result.geometry_similarity == pytest.approx(1.0) | |
| assert result.score == pytest.approx(1.0) | |
| def test_3d_copied_views_without_true_geometry_cannot_earn_geometry_credit() -> None: | |
| extents = SpatialExtents( | |
| min_x=0, | |
| min_y=0, | |
| min_z=0, | |
| max_x=100, | |
| max_y=80, | |
| max_z=40, | |
| ) | |
| views = (_viewport(2, center=(40, 40), direction=(0, 0, 1)),) | |
| gold = _three_d_drawing(b"gold", extents=extents, viewports=views) | |
| candidate = _three_d_drawing( | |
| b"candidate", | |
| extents=extents, | |
| viewports=views, | |
| three_d_entities=0, | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-049", candidate=candidate, gold=gold | |
| ).result | |
| assert result.visual_similarity == pytest.approx(1.0) | |
| assert result.geometry_similarity == 0.0 | |
| assert result.score < 0.5 | |
| def test_3d_native_drawing_view_support_viewports_are_deduplicated() -> None: | |
| extents = SpatialExtents( | |
| min_x=0, | |
| min_y=0, | |
| min_z=0, | |
| max_x=100, | |
| max_y=80, | |
| max_z=40, | |
| ) | |
| gold_views = ( | |
| _viewport(2, center=(40, 40), direction=(0, 0, 1)), | |
| _viewport(3, center=(40, 40), direction=(0, -1, 0)), | |
| _viewport(4, center=(100, 40), direction=(0, 0, 1)), | |
| _viewport(5, center=(100, 40), direction=(1, -1, 1)), | |
| ) | |
| candidate_views = ( | |
| _viewport(2, center=(20, 20), direction=(0, -2, 0)), | |
| _viewport(3, center=(80, 20), direction=(10, -10, 10)), | |
| ) | |
| gold = _three_d_drawing(b"gold", extents=extents, viewports=gold_views) | |
| candidate = _three_d_drawing( | |
| b"candidate", extents=extents, viewports=candidate_views | |
| ) | |
| result = compare_evaluated_drawings( | |
| task_id="task-026", candidate=candidate, gold=gold | |
| ).result | |
| assert result.gold_views == 2 | |
| assert result.candidate_views == 2 | |
| assert result.view_direction_similarity == pytest.approx(1.0) | |