| from __future__ import annotations |
|
|
| from autocad_bench.tasks.manifest import MAX_ROLLOUT_ACTIONS |
| from autocad_bench.tasks.models import AutoCADTaskData |
| from autocad_bench.tasks.taskset import AutoCADTaskset, AutoCADTasksetConfig |
|
|
|
|
| def test_taskset_loads_seven_typed_development_tasks() -> None: |
| tasks = AutoCADTaskset(AutoCADTasksetConfig(split="dev")).load() |
|
|
| assert len(tasks) == 7 |
| assert all(isinstance(task.data, AutoCADTaskData) for task in tasks) |
| assert [task.data.task_id for task in tasks] == [ |
| "task-004", |
| "task-010", |
| "task-016", |
| "task-023", |
| "task-029", |
| "task-035", |
| "task-043", |
| ] |
|
|
|
|
| def test_task_data_contains_prompt_image_and_budgets() -> None: |
| task = AutoCADTaskset(AutoCADTasksetConfig(split="dev")).load()[0] |
|
|
| assert task.data.prompt_text.startswith("Recreate the supplied") |
| assert "Revisit the reference at any time" in task.data.prompt_text |
| assert "complete drawing content" in task.data.prompt_text |
| assert "dimensions or annotations" in task.data.prompt_text |
| assert task.data.image.endswith("task-004/reference.png") |
| assert task.data.time_limit_s == 900 |
| assert task.data.max_actions == 7_000 |
| assert min(task.data.max_actions, MAX_ROLLOUT_ACTIONS) == 5_000 |
| assert task.data.timeout.harness == 900.0 |
|
|
|
|
| def test_3d_task_prompt_requires_true_model_and_presentation_sheet() -> None: |
| tasks = AutoCADTaskset(AutoCADTasksetConfig(split="dev")).load() |
| task = next(task for task in tasks if task.data.task_id == "task-023") |
|
|
| assert task.data.drawing_type == "3d" |
| assert "complete 3D AutoCAD model and presentation drawing" in task.data.prompt_text |
| assert "complementary evidence for one 3D part or assembly" in task.data.prompt_text |
| assert "MODEL SPACE" in task.data.prompt_text |
| assert "PRESENTATION LAYOUT" in task.data.prompt_text |
| assert "PRESENTATION LAYOUT (`Layout1`)" in task.data.prompt_text |
| assert "every isometric, orthographic, section, and detail view" in ( |
| task.data.prompt_text |
| ) |
| assert "Do not put the required presentation on `Layout2`" in task.data.prompt_text |
| assert "Leave `Layout1` active" in task.data.prompt_text |
| assert task.data.time_limit_s == 1800 |
| assert task.data.max_actions == 7_000 |
|
|
|
|
| def test_task_data_serialization_does_not_leak_gold() -> None: |
| tasks = AutoCADTaskset(AutoCADTasksetConfig(split="all")).load() |
|
|
| assert len(tasks) == 50 |
| for task in tasks: |
| serialized = task.data.model_dump_json() |
| assert "gold" not in serialized.lower() |
| assert ".dwg" not in serialized.lower() |
|
|
