"""Build the public 50-task AutoCAD Bench Hugging Face dataset payload.""" from __future__ import annotations import argparse import hashlib import json import shutil from collections import Counter from pathlib import Path from autocad_bench.tasks.manifest import load_manifest from autocad_bench.tasks.models import prompt_for_drawing_type EXPECTED_TASK_COUNT = 50 EXPECTED_DRAWING_TYPES = {"2d": 21, "3d": 29} def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def write_text(path: Path, value: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(value.rstrip() + "\n", encoding="utf-8") def dataset_card() -> str: return """--- license: cc-by-4.0 language: - en pretty_name: AutoCAD-Bench Tasks task_categories: - robotics size_categories: - n<1K tags: - autocad - cad - computer-use - gui-agent - benchmark --- # AutoCAD-Bench Tasks This dataset contains the **50 AutoCAD tasks used for model evaluation** in AutoCAD-Bench. ## Contents - 50 total evaluation tasks - 21 dimensioned 2D drafting tasks - 29 3D modeling and presentation tasks - millimetres as the task unit - one reference PNG and one scorer-side gold DWG per task - the exact 2D and 3D task prompts used by the benchmark Each task is stored under `data/task-NNN/`: ```text data/task-001/ ├── reference.png └── gold.dwg ``` `tasks.jsonl` records the sequential task ID, drawing type, difficulty level, units, task prompt, artifact paths, file sizes, and SHA-256 checksums. ## Evaluation boundary The model should receive only the task prompt and `reference.png`. The gold DWG is an evaluator-side artifact and must never be mounted in the model-controlled desktop, included in the model prompt, or otherwise made available to the agent during a rollout. For 2D tasks, the expected submission is a complete dimensioned drawing in Model Space. For 3D tasks, the expected submission contains genuine 3D geometry in Model Space and the complete reference-sheet presentation on `Layout1`. ## Loading the manifest ```python import json from pathlib import Path tasks = [ json.loads(line) for line in Path("tasks.jsonl").read_text().splitlines() if line.strip() ] ``` The DWG files use the AutoCAD 2018 `AC1032` file format. """ def export_dataset(*, benchmark_root: Path, output_dir: Path) -> None: if output_dir.exists(): raise ValueError(f"output directory already exists: {output_dir}") suite_path = benchmark_root / "tasks" / "benchmark-suite-50.txt" task_ids = [ line.strip() for line in suite_path.read_text(encoding="utf-8").splitlines() if line.strip() and not line.lstrip().startswith("#") ] if len(task_ids) != EXPECTED_TASK_COUNT or len(set(task_ids)) != len(task_ids): raise ValueError("canonical suite must contain 50 unique task IDs") entries = {entry.task_id: entry for entry in load_manifest()} missing = [task_id for task_id in task_ids if task_id not in entries] if missing: raise ValueError(f"suite contains unknown tasks: {missing}") selected = [entries[task_id] for task_id in task_ids] drawing_types = Counter(entry.drawing_type for entry in selected) if dict(drawing_types) != EXPECTED_DRAWING_TYPES: raise ValueError( f"unexpected drawing-type counts: {dict(drawing_types)}" ) output_dir.mkdir(parents=True) write_text(output_dir / "README.md", dataset_card()) write_text( output_dir / "prompts" / "2d.md", prompt_for_drawing_type("2d"), ) write_text( output_dir / "prompts" / "3d.md", prompt_for_drawing_type("3d"), ) manifest_rows: list[dict[str, object]] = [] for number, entry in enumerate(selected, start=1): public_id = f"task-{number:03d}" task_dir = output_dir / "data" / public_id task_dir.mkdir(parents=True) source_reference = entry.resolve_image_path() source_gold = entry.resolve_gold_path() target_reference = task_dir / "reference.png" target_gold = task_dir / "gold.dwg" shutil.copyfile(source_reference, target_reference) shutil.copyfile(source_gold, target_gold) if target_gold.read_bytes()[:6] != b"AC1032": raise ValueError(f"{public_id} has an unsupported DWG signature") manifest_rows.append( { "task_id": number, "task_name": public_id, "split": "evaluation", "drawing_type": entry.drawing_type, "level": entry.level, "units": entry.units, "time_limit_s": entry.time_limit_s, "max_actions": 5000, "prompt_path": f"prompts/{entry.drawing_type}.md", "reference_image": f"data/{public_id}/reference.png", "reference_sha256": sha256_file(target_reference), "reference_bytes": target_reference.stat().st_size, "gold_dwg": f"data/{public_id}/gold.dwg", "gold_sha256": sha256_file(target_gold), "gold_bytes": target_gold.stat().st_size, } ) write_text( output_dir / "tasks.jsonl", "\n".join(json.dumps(row, sort_keys=True) for row in manifest_rows), ) checksum_paths = sorted( path for path in output_dir.rglob("*") if path.is_file() and path.name != "checksums.sha256" ) checksum_lines = [ f"{sha256_file(path)} {path.relative_to(output_dir).as_posix()}" for path in checksum_paths ] write_text(output_dir / "checksums.sha256", "\n".join(checksum_lines)) def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("output_dir", type=Path) parser.add_argument( "--benchmark-root", type=Path, default=Path(__file__).resolve().parents[1], ) args = parser.parse_args() export_dataset( benchmark_root=args.benchmark_root.resolve(), output_dir=args.output_dir.resolve(), ) return 0 if __name__ == "__main__": raise SystemExit(main())