example.py

"""Run box-prompted segmentation with EfficientSAM-S and ExecuTorch."""

import argparse
import json
from pathlib import Path

import numpy as np
import torch
from executorch.runtime import Runtime
from PIL import Image, ImageDraw
from torchvision import transforms
from torchvision.transforms import InterpolationMode


MODEL_DIR = Path(__file__).resolve().parent
MODEL_PATH = MODEL_DIR / "efficient_sam_s_graviton_executorch_optimized.pte"
SAMPLE_IMAGE_PATH = MODEL_DIR / "sample_input.jpg"

INPUT_SIZE = 1024
SAMPLE_BOX_PROMPT = [410, 510, 600, 830]


def required_file(path: Path) -> Path:
    if not path.is_file():
        raise FileNotFoundError(f"required model file not found: {path}")
    return path


def load_model() -> object:
    runtime = Runtime.get()
    if not runtime.backend_registry.is_available("XnnpackBackend"):
        available = ", ".join(runtime.backend_registry.registered_backend_names) or "none"
        raise RuntimeError(f"XnnpackBackend is unavailable; registered backends: {available}")

    program = runtime.load_program(required_file(MODEL_PATH))
    if program.method_names != {"forward"}:
        raise RuntimeError(f"expected PTE method 'forward', found: {sorted(program.method_names)}")
    return program.load_method("forward")


def preprocess(image_path: Path) -> torch.Tensor:
    transform = transforms.Compose(
        [
            transforms.Resize(
                (INPUT_SIZE, INPUT_SIZE),
                interpolation=InterpolationMode.BILINEAR,
                antialias=True,
            ),
            transforms.ToTensor(),
        ]
    )
    with Image.open(required_file(image_path)) as image:
        return transform(image.convert("RGB")).unsqueeze(0).contiguous()


def make_box_prompt(box: list[int]) -> tuple[torch.Tensor, torch.Tensor]:
    x1, y1, x2, y2 = box
    point_coords = torch.tensor([[[[x1, y1], [x2, y2]]]], dtype=torch.float32)
    point_labels = torch.tensor([[[2, 3]]], dtype=torch.float32)
    return point_coords, point_labels


def validate_box(box: list[int]) -> list[int]:
    x1, y1, x2, y2 = box
    if not (0 <= x1 < x2 <= INPUT_SIZE and 0 <= y1 < y2 <= INPUT_SIZE):
        raise ValueError(
            f"box must satisfy 0 <= x1 < x2 <= {INPUT_SIZE} and "
            f"0 <= y1 < y2 <= {INPUT_SIZE}; received {box}"
        )
    return box


def postprocess(raw_output: torch.Tensor, box: list[int]) -> tuple[np.ndarray, dict[str, object]]:
    if tuple(raw_output.shape) != (1, INPUT_SIZE, INPUT_SIZE):
        raise ValueError(
            f"expected output shape (1, {INPUT_SIZE}, {INPUT_SIZE}), "
            f"found {tuple(raw_output.shape)}"
        )
    if not torch.isfinite(raw_output).all():
        raise ValueError("model output contains non-finite mask values")

    mask = (raw_output.squeeze(0) > 0.5).cpu().numpy().astype(np.uint8)
    pixel_count = int(mask.sum())
    total_pixels = int(mask.size)
    result = {
        "pixel_count": pixel_count,
        "total_pixels": total_pixels,
        "coverage_pct": round(100.0 * pixel_count / total_pixels, 2),
        "box_prompt": box,
    }
    return mask, result


def create_overlay(mask: np.ndarray, image_path: Path, box: list[int]) -> Image.Image:
    with Image.open(required_file(image_path)) as image:
        source = image.convert("RGB")

    source_width, source_height = source.size
    source_mask = Image.fromarray(mask * 255).resize(
        source.size,
        resample=Image.Resampling.NEAREST,
    )
    source_mask_array = np.asarray(source_mask, dtype=np.uint8) > 0

    mask_rgba = np.zeros((source_height, source_width, 4), dtype=np.uint8)
    mask_rgba[source_mask_array] = [0, 200, 0, 200]
    overlay = Image.fromarray(mask_rgba)
    composited = Image.alpha_composite(source.convert("RGBA"), overlay).convert("RGB")

    x1, y1, x2, y2 = box
    scaled_box = [
        round(x1 * source_width / INPUT_SIZE),
        round(y1 * source_height / INPUT_SIZE),
        round(x2 * source_width / INPUT_SIZE),
        round(y2 * source_height / INPUT_SIZE),
    ]
    draw = ImageDraw.Draw(composited)
    draw.rectangle(scaled_box, outline=(255, 0, 0), width=3)
    return composited


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--image",
        type=Path,
        default=SAMPLE_IMAGE_PATH,
        help="input image (default: sample_input.jpg)",
    )
    parser.add_argument(
        "--box",
        nargs=4,
        type=int,
        metavar=("X1", "Y1", "X2", "Y2"),
        default=SAMPLE_BOX_PROMPT,
        help="box prompt in resized 1024x1024 coordinates (default: sample box)",
    )
    parser.add_argument("--output-dir", type=Path, help="optional output directory")
    return parser.parse_args()


def main() -> None:
    args = parse_args()
    image_path = required_file(args.image)
    box = validate_box(args.box)
    image_tensor = preprocess(image_path)
    point_coords, point_labels = make_box_prompt(box)
    method = load_model()
    raw_output = method.execute([image_tensor, point_coords, point_labels])[0]
    mask, result = postprocess(raw_output, box)

    print(f"Input: {image_path.name}; tensor shape: {tuple(image_tensor.shape)}")
    print(f"Box prompt: {box}")
    print(
        f"Segmented pixels: {result['pixel_count']:,} / {result['total_pixels']:,} "
        f"({result['coverage_pct']:.2f}%)"
    )

    if args.output_dir is not None:
        args.output_dir.mkdir(parents=True, exist_ok=True)
        overlay_path = args.output_dir / "sample_output.png"
        result_path = args.output_dir / "segmentation.json"
        create_overlay(mask, image_path, box).save(overlay_path)
        result_path.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
        print(f"Saved: {overlay_path}")
        print(f"Saved: {result_path}")


if __name__ == "__main__":
    main()