File size: 4,667 Bytes
ba35c67
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
"""Predict AI Worker arm masks with the native SAM3 image model."""

import argparse
import hashlib
import json
from pathlib import Path

import numpy as np
import torch
from huggingface_hub import hf_hub_download
from PIL import Image


CONFIG_PATH = Path(__file__).with_name("inference_config.json")


def _activation(activation, linear, values):
    """Use the same unfused activation path as the fine-tuned checkpoint."""
    if activation in (torch.nn.GELU, torch.nn.functional.gelu):
        return torch.nn.functional.gelu(linear(values))
    if activation in (torch.nn.ReLU, torch.nn.functional.relu):
        return torch.nn.functional.relu(linear(values))
    raise ValueError(f"Unsupported activation: {activation}")


def load_predictor(checkpoint=None):
    """Load the release checkpoint and its fixed 504px inference settings."""
    from sam3.model import vitdet
    from sam3.model.sam3_image_processor import Sam3Processor
    from sam3.model_builder import build_sam3_image_model

    if not torch.cuda.is_available() or not torch.cuda.is_bf16_supported():
        raise RuntimeError("A BF16-capable CUDA GPU is required. See README.md.")
    config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
    if checkpoint is None:
        checkpoint = hf_hub_download(config["model_id"], config["checkpoint"])
    checkpoint = Path(checkpoint)
    with checkpoint.open("rb") as stream:
        actual = hashlib.file_digest(stream, "sha256").hexdigest()
    if actual != config["checkpoint_sha256"]:
        raise ValueError("Checkpoint checksum mismatch. Download this release again.")

    torch.set_num_threads(1)
    vitdet.addmm_act = _activation
    model = build_sam3_image_model(
        checkpoint_path=str(checkpoint), load_from_HF=False,
        device="cpu", eval_mode=True, enable_inst_interactivity=False,
    )
    trunk = model.backbone.vision_backbone.trunk
    grid = config["resolution"] // trunk.patch_embed.proj.kernel_size[0]
    for block in trunk.blocks:
        attention = block.attn
        if block.window_size > 0 or attention.freqs_cis is None:
            continue
        attention.input_size = (grid, grid)
        scale = attention.rope_pt_size[0] / grid if attention.rope_interp else 1.0
        frequencies = attention.compute_cis(end_x=grid, end_y=grid, scale_pos=scale)
        attention.freqs_cis = frequencies
        if attention.use_rope_real:
            attention.freqs_cis_real = frequencies.real
            attention.freqs_cis_imag = frequencies.imag
    torch.set_num_threads(4)
    torch.backends.cuda.matmul.allow_tf32 = True
    torch.manual_seed(438)
    model = model.cuda().eval()
    processor = Sam3Processor(
        model, resolution=config["resolution"], device="cuda",
        confidence_threshold=config["confidence_threshold"],
    )
    return model, processor, config


@torch.inference_mode()
def predict_image(image, processor, config):
    """Return native-resolution instance masks, XYXY boxes and scores."""
    with torch.autocast("cuda", dtype=torch.bfloat16):
        state = processor.set_image(image.convert("RGB"))
        result = processor.set_text_prompt(prompt=config["prompt"], state=state)
    return (
        result["masks"][:, 0].cpu().numpy(),
        result["boxes"].float().cpu().numpy(),
        result["scores"].float().cpu().numpy(),
    )


def save_outputs(directory, masks, boxes, scores):
    """Save a white foreground union mask and all individual predictions."""
    directory = Path(directory)
    directory.mkdir(parents=True, exist_ok=True)
    Image.fromarray(masks.any(axis=0).astype(np.uint8) * 255).save(directory / "mask.png")
    np.savez_compressed(directory / "instances.npz", masks=masks, boxes_xyxy=boxes, scores=scores)


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("image", type=Path, help="Input RGB image")
    parser.add_argument("--checkpoint", type=Path, help="Local model.pt; otherwise download it")
    parser.add_argument("--output-dir", type=Path, default=Path("prediction"))
    args = parser.parse_args()
    if args.image.resolve() in {(args.output_dir / name).resolve() for name in ["mask.png", "instances.npz"]}:
        parser.error("The output directory would overwrite the input image.")
    with Image.open(args.image) as source:
        image = source.convert("RGB")
    model, processor, config = load_predictor(args.checkpoint)
    masks, boxes, scores = predict_image(image, processor, config)
    save_outputs(args.output_dir, masks, boxes, scores)
    print(f"Saved {len(scores)} predicted instances to {args.output_dir}")


if __name__ == "__main__":
    main()