zsad / methods /clip_based.py
addy0606's picture
Update demo
98f8023 verified
Raw History Blame Contribute Delete
4.16 kB
"""CLIP-based family: AnomalyCLIP (Zhou et al., ICLR 2024), wrapping the official code (MIT).
Learned object-agnostic text prompts ("normal [object]" / "damaged [object]") are matched against
CLIP ViT-L/14@336 patch features. Prompts were trained on one industrial dataset's test set:
the MVTec-trained checkpoint is used for every target except MVTec AD itself, which uses the
VisA-trained one (the paper's cross-dataset protocol).
Matches the official test.py: 518 px input, layers 6/12/18/24 (configurable), DPAM from layer 20,
image score = softmax probability of the "anomalous" prompt for the global image embedding,
map = sum over layers of patch-prompt similarity, Gaussian-smoothed with sigma 4.
"""
from __future__ import annotations
import argparse
import numpy as np
import torch
from scipy.ndimage import gaussian_filter
from methods.base import ZeroShotDetector, resize_map, to_pil, use_third_party
PROMPT_PARAMS = {
"Prompt_length": 12,
"learnabel_text_embedding_depth": 9,
"learnabel_text_embedding_length": 4,
}
class AnomalyCLIP(ZeroShotDetector):
name = "anomalyclip"
family = "clip"
def __init__(
self,
checkpoint: str,
image_size: int = 518,
sigma: float = 4.0,
features_list: tuple[int, ...] = (6, 12, 18, 24),
clip_cache: str | None = None,
device: str = "cuda",
) -> None:
"""features_list: ViT layers whose patch features are scored. The paper and test.py default use
6/12/18/24; the repo's test.sh passes only 24. Kept configurable (see PROGRESS.md, Phase 2)."""
super().__init__(device)
self.checkpoint, self.image_size, self.sigma, self.clip_cache = (
checkpoint,
image_size,
sigma,
clip_cache,
)
self.features_list = list(features_list)
def setup(self, device: str) -> None:
self.device = device
use_third_party("AnomalyCLIP")
import AnomalyCLIP_lib
from prompt_ensemble import AnomalyCLIP_PromptLearner
from utils import get_transform
self._lib = AnomalyCLIP_lib
model, _ = AnomalyCLIP_lib.load(
"ViT-L/14@336px", device=device, design_details=PROMPT_PARAMS, download_root=self.clip_cache
)
model.eval()
self.preprocess, _ = get_transform(argparse.Namespace(image_size=self.image_size))
learner = AnomalyCLIP_PromptLearner(model.to("cpu"), PROMPT_PARAMS)
learner.load_state_dict(torch.load(self.checkpoint, map_location="cpu")["prompt_learner"])
learner.to(device)
model.to(device)
model.visual.DAPM_replace(DPAM_layer=20)
with torch.no_grad(): # text features don't depend on the image: compute once
prompts, tokenized, compound = learner(cls_id=None)
text = model.encode_text_learn(prompts, tokenized, compound).float()
text = torch.stack(torch.chunk(text, dim=0, chunks=2), dim=1)
self.text_features = text / text.norm(dim=-1, keepdim=True)
self.model = model
@torch.no_grad()
def predict(self, image: torch.Tensor, category: str) -> tuple[float, np.ndarray]:
x = self.preprocess(to_pil(image)).unsqueeze(0).to(self.device)
img_feat, patch_feats = self.model.encode_image(x, self.features_list, DPAM_layer=20)
img_feat = img_feat / img_feat.norm(dim=-1, keepdim=True)
probs = ((img_feat @ self.text_features.permute(0, 2, 1)) / 0.07).softmax(-1)
score = float(probs[0, 0, 1])
maps = []
for pf in patch_feats:
pf = pf / pf.norm(dim=-1, keepdim=True)
sim, _ = self._lib.compute_similarity(pf, self.text_features[0])
sim_map = self._lib.get_similarity_map(sim[:, 1:, :], self.image_size)
maps.append((sim_map[..., 1] + 1 - sim_map[..., 0]) / 2.0)
amap = torch.stack(maps).sum(0)[0].cpu().numpy()
amap = gaussian_filter(amap, sigma=self.sigma)
return score, resize_map(amap, tuple(image.shape[1:]))