Download methods/clip_based.py from addy0606/zsad: direct link, hf CLI and curl.
- Browser
- Download file 4.16 kB
-
https://huggingface.co/spaces/addy0606/zsad/resolve/main/methods/clip_based.py
- Command line
-
hf download hf://spaces/addy0606/zsad/methods/clip_based.py
-
curl -L -o clip_based.py https://huggingface.co/spaces/addy0606/zsad/resolve/main/methods/clip_based.py
4.16 kB
| """CLIP-based family: AnomalyCLIP (Zhou et al., ICLR 2024), wrapping the official code (MIT). | |
| Learned object-agnostic text prompts ("normal [object]" / "damaged [object]") are matched against | |
| CLIP ViT-L/14@336 patch features. Prompts were trained on one industrial dataset's test set: | |
| the MVTec-trained checkpoint is used for every target except MVTec AD itself, which uses the | |
| VisA-trained one (the paper's cross-dataset protocol). | |
| Matches the official test.py: 518 px input, layers 6/12/18/24 (configurable), DPAM from layer 20, | |
| image score = softmax probability of the "anomalous" prompt for the global image embedding, | |
| map = sum over layers of patch-prompt similarity, Gaussian-smoothed with sigma 4. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import numpy as np | |
| import torch | |
| from scipy.ndimage import gaussian_filter | |
| from methods.base import ZeroShotDetector, resize_map, to_pil, use_third_party | |
| PROMPT_PARAMS = { | |
| "Prompt_length": 12, | |
| "learnabel_text_embedding_depth": 9, | |
| "learnabel_text_embedding_length": 4, | |
| } | |
| class AnomalyCLIP(ZeroShotDetector): | |
| name = "anomalyclip" | |
| family = "clip" | |
| def __init__( | |
| self, | |
| checkpoint: str, | |
| image_size: int = 518, | |
| sigma: float = 4.0, | |
| features_list: tuple[int, ...] = (6, 12, 18, 24), | |
| clip_cache: str | None = None, | |
| device: str = "cuda", | |
| ) -> None: | |
| """features_list: ViT layers whose patch features are scored. The paper and test.py default use | |
| 6/12/18/24; the repo's test.sh passes only 24. Kept configurable (see PROGRESS.md, Phase 2).""" | |
| super().__init__(device) | |
| self.checkpoint, self.image_size, self.sigma, self.clip_cache = ( | |
| checkpoint, | |
| image_size, | |
| sigma, | |
| clip_cache, | |
| ) | |
| self.features_list = list(features_list) | |
| def setup(self, device: str) -> None: | |
| self.device = device | |
| use_third_party("AnomalyCLIP") | |
| import AnomalyCLIP_lib | |
| from prompt_ensemble import AnomalyCLIP_PromptLearner | |
| from utils import get_transform | |
| self._lib = AnomalyCLIP_lib | |
| model, _ = AnomalyCLIP_lib.load( | |
| "ViT-L/14@336px", device=device, design_details=PROMPT_PARAMS, download_root=self.clip_cache | |
| ) | |
| model.eval() | |
| self.preprocess, _ = get_transform(argparse.Namespace(image_size=self.image_size)) | |
| learner = AnomalyCLIP_PromptLearner(model.to("cpu"), PROMPT_PARAMS) | |
| learner.load_state_dict(torch.load(self.checkpoint, map_location="cpu")["prompt_learner"]) | |
| learner.to(device) | |
| model.to(device) | |
| model.visual.DAPM_replace(DPAM_layer=20) | |
| with torch.no_grad(): # text features don't depend on the image: compute once | |
| prompts, tokenized, compound = learner(cls_id=None) | |
| text = model.encode_text_learn(prompts, tokenized, compound).float() | |
| text = torch.stack(torch.chunk(text, dim=0, chunks=2), dim=1) | |
| self.text_features = text / text.norm(dim=-1, keepdim=True) | |
| self.model = model | |
| def predict(self, image: torch.Tensor, category: str) -> tuple[float, np.ndarray]: | |
| x = self.preprocess(to_pil(image)).unsqueeze(0).to(self.device) | |
| img_feat, patch_feats = self.model.encode_image(x, self.features_list, DPAM_layer=20) | |
| img_feat = img_feat / img_feat.norm(dim=-1, keepdim=True) | |
| probs = ((img_feat @ self.text_features.permute(0, 2, 1)) / 0.07).softmax(-1) | |
| score = float(probs[0, 0, 1]) | |
| maps = [] | |
| for pf in patch_feats: | |
| pf = pf / pf.norm(dim=-1, keepdim=True) | |
| sim, _ = self._lib.compute_similarity(pf, self.text_features[0]) | |
| sim_map = self._lib.get_similarity_map(sim[:, 1:, :], self.image_size) | |
| maps.append((sim_map[..., 1] + 1 - sim_map[..., 0]) / 2.0) | |
| amap = torch.stack(maps).sum(0)[0].cpu().numpy() | |
| amap = gaussian_filter(amap, sigma=self.sigma) | |
| return score, resize_map(amap, tuple(image.shape[1:])) | |