Download engine.py from m4az19/saef-api: direct link, hf CLI and curl.
- Browser
- Download file 8.29 kB
-
https://huggingface.co/spaces/m4az19/saef-api/resolve/main/engine.py
- Command line
-
hf download hf://spaces/m4az19/saef-api/engine.py
-
curl -L -o engine.py https://huggingface.co/spaces/m4az19/saef-api/resolve/main/engine.py
8.29 kB
| """Official full-precision image encoder with cached prompted text embeddings.""" | |
| from __future__ import annotations | |
| import copy | |
| import gc | |
| import hashlib | |
| import io | |
| import json | |
| import os | |
| import threading | |
| import time | |
| import warnings | |
| from pathlib import Path | |
| os.environ.setdefault("USE_TF", "0") | |
| os.environ.setdefault("USE_FLAX", "0") | |
| os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") | |
| import numpy as np | |
| import open_clip | |
| import torch | |
| from PIL import Image, ImageOps, UnidentifiedImageError | |
| from download_model import DEFAULT_DIR, MODEL_ID, MODEL_REVISION, ensure_model | |
| ROOT = Path(__file__).resolve().parent | |
| Image.MAX_IMAGE_PIXELS = 16000000 | |
| class BiomedEngine: | |
| def __init__(self, model_dir: Path | None = None, labels_path: Path | None = None): | |
| self.lock = threading.Lock() | |
| self.labels = json.loads((labels_path or ROOT / "labels.json").read_text("utf-8")) | |
| if (not self.labels or len({row["id"] for row in self.labels}) != len(self.labels) | |
| or not any(row["is_background"] for row in self.labels) | |
| or any(not row["prompts"] for row in self.labels)): | |
| raise ValueError("Labels must be unique and include prompts and background entries.") | |
| self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| torch.set_num_threads(max(1, int(os.environ.get("TORCH_THREADS", "2")))) | |
| if self.device.type == "cuda": | |
| # Explicit FP32 comparison: do not enable AMP, FP16, INT8 or TF32. | |
| torch.backends.cuda.matmul.allow_tf32 = False | |
| torch.backends.cudnn.allow_tf32 = False | |
| directory = (model_dir or Path(os.environ.get("MODEL_DIR", DEFAULT_DIR))).resolve() | |
| ensure_model(directory) | |
| config = json.loads((directory / "open_clip_config.json").read_text("utf-8")) | |
| cfg = copy.deepcopy(config["model_cfg"]) | |
| cfg["text_cfg"]["hf_model_name"] = str(directory) | |
| cfg["text_cfg"]["hf_tokenizer_name"] = str(directory) | |
| from open_clip.factory import _MODEL_CONFIGS | |
| name = "saef_biomedclip_full_precision" | |
| _MODEL_CONFIGS[name] = cfg | |
| model, _, self.preprocess = open_clip.create_model_and_transforms( | |
| name, pretrained=str(directory / "open_clip_pytorch_model.bin"), | |
| precision="fp32", device="cpu", | |
| **{f"image_{key}": value for key, value in config["preprocess_cfg"].items()}) | |
| model = model.float().eval() | |
| tokenizer = open_clip.get_tokenizer(name) | |
| fingerprint = hashlib.sha256((MODEL_REVISION + json.dumps(self.labels, sort_keys=True)).encode()).hexdigest() | |
| embedding_path = directory / f"prompt_embeddings_{fingerprint[:16]}.npy" | |
| if embedding_path.exists(): | |
| embeddings = np.load(embedding_path, allow_pickle=False) | |
| if embeddings.shape != (len(self.labels), 512) or not np.isfinite(embeddings).all(): | |
| raise ValueError("Invalid cached text embeddings; remove the prompt_embeddings file and restart.") | |
| else: | |
| prompts = [p for row in self.labels for p in row["prompts"]] | |
| vectors = [] | |
| with torch.inference_mode(): | |
| for start in range(0, len(prompts), 4): | |
| tokens = tokenizer(prompts[start:start + 4], context_length=256) | |
| vectors.append(model.encode_text(tokens, normalize=True).float()) | |
| vectors = torch.cat(vectors) | |
| groups, offset = [], 0 | |
| for row in self.labels: | |
| count = len(row["prompts"]) | |
| vector = vectors[offset:offset + count].mean(dim=0) | |
| groups.append(torch.nn.functional.normalize(vector, dim=0)) | |
| offset += count | |
| embeddings = torch.stack(groups).numpy().astype(np.float32) | |
| np.save(embedding_path, embeddings, allow_pickle=False) | |
| self.embeddings = torch.from_numpy(embeddings.copy()).to(self.device).float() | |
| self.visual = model.visual.to(self.device).float().eval() | |
| del model, tokenizer | |
| gc.collect() | |
| self.min_similarity = float(os.environ.get("MIN_SIMILARITY", "0.38")) | |
| self.background_margin = float(os.environ.get("BACKGROUND_MARGIN", "0.015")) | |
| with torch.inference_mode(): | |
| self.visual(torch.zeros((1, 3, 224, 224), device=self.device)) | |
| if self.device.type == "cuda": | |
| torch.cuda.synchronize() | |
| self.ready_at = time.time() | |
| def classify(self, data: bytes, *, frame_id=None, roi: dict | None = None) -> dict: | |
| started = time.perf_counter() | |
| try: | |
| with warnings.catch_warnings(): | |
| warnings.simplefilter("error", Image.DecompressionBombWarning) | |
| with Image.open(io.BytesIO(data)) as image: | |
| if image.format not in {"JPEG", "PNG", "WEBP"}: | |
| raise ValueError("Use a JPEG, PNG or WebP image.") | |
| image = ImageOps.exif_transpose(image).convert("RGB") | |
| image.load() | |
| except (UnidentifiedImageError, OSError, Image.DecompressionBombError, | |
| Image.DecompressionBombWarning) as error: | |
| raise ValueError("Image is invalid or exceeds the 16-megapixel limit.") from error | |
| width, height = image.size | |
| bounds = (0, 0, width, height) | |
| if roi is not None: | |
| coords = [float(roi[key]) for key in ("left", "top", "right", "bottom")] | |
| if (not all(np.isfinite(v) and 0 <= v <= 1 for v in coords) | |
| or coords[0] >= coords[2] or coords[1] >= coords[3]): | |
| raise ValueError("ROI must be an ordered normalized rectangle inside the image.") | |
| bounds = (int(coords[0] * width), int(coords[1] * height), | |
| int(coords[2] * width), int(coords[3] * height)) | |
| if bounds[2] - bounds[0] < 2 or bounds[3] - bounds[1] < 2: | |
| raise ValueError("ROI is too small.") | |
| image = image.crop(bounds) | |
| tensor = self.preprocess(image).unsqueeze(0).to(self.device, dtype=torch.float32) | |
| prepared = time.perf_counter() | |
| if self.device.type == "cuda": | |
| torch.cuda.synchronize() | |
| infer_start = time.perf_counter() | |
| with torch.inference_mode(): | |
| vector = torch.nn.functional.normalize(self.visual(tensor).float(), dim=-1) | |
| similarities = (vector @ self.embeddings.T).squeeze(0).cpu().tolist() | |
| inference_ms = (time.perf_counter() - infer_start) * 1000 | |
| candidates = sorted([ | |
| {"id": row["id"], "label": row["label"], "protocol_id": row["protocol_id"], | |
| "is_background": row["is_background"], "similarity": float(score)} | |
| for row, score in zip(self.labels, similarities) | |
| ], key=lambda row: row["similarity"], reverse=True) | |
| top = candidates[0] | |
| background = max(row["similarity"] for row in candidates if row["is_background"]) | |
| accepted = (not top["is_background"] and top["similarity"] >= self.min_similarity | |
| and top["similarity"] >= background + self.background_margin) | |
| # This describes the classifier's input field of view, NOT a wound box. | |
| view_w, view_h = bounds[2] - bounds[0], bounds[3] - bounds[1] | |
| side = min(view_w, view_h) | |
| left = bounds[0] + (view_w - side) / 2 | |
| top_y = bounds[1] + (view_h - side) / 2 | |
| return { | |
| "status": "success", "frame_id": frame_id, "candidates": candidates, | |
| "top_match": top, "accepted_match": top if accepted else None, | |
| "inference_ms": round(inference_ms, 2), | |
| "preprocess_ms": round((prepared - started) * 1000, 2), | |
| "server_ms": round((time.perf_counter() - started) * 1000, 2), | |
| "image_width": width, "image_height": height, | |
| "classified_view": {"left": left / width, "top": top_y / height, | |
| "right": (left + side) / width, "bottom": (top_y + side) / height}, | |
| "score_type": "cosine_similarity", "precision": "float32", "device": self.device.type, | |
| "model_id": MODEL_ID, "model_revision": MODEL_REVISION, | |
| "supports_localization": False, "clinical_validation": False, | |
| } | |