"""Official full-precision image encoder with cached prompted text embeddings.""" from __future__ import annotations import copy import gc import hashlib import io import json import os import threading import time import warnings from pathlib import Path os.environ.setdefault("USE_TF", "0") os.environ.setdefault("USE_FLAX", "0") os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") import numpy as np import open_clip import torch from PIL import Image, ImageOps, UnidentifiedImageError from download_model import DEFAULT_DIR, MODEL_ID, MODEL_REVISION, ensure_model ROOT = Path(__file__).resolve().parent Image.MAX_IMAGE_PIXELS = 16000000 class BiomedEngine: def __init__(self, model_dir: Path | None = None, labels_path: Path | None = None): self.lock = threading.Lock() self.labels = json.loads((labels_path or ROOT / "labels.json").read_text("utf-8")) if (not self.labels or len({row["id"] for row in self.labels}) != len(self.labels) or not any(row["is_background"] for row in self.labels) or any(not row["prompts"] for row in self.labels)): raise ValueError("Labels must be unique and include prompts and background entries.") self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") torch.set_num_threads(max(1, int(os.environ.get("TORCH_THREADS", "2")))) if self.device.type == "cuda": # Explicit FP32 comparison: do not enable AMP, FP16, INT8 or TF32. torch.backends.cuda.matmul.allow_tf32 = False torch.backends.cudnn.allow_tf32 = False directory = (model_dir or Path(os.environ.get("MODEL_DIR", DEFAULT_DIR))).resolve() ensure_model(directory) config = json.loads((directory / "open_clip_config.json").read_text("utf-8")) cfg = copy.deepcopy(config["model_cfg"]) cfg["text_cfg"]["hf_model_name"] = str(directory) cfg["text_cfg"]["hf_tokenizer_name"] = str(directory) from open_clip.factory import _MODEL_CONFIGS name = "saef_biomedclip_full_precision" _MODEL_CONFIGS[name] = cfg model, _, self.preprocess = open_clip.create_model_and_transforms( name, pretrained=str(directory / "open_clip_pytorch_model.bin"), precision="fp32", device="cpu", **{f"image_{key}": value for key, value in config["preprocess_cfg"].items()}) model = model.float().eval() tokenizer = open_clip.get_tokenizer(name) fingerprint = hashlib.sha256((MODEL_REVISION + json.dumps(self.labels, sort_keys=True)).encode()).hexdigest() embedding_path = directory / f"prompt_embeddings_{fingerprint[:16]}.npy" if embedding_path.exists(): embeddings = np.load(embedding_path, allow_pickle=False) if embeddings.shape != (len(self.labels), 512) or not np.isfinite(embeddings).all(): raise ValueError("Invalid cached text embeddings; remove the prompt_embeddings file and restart.") else: prompts = [p for row in self.labels for p in row["prompts"]] vectors = [] with torch.inference_mode(): for start in range(0, len(prompts), 4): tokens = tokenizer(prompts[start:start + 4], context_length=256) vectors.append(model.encode_text(tokens, normalize=True).float()) vectors = torch.cat(vectors) groups, offset = [], 0 for row in self.labels: count = len(row["prompts"]) vector = vectors[offset:offset + count].mean(dim=0) groups.append(torch.nn.functional.normalize(vector, dim=0)) offset += count embeddings = torch.stack(groups).numpy().astype(np.float32) np.save(embedding_path, embeddings, allow_pickle=False) self.embeddings = torch.from_numpy(embeddings.copy()).to(self.device).float() self.visual = model.visual.to(self.device).float().eval() del model, tokenizer gc.collect() self.min_similarity = float(os.environ.get("MIN_SIMILARITY", "0.38")) self.background_margin = float(os.environ.get("BACKGROUND_MARGIN", "0.015")) with torch.inference_mode(): self.visual(torch.zeros((1, 3, 224, 224), device=self.device)) if self.device.type == "cuda": torch.cuda.synchronize() self.ready_at = time.time() def classify(self, data: bytes, *, frame_id=None, roi: dict | None = None) -> dict: started = time.perf_counter() try: with warnings.catch_warnings(): warnings.simplefilter("error", Image.DecompressionBombWarning) with Image.open(io.BytesIO(data)) as image: if image.format not in {"JPEG", "PNG", "WEBP"}: raise ValueError("Use a JPEG, PNG or WebP image.") image = ImageOps.exif_transpose(image).convert("RGB") image.load() except (UnidentifiedImageError, OSError, Image.DecompressionBombError, Image.DecompressionBombWarning) as error: raise ValueError("Image is invalid or exceeds the 16-megapixel limit.") from error width, height = image.size bounds = (0, 0, width, height) if roi is not None: coords = [float(roi[key]) for key in ("left", "top", "right", "bottom")] if (not all(np.isfinite(v) and 0 <= v <= 1 for v in coords) or coords[0] >= coords[2] or coords[1] >= coords[3]): raise ValueError("ROI must be an ordered normalized rectangle inside the image.") bounds = (int(coords[0] * width), int(coords[1] * height), int(coords[2] * width), int(coords[3] * height)) if bounds[2] - bounds[0] < 2 or bounds[3] - bounds[1] < 2: raise ValueError("ROI is too small.") image = image.crop(bounds) tensor = self.preprocess(image).unsqueeze(0).to(self.device, dtype=torch.float32) prepared = time.perf_counter() if self.device.type == "cuda": torch.cuda.synchronize() infer_start = time.perf_counter() with torch.inference_mode(): vector = torch.nn.functional.normalize(self.visual(tensor).float(), dim=-1) similarities = (vector @ self.embeddings.T).squeeze(0).cpu().tolist() inference_ms = (time.perf_counter() - infer_start) * 1000 candidates = sorted([ {"id": row["id"], "label": row["label"], "protocol_id": row["protocol_id"], "is_background": row["is_background"], "similarity": float(score)} for row, score in zip(self.labels, similarities) ], key=lambda row: row["similarity"], reverse=True) top = candidates[0] background = max(row["similarity"] for row in candidates if row["is_background"]) accepted = (not top["is_background"] and top["similarity"] >= self.min_similarity and top["similarity"] >= background + self.background_margin) # This describes the classifier's input field of view, NOT a wound box. view_w, view_h = bounds[2] - bounds[0], bounds[3] - bounds[1] side = min(view_w, view_h) left = bounds[0] + (view_w - side) / 2 top_y = bounds[1] + (view_h - side) / 2 return { "status": "success", "frame_id": frame_id, "candidates": candidates, "top_match": top, "accepted_match": top if accepted else None, "inference_ms": round(inference_ms, 2), "preprocess_ms": round((prepared - started) * 1000, 2), "server_ms": round((time.perf_counter() - started) * 1000, 2), "image_width": width, "image_height": height, "classified_view": {"left": left / width, "top": top_y / height, "right": (left + side) / width, "bottom": (top_y + side) / height}, "score_type": "cosine_similarity", "precision": "float32", "device": self.device.type, "model_id": MODEL_ID, "model_revision": MODEL_REVISION, "supports_localization": False, "clinical_validation": False, }