File size: 8,292 Bytes
174cc12 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 | """Official full-precision image encoder with cached prompted text embeddings."""
from __future__ import annotations
import copy
import gc
import hashlib
import io
import json
import os
import threading
import time
import warnings
from pathlib import Path
os.environ.setdefault("USE_TF", "0")
os.environ.setdefault("USE_FLAX", "0")
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
import numpy as np
import open_clip
import torch
from PIL import Image, ImageOps, UnidentifiedImageError
from download_model import DEFAULT_DIR, MODEL_ID, MODEL_REVISION, ensure_model
ROOT = Path(__file__).resolve().parent
Image.MAX_IMAGE_PIXELS = 16000000
class BiomedEngine:
def __init__(self, model_dir: Path | None = None, labels_path: Path | None = None):
self.lock = threading.Lock()
self.labels = json.loads((labels_path or ROOT / "labels.json").read_text("utf-8"))
if (not self.labels or len({row["id"] for row in self.labels}) != len(self.labels)
or not any(row["is_background"] for row in self.labels)
or any(not row["prompts"] for row in self.labels)):
raise ValueError("Labels must be unique and include prompts and background entries.")
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
torch.set_num_threads(max(1, int(os.environ.get("TORCH_THREADS", "2"))))
if self.device.type == "cuda":
# Explicit FP32 comparison: do not enable AMP, FP16, INT8 or TF32.
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
directory = (model_dir or Path(os.environ.get("MODEL_DIR", DEFAULT_DIR))).resolve()
ensure_model(directory)
config = json.loads((directory / "open_clip_config.json").read_text("utf-8"))
cfg = copy.deepcopy(config["model_cfg"])
cfg["text_cfg"]["hf_model_name"] = str(directory)
cfg["text_cfg"]["hf_tokenizer_name"] = str(directory)
from open_clip.factory import _MODEL_CONFIGS
name = "saef_biomedclip_full_precision"
_MODEL_CONFIGS[name] = cfg
model, _, self.preprocess = open_clip.create_model_and_transforms(
name, pretrained=str(directory / "open_clip_pytorch_model.bin"),
precision="fp32", device="cpu",
**{f"image_{key}": value for key, value in config["preprocess_cfg"].items()})
model = model.float().eval()
tokenizer = open_clip.get_tokenizer(name)
fingerprint = hashlib.sha256((MODEL_REVISION + json.dumps(self.labels, sort_keys=True)).encode()).hexdigest()
embedding_path = directory / f"prompt_embeddings_{fingerprint[:16]}.npy"
if embedding_path.exists():
embeddings = np.load(embedding_path, allow_pickle=False)
if embeddings.shape != (len(self.labels), 512) or not np.isfinite(embeddings).all():
raise ValueError("Invalid cached text embeddings; remove the prompt_embeddings file and restart.")
else:
prompts = [p for row in self.labels for p in row["prompts"]]
vectors = []
with torch.inference_mode():
for start in range(0, len(prompts), 4):
tokens = tokenizer(prompts[start:start + 4], context_length=256)
vectors.append(model.encode_text(tokens, normalize=True).float())
vectors = torch.cat(vectors)
groups, offset = [], 0
for row in self.labels:
count = len(row["prompts"])
vector = vectors[offset:offset + count].mean(dim=0)
groups.append(torch.nn.functional.normalize(vector, dim=0))
offset += count
embeddings = torch.stack(groups).numpy().astype(np.float32)
np.save(embedding_path, embeddings, allow_pickle=False)
self.embeddings = torch.from_numpy(embeddings.copy()).to(self.device).float()
self.visual = model.visual.to(self.device).float().eval()
del model, tokenizer
gc.collect()
self.min_similarity = float(os.environ.get("MIN_SIMILARITY", "0.38"))
self.background_margin = float(os.environ.get("BACKGROUND_MARGIN", "0.015"))
with torch.inference_mode():
self.visual(torch.zeros((1, 3, 224, 224), device=self.device))
if self.device.type == "cuda":
torch.cuda.synchronize()
self.ready_at = time.time()
def classify(self, data: bytes, *, frame_id=None, roi: dict | None = None) -> dict:
started = time.perf_counter()
try:
with warnings.catch_warnings():
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(io.BytesIO(data)) as image:
if image.format not in {"JPEG", "PNG", "WEBP"}:
raise ValueError("Use a JPEG, PNG or WebP image.")
image = ImageOps.exif_transpose(image).convert("RGB")
image.load()
except (UnidentifiedImageError, OSError, Image.DecompressionBombError,
Image.DecompressionBombWarning) as error:
raise ValueError("Image is invalid or exceeds the 16-megapixel limit.") from error
width, height = image.size
bounds = (0, 0, width, height)
if roi is not None:
coords = [float(roi[key]) for key in ("left", "top", "right", "bottom")]
if (not all(np.isfinite(v) and 0 <= v <= 1 for v in coords)
or coords[0] >= coords[2] or coords[1] >= coords[3]):
raise ValueError("ROI must be an ordered normalized rectangle inside the image.")
bounds = (int(coords[0] * width), int(coords[1] * height),
int(coords[2] * width), int(coords[3] * height))
if bounds[2] - bounds[0] < 2 or bounds[3] - bounds[1] < 2:
raise ValueError("ROI is too small.")
image = image.crop(bounds)
tensor = self.preprocess(image).unsqueeze(0).to(self.device, dtype=torch.float32)
prepared = time.perf_counter()
if self.device.type == "cuda":
torch.cuda.synchronize()
infer_start = time.perf_counter()
with torch.inference_mode():
vector = torch.nn.functional.normalize(self.visual(tensor).float(), dim=-1)
similarities = (vector @ self.embeddings.T).squeeze(0).cpu().tolist()
inference_ms = (time.perf_counter() - infer_start) * 1000
candidates = sorted([
{"id": row["id"], "label": row["label"], "protocol_id": row["protocol_id"],
"is_background": row["is_background"], "similarity": float(score)}
for row, score in zip(self.labels, similarities)
], key=lambda row: row["similarity"], reverse=True)
top = candidates[0]
background = max(row["similarity"] for row in candidates if row["is_background"])
accepted = (not top["is_background"] and top["similarity"] >= self.min_similarity
and top["similarity"] >= background + self.background_margin)
# This describes the classifier's input field of view, NOT a wound box.
view_w, view_h = bounds[2] - bounds[0], bounds[3] - bounds[1]
side = min(view_w, view_h)
left = bounds[0] + (view_w - side) / 2
top_y = bounds[1] + (view_h - side) / 2
return {
"status": "success", "frame_id": frame_id, "candidates": candidates,
"top_match": top, "accepted_match": top if accepted else None,
"inference_ms": round(inference_ms, 2),
"preprocess_ms": round((prepared - started) * 1000, 2),
"server_ms": round((time.perf_counter() - started) * 1000, 2),
"image_width": width, "image_height": height,
"classified_view": {"left": left / width, "top": top_y / height,
"right": (left + side) / width, "bottom": (top_y + side) / height},
"score_type": "cosine_similarity", "precision": "float32", "device": self.device.type,
"model_id": MODEL_ID, "model_revision": MODEL_REVISION,
"supports_localization": False, "clinical_validation": False,
}
|