saef-api / engine.py
m4az19's picture
Upload 10 files
174cc12 verified
Raw History Blame Contribute Delete
8.29 kB
"""Official full-precision image encoder with cached prompted text embeddings."""
from __future__ import annotations
import copy
import gc
import hashlib
import io
import json
import os
import threading
import time
import warnings
from pathlib import Path
os.environ.setdefault("USE_TF", "0")
os.environ.setdefault("USE_FLAX", "0")
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
import numpy as np
import open_clip
import torch
from PIL import Image, ImageOps, UnidentifiedImageError
from download_model import DEFAULT_DIR, MODEL_ID, MODEL_REVISION, ensure_model
ROOT = Path(__file__).resolve().parent
Image.MAX_IMAGE_PIXELS = 16000000
class BiomedEngine:
def __init__(self, model_dir: Path | None = None, labels_path: Path | None = None):
self.lock = threading.Lock()
self.labels = json.loads((labels_path or ROOT / "labels.json").read_text("utf-8"))
if (not self.labels or len({row["id"] for row in self.labels}) != len(self.labels)
or not any(row["is_background"] for row in self.labels)
or any(not row["prompts"] for row in self.labels)):
raise ValueError("Labels must be unique and include prompts and background entries.")
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
torch.set_num_threads(max(1, int(os.environ.get("TORCH_THREADS", "2"))))
if self.device.type == "cuda":
# Explicit FP32 comparison: do not enable AMP, FP16, INT8 or TF32.
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
directory = (model_dir or Path(os.environ.get("MODEL_DIR", DEFAULT_DIR))).resolve()
ensure_model(directory)
config = json.loads((directory / "open_clip_config.json").read_text("utf-8"))
cfg = copy.deepcopy(config["model_cfg"])
cfg["text_cfg"]["hf_model_name"] = str(directory)
cfg["text_cfg"]["hf_tokenizer_name"] = str(directory)
from open_clip.factory import _MODEL_CONFIGS
name = "saef_biomedclip_full_precision"
_MODEL_CONFIGS[name] = cfg
model, _, self.preprocess = open_clip.create_model_and_transforms(
name, pretrained=str(directory / "open_clip_pytorch_model.bin"),
precision="fp32", device="cpu",
**{f"image_{key}": value for key, value in config["preprocess_cfg"].items()})
model = model.float().eval()
tokenizer = open_clip.get_tokenizer(name)
fingerprint = hashlib.sha256((MODEL_REVISION + json.dumps(self.labels, sort_keys=True)).encode()).hexdigest()
embedding_path = directory / f"prompt_embeddings_{fingerprint[:16]}.npy"
if embedding_path.exists():
embeddings = np.load(embedding_path, allow_pickle=False)
if embeddings.shape != (len(self.labels), 512) or not np.isfinite(embeddings).all():
raise ValueError("Invalid cached text embeddings; remove the prompt_embeddings file and restart.")
else:
prompts = [p for row in self.labels for p in row["prompts"]]
vectors = []
with torch.inference_mode():
for start in range(0, len(prompts), 4):
tokens = tokenizer(prompts[start:start + 4], context_length=256)
vectors.append(model.encode_text(tokens, normalize=True).float())
vectors = torch.cat(vectors)
groups, offset = [], 0
for row in self.labels:
count = len(row["prompts"])
vector = vectors[offset:offset + count].mean(dim=0)
groups.append(torch.nn.functional.normalize(vector, dim=0))
offset += count
embeddings = torch.stack(groups).numpy().astype(np.float32)
np.save(embedding_path, embeddings, allow_pickle=False)
self.embeddings = torch.from_numpy(embeddings.copy()).to(self.device).float()
self.visual = model.visual.to(self.device).float().eval()
del model, tokenizer
gc.collect()
self.min_similarity = float(os.environ.get("MIN_SIMILARITY", "0.38"))
self.background_margin = float(os.environ.get("BACKGROUND_MARGIN", "0.015"))
with torch.inference_mode():
self.visual(torch.zeros((1, 3, 224, 224), device=self.device))
if self.device.type == "cuda":
torch.cuda.synchronize()
self.ready_at = time.time()
def classify(self, data: bytes, *, frame_id=None, roi: dict | None = None) -> dict:
started = time.perf_counter()
try:
with warnings.catch_warnings():
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(io.BytesIO(data)) as image:
if image.format not in {"JPEG", "PNG", "WEBP"}:
raise ValueError("Use a JPEG, PNG or WebP image.")
image = ImageOps.exif_transpose(image).convert("RGB")
image.load()
except (UnidentifiedImageError, OSError, Image.DecompressionBombError,
Image.DecompressionBombWarning) as error:
raise ValueError("Image is invalid or exceeds the 16-megapixel limit.") from error
width, height = image.size
bounds = (0, 0, width, height)
if roi is not None:
coords = [float(roi[key]) for key in ("left", "top", "right", "bottom")]
if (not all(np.isfinite(v) and 0 <= v <= 1 for v in coords)
or coords[0] >= coords[2] or coords[1] >= coords[3]):
raise ValueError("ROI must be an ordered normalized rectangle inside the image.")
bounds = (int(coords[0] * width), int(coords[1] * height),
int(coords[2] * width), int(coords[3] * height))
if bounds[2] - bounds[0] < 2 or bounds[3] - bounds[1] < 2:
raise ValueError("ROI is too small.")
image = image.crop(bounds)
tensor = self.preprocess(image).unsqueeze(0).to(self.device, dtype=torch.float32)
prepared = time.perf_counter()
if self.device.type == "cuda":
torch.cuda.synchronize()
infer_start = time.perf_counter()
with torch.inference_mode():
vector = torch.nn.functional.normalize(self.visual(tensor).float(), dim=-1)
similarities = (vector @ self.embeddings.T).squeeze(0).cpu().tolist()
inference_ms = (time.perf_counter() - infer_start) * 1000
candidates = sorted([
{"id": row["id"], "label": row["label"], "protocol_id": row["protocol_id"],
"is_background": row["is_background"], "similarity": float(score)}
for row, score in zip(self.labels, similarities)
], key=lambda row: row["similarity"], reverse=True)
top = candidates[0]
background = max(row["similarity"] for row in candidates if row["is_background"])
accepted = (not top["is_background"] and top["similarity"] >= self.min_similarity
and top["similarity"] >= background + self.background_margin)
# This describes the classifier's input field of view, NOT a wound box.
view_w, view_h = bounds[2] - bounds[0], bounds[3] - bounds[1]
side = min(view_w, view_h)
left = bounds[0] + (view_w - side) / 2
top_y = bounds[1] + (view_h - side) / 2
return {
"status": "success", "frame_id": frame_id, "candidates": candidates,
"top_match": top, "accepted_match": top if accepted else None,
"inference_ms": round(inference_ms, 2),
"preprocess_ms": round((prepared - started) * 1000, 2),
"server_ms": round((time.perf_counter() - started) * 1000, 2),
"image_width": width, "image_height": height,
"classified_view": {"left": left / width, "top": top_y / height,
"right": (left + side) / width, "bottom": (top_y + side) / height},
"score_type": "cosine_similarity", "precision": "float32", "device": self.device.type,
"model_id": MODEL_ID, "model_revision": MODEL_REVISION,
"supports_localization": False, "clinical_validation": False,
}