File size: 8,292 Bytes
174cc12
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
"""Official full-precision image encoder with cached prompted text embeddings."""
from __future__ import annotations

import copy
import gc
import hashlib
import io
import json
import os
import threading
import time
import warnings
from pathlib import Path

os.environ.setdefault("USE_TF", "0")
os.environ.setdefault("USE_FLAX", "0")
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")

import numpy as np
import open_clip
import torch
from PIL import Image, ImageOps, UnidentifiedImageError

from download_model import DEFAULT_DIR, MODEL_ID, MODEL_REVISION, ensure_model

ROOT = Path(__file__).resolve().parent
Image.MAX_IMAGE_PIXELS = 16000000


class BiomedEngine:
    def __init__(self, model_dir: Path | None = None, labels_path: Path | None = None):
        self.lock = threading.Lock()
        self.labels = json.loads((labels_path or ROOT / "labels.json").read_text("utf-8"))
        if (not self.labels or len({row["id"] for row in self.labels}) != len(self.labels)
                or not any(row["is_background"] for row in self.labels)
                or any(not row["prompts"] for row in self.labels)):
            raise ValueError("Labels must be unique and include prompts and background entries.")
        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        torch.set_num_threads(max(1, int(os.environ.get("TORCH_THREADS", "2"))))
        if self.device.type == "cuda":
            # Explicit FP32 comparison: do not enable AMP, FP16, INT8 or TF32.
            torch.backends.cuda.matmul.allow_tf32 = False
            torch.backends.cudnn.allow_tf32 = False
        directory = (model_dir or Path(os.environ.get("MODEL_DIR", DEFAULT_DIR))).resolve()
        ensure_model(directory)
        config = json.loads((directory / "open_clip_config.json").read_text("utf-8"))
        cfg = copy.deepcopy(config["model_cfg"])
        cfg["text_cfg"]["hf_model_name"] = str(directory)
        cfg["text_cfg"]["hf_tokenizer_name"] = str(directory)
        from open_clip.factory import _MODEL_CONFIGS
        name = "saef_biomedclip_full_precision"
        _MODEL_CONFIGS[name] = cfg
        model, _, self.preprocess = open_clip.create_model_and_transforms(
            name, pretrained=str(directory / "open_clip_pytorch_model.bin"),
            precision="fp32", device="cpu",
            **{f"image_{key}": value for key, value in config["preprocess_cfg"].items()})
        model = model.float().eval()
        tokenizer = open_clip.get_tokenizer(name)
        fingerprint = hashlib.sha256((MODEL_REVISION + json.dumps(self.labels, sort_keys=True)).encode()).hexdigest()
        embedding_path = directory / f"prompt_embeddings_{fingerprint[:16]}.npy"
        if embedding_path.exists():
            embeddings = np.load(embedding_path, allow_pickle=False)
            if embeddings.shape != (len(self.labels), 512) or not np.isfinite(embeddings).all():
                raise ValueError("Invalid cached text embeddings; remove the prompt_embeddings file and restart.")
        else:
            prompts = [p for row in self.labels for p in row["prompts"]]
            vectors = []
            with torch.inference_mode():
                for start in range(0, len(prompts), 4):
                    tokens = tokenizer(prompts[start:start + 4], context_length=256)
                    vectors.append(model.encode_text(tokens, normalize=True).float())
                vectors = torch.cat(vectors)
                groups, offset = [], 0
                for row in self.labels:
                    count = len(row["prompts"])
                    vector = vectors[offset:offset + count].mean(dim=0)
                    groups.append(torch.nn.functional.normalize(vector, dim=0))
                    offset += count
                embeddings = torch.stack(groups).numpy().astype(np.float32)
            np.save(embedding_path, embeddings, allow_pickle=False)
        self.embeddings = torch.from_numpy(embeddings.copy()).to(self.device).float()
        self.visual = model.visual.to(self.device).float().eval()
        del model, tokenizer
        gc.collect()
        self.min_similarity = float(os.environ.get("MIN_SIMILARITY", "0.38"))
        self.background_margin = float(os.environ.get("BACKGROUND_MARGIN", "0.015"))
        with torch.inference_mode():
            self.visual(torch.zeros((1, 3, 224, 224), device=self.device))
        if self.device.type == "cuda":
            torch.cuda.synchronize()
        self.ready_at = time.time()

    def classify(self, data: bytes, *, frame_id=None, roi: dict | None = None) -> dict:
        started = time.perf_counter()
        try:
            with warnings.catch_warnings():
                warnings.simplefilter("error", Image.DecompressionBombWarning)
                with Image.open(io.BytesIO(data)) as image:
                    if image.format not in {"JPEG", "PNG", "WEBP"}:
                        raise ValueError("Use a JPEG, PNG or WebP image.")
                    image = ImageOps.exif_transpose(image).convert("RGB")
                    image.load()
        except (UnidentifiedImageError, OSError, Image.DecompressionBombError,
                Image.DecompressionBombWarning) as error:
            raise ValueError("Image is invalid or exceeds the 16-megapixel limit.") from error
        width, height = image.size
        bounds = (0, 0, width, height)
        if roi is not None:
            coords = [float(roi[key]) for key in ("left", "top", "right", "bottom")]
            if (not all(np.isfinite(v) and 0 <= v <= 1 for v in coords)
                    or coords[0] >= coords[2] or coords[1] >= coords[3]):
                raise ValueError("ROI must be an ordered normalized rectangle inside the image.")
            bounds = (int(coords[0] * width), int(coords[1] * height),
                      int(coords[2] * width), int(coords[3] * height))
            if bounds[2] - bounds[0] < 2 or bounds[3] - bounds[1] < 2:
                raise ValueError("ROI is too small.")
            image = image.crop(bounds)
        tensor = self.preprocess(image).unsqueeze(0).to(self.device, dtype=torch.float32)
        prepared = time.perf_counter()
        if self.device.type == "cuda":
            torch.cuda.synchronize()
        infer_start = time.perf_counter()
        with torch.inference_mode():
            vector = torch.nn.functional.normalize(self.visual(tensor).float(), dim=-1)
            similarities = (vector @ self.embeddings.T).squeeze(0).cpu().tolist()
        inference_ms = (time.perf_counter() - infer_start) * 1000
        candidates = sorted([
            {"id": row["id"], "label": row["label"], "protocol_id": row["protocol_id"],
             "is_background": row["is_background"], "similarity": float(score)}
            for row, score in zip(self.labels, similarities)
        ], key=lambda row: row["similarity"], reverse=True)
        top = candidates[0]
        background = max(row["similarity"] for row in candidates if row["is_background"])
        accepted = (not top["is_background"] and top["similarity"] >= self.min_similarity
                    and top["similarity"] >= background + self.background_margin)
        # This describes the classifier's input field of view, NOT a wound box.
        view_w, view_h = bounds[2] - bounds[0], bounds[3] - bounds[1]
        side = min(view_w, view_h)
        left = bounds[0] + (view_w - side) / 2
        top_y = bounds[1] + (view_h - side) / 2
        return {
            "status": "success", "frame_id": frame_id, "candidates": candidates,
            "top_match": top, "accepted_match": top if accepted else None,
            "inference_ms": round(inference_ms, 2),
            "preprocess_ms": round((prepared - started) * 1000, 2),
            "server_ms": round((time.perf_counter() - started) * 1000, 2),
            "image_width": width, "image_height": height,
            "classified_view": {"left": left / width, "top": top_y / height,
                                "right": (left + side) / width, "bottom": (top_y + side) / height},
            "score_type": "cosine_similarity", "precision": "float32", "device": self.device.type,
            "model_id": MODEL_ID, "model_revision": MODEL_REVISION,
            "supports_localization": False, "clinical_validation": False,
        }