"""Load and run the pin / no-pin classifier (front + wrist cameras, shared EfficientNet-B0). from pin_classifier import load_model, predict model = load_model("model.pt") label, p_pin = predict(model, front_rgb, wrist_rgb) # HxWx3 uint8 RGB arrays (any size) Only needs torch, torchvision, numpy and Pillow. """ from __future__ import annotations import numpy as np import torch from PIL import Image from torch import nn from torchvision.models import efficientnet_b0 CAMERA_ORDER = ("front", "wrist") # order in which the two cameras' features are concatenated POOL_ORDER = ("mean", "max") # order of the two global poolings per camera class PinClassifier(nn.Module): def __init__(self, classes, height, width): super().__init__() self.classes, self.height, self.width = list(classes), height, width self.register_buffer("mean", torch.zeros(1, 3, 1, 1)) self.register_buffer("std", torch.ones(1, 3, 1, 1)) self.backbone = efficientnet_b0(weights=None).features self.head = nn.Sequential(nn.Dropout(0.0), nn.Linear(2 * 2 * 1280, len(self.classes))) def embed(self, x: torch.Tensor) -> torch.Tensor: f = self.backbone((x - self.mean) / self.std) pooled = {"mean": f.mean(dim=(2, 3)), "max": f.amax(dim=(2, 3))} return torch.cat([pooled[p] for p in POOL_ORDER], dim=1) def forward(self, front: torch.Tensor, wrist: torch.Tensor) -> torch.Tensor: views = {"front": front, "wrist": wrist} return self.head(torch.cat([self.embed(views[c]) for c in CAMERA_ORDER], dim=1)) def load_model(path: str, device: str = "cpu") -> PinClassifier: ckpt = torch.load(path, map_location="cpu", weights_only=False) assert ckpt.get("backbone") == "efficientnet_b0" and ckpt.get("cameras") == "both", "unexpected checkpoint" model = PinClassifier(ckpt["classes"], ckpt["height"], ckpt["width"]) state = ckpt["state_dict"] model.mean.resize_as_(state["mean"]); model.std.resize_as_(state["std"]) model.load_state_dict(state, strict=True) return model.to(device).eval() def to_tensor(rgb: np.ndarray, width: int, height: int) -> torch.Tensor: """Centre-crop to the model's aspect ratio (never stretch), resize, scale to [0, 1].""" image = Image.fromarray(rgb) w, h = image.size target = width / height if w / h > target: cw = round(h * target); image = image.crop(((w - cw) // 2, 0, (w - cw) // 2 + cw, h)) else: ch = round(w / target); image = image.crop((0, (h - ch) // 2, w, (h - ch) // 2 + ch)) image = image.resize((width, height), Image.BILINEAR) return torch.from_numpy(np.asarray(image, dtype=np.float32) / 255.0).permute(2, 0, 1).unsqueeze(0) @torch.no_grad() def predict(model: PinClassifier, front_rgb: np.ndarray, wrist_rgb: np.ndarray) -> tuple[str, float]: device = next(model.parameters()).device front = to_tensor(front_rgb, model.width, model.height).to(device) wrist = to_tensor(wrist_rgb, model.width, model.height).to(device) probs = model(front, wrist).softmax(dim=1)[0] index = int(probs.argmax()) return model.classes[index], float(probs[model.classes.index("pin")])