v802-classifier / src /predict_v802.py
sneo's picture
Add v802 classifier head and source code
59839e3 verified
Raw History Blame Contribute Delete
5.8 kB
#!/usr/bin/env python3
"""Predict pony labels with the v802 bundle (classifier.joblib from build_v802.py).
Needs only the bundle and the DINOv3 ViT-L/16 weights; features are computed
on the fly exactly as in training (640x352 raw + horizontal flip, bf16 autocast).
python predict_v802.py --bundle ../artifacts/models/v802/classifier.joblib \
--model ../../models/dinov3-vitl16-pretrain-lvd1689m image1.jpg dir/ ...
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import joblib
import numpy as np
import torch
from PIL import Image
from transformers import AutoModel
from build_v802 import decode
from extract_features_v802 import preprocess
from experiment_v802_probe import l2
from train_v801 import binary_probabilities
from v802_mil import MilConfig, MilHead, project
IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp", ".bmp"}
def image_paths(inputs: list[Path]) -> list[Path]:
paths = []
for item in inputs:
if item.is_dir():
paths += sorted(p for p in item.iterdir() if p.suffix.lower() in IMAGE_SUFFIXES)
else:
paths.append(item)
return paths
def view_features(model, image: Image.Image, flip: bool, bf16: bool, probe_spec, mil_state):
"""Return (probe descriptor, projected MIL patches) for one view of one image."""
depth = int(model.config.num_hidden_layers)
registers = int(model.config.num_register_tokens)
x = preprocess(image, 640, 352, "raw", flip)
with torch.inference_mode(), torch.autocast("cpu", dtype=torch.bfloat16, enabled=bf16):
output = model(pixel_values=torch.from_numpy(x)[None], output_hidden_states=True)
def hidden(layer):
return (output.last_hidden_state if layer == depth else output.hidden_states[layer])[0].float()
stat = {"cls": lambda h: h[0], "mean": lambda h: h[1 + registers:].mean(0)}
blocks = [stat[k](hidden(layer)).numpy() for layer in probe_spec["layers"] for k in probe_spec["stats"]]
descriptor = l2(np.concatenate([l2(b) for b in blocks]))
parts = []
for entry in mil_state["pca"]:
tokens = hidden(entry["layer"])[1 + registers:].numpy().astype(np.float16) # as stored in training
parts.append(project(tokens[None], entry["mean"], entry["std"], entry["basis"])[0])
return descriptor, np.concatenate(parts, axis=1)
class V802Predictor:
"""Loads the bundle and DINOv3 once; predicts label sets for PIL images."""
def __init__(self, bundle_path: Path, model_path: Path, bf16: bool = True):
bundle = joblib.load(bundle_path)
self.config, self.classes, self.decoder = bundle["config"], bundle["classes"], bundle["decoder"]
names = [c["name"] for c in self.config["components"]]
self.specs = dict(zip(names, self.config["components"]))
self.states = dict(zip(names, bundle["components"]))
assert set(self.specs) == {"mil", "probe"}, "this predictor supports the mil + probe v802 bundle"
mil_state = self.states["mil"]
cfg = MilConfig(**mil_state["mil_config"])
self.heads = []
for weights in mil_state["heads"]:
head = MilHead(len(mil_state["pca"]) * mil_state["dims"], len(self.classes), cfg)
head.load_state_dict(weights)
self.heads.append(head.eval())
self.model = AutoModel.from_pretrained(model_path, local_files_only=True).eval()
self.bf16 = bf16
def probabilities(self, image: Image.Image) -> np.ndarray:
views = [view_features(self.model, image, flip, self.bf16, self.specs["probe"], self.states["mil"])
for flip in (False, True)]
probe = np.mean([binary_probabilities(self.states["probe"]["probe"], d[None])[0] for d, _ in views], axis=0)
with torch.inference_mode():
mil = np.mean([torch.sigmoid(head(torch.from_numpy(patches[None]))[0])[0].numpy()
for head in self.heads for _, patches in views], axis=0)
return self.specs["mil"]["weight"] * mil + self.specs["probe"]["weight"] * probe
def predict(self, images: list[Image.Image]) -> list[dict]:
probs = np.asarray([self.probabilities(image) for image in images])
labels = decode(self.decoder, probs, self.config["prior"], self.config["card"])
return [{"labels": [self.classes[j] for j in np.flatnonzero(row)],
"probabilities": {c: round(float(p), 4) for c, p in zip(self.classes, prob)}}
for row, prob in zip(labels, probs)]
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("images", type=Path, nargs="+", help="image files or directories")
parser.add_argument("--bundle", type=Path, required=True)
parser.add_argument("--model", type=Path, required=True, help="DINOv3 ViT-L/16 directory")
parser.add_argument("--threads", type=int, default=4)
parser.add_argument("--fp32", action="store_true", help="disable bf16 autocast (slower on AVX512-BF16 CPUs)")
parser.add_argument("--output", type=Path, help="write JSON lines here instead of stdout")
args = parser.parse_args()
torch.set_num_threads(args.threads)
predictor = V802Predictor(args.bundle, args.model, bf16=not args.fp32)
paths = image_paths(args.images)
images = []
for path in paths:
with Image.open(path) as image:
images.append(image.copy())
lines = [json.dumps({"file": str(path), **result}, ensure_ascii=False)
for path, result in zip(paths, predictor.predict(images))]
if args.output:
args.output.write_text("\n".join(lines) + "\n", encoding="utf-8")
else:
print("\n".join(lines))
if __name__ == "__main__":
main()