File size: 5,799 Bytes
59839e3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
#!/usr/bin/env python3
"""Predict pony labels with the v802 bundle (classifier.joblib from build_v802.py).

Needs only the bundle and the DINOv3 ViT-L/16 weights; features are computed
on the fly exactly as in training (640x352 raw + horizontal flip, bf16 autocast).

  python predict_v802.py --bundle ../artifacts/models/v802/classifier.joblib \
      --model ../../models/dinov3-vitl16-pretrain-lvd1689m image1.jpg dir/ ...
"""

from __future__ import annotations

import argparse
import json
from pathlib import Path

import joblib
import numpy as np
import torch
from PIL import Image
from transformers import AutoModel

from build_v802 import decode
from extract_features_v802 import preprocess
from experiment_v802_probe import l2
from train_v801 import binary_probabilities
from v802_mil import MilConfig, MilHead, project

IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp", ".bmp"}


def image_paths(inputs: list[Path]) -> list[Path]:
    paths = []
    for item in inputs:
        if item.is_dir():
            paths += sorted(p for p in item.iterdir() if p.suffix.lower() in IMAGE_SUFFIXES)
        else:
            paths.append(item)
    return paths


def view_features(model, image: Image.Image, flip: bool, bf16: bool, probe_spec, mil_state):
    """Return (probe descriptor, projected MIL patches) for one view of one image."""
    depth = int(model.config.num_hidden_layers)
    registers = int(model.config.num_register_tokens)
    x = preprocess(image, 640, 352, "raw", flip)
    with torch.inference_mode(), torch.autocast("cpu", dtype=torch.bfloat16, enabled=bf16):
        output = model(pixel_values=torch.from_numpy(x)[None], output_hidden_states=True)

    def hidden(layer):
        return (output.last_hidden_state if layer == depth else output.hidden_states[layer])[0].float()

    stat = {"cls": lambda h: h[0], "mean": lambda h: h[1 + registers:].mean(0)}
    blocks = [stat[k](hidden(layer)).numpy() for layer in probe_spec["layers"] for k in probe_spec["stats"]]
    descriptor = l2(np.concatenate([l2(b) for b in blocks]))
    parts = []
    for entry in mil_state["pca"]:
        tokens = hidden(entry["layer"])[1 + registers:].numpy().astype(np.float16)  # as stored in training
        parts.append(project(tokens[None], entry["mean"], entry["std"], entry["basis"])[0])
    return descriptor, np.concatenate(parts, axis=1)


class V802Predictor:
    """Loads the bundle and DINOv3 once; predicts label sets for PIL images."""

    def __init__(self, bundle_path: Path, model_path: Path, bf16: bool = True):
        bundle = joblib.load(bundle_path)
        self.config, self.classes, self.decoder = bundle["config"], bundle["classes"], bundle["decoder"]
        names = [c["name"] for c in self.config["components"]]
        self.specs = dict(zip(names, self.config["components"]))
        self.states = dict(zip(names, bundle["components"]))
        assert set(self.specs) == {"mil", "probe"}, "this predictor supports the mil + probe v802 bundle"
        mil_state = self.states["mil"]
        cfg = MilConfig(**mil_state["mil_config"])
        self.heads = []
        for weights in mil_state["heads"]:
            head = MilHead(len(mil_state["pca"]) * mil_state["dims"], len(self.classes), cfg)
            head.load_state_dict(weights)
            self.heads.append(head.eval())
        self.model = AutoModel.from_pretrained(model_path, local_files_only=True).eval()
        self.bf16 = bf16

    def probabilities(self, image: Image.Image) -> np.ndarray:
        views = [view_features(self.model, image, flip, self.bf16, self.specs["probe"], self.states["mil"])
                 for flip in (False, True)]
        probe = np.mean([binary_probabilities(self.states["probe"]["probe"], d[None])[0] for d, _ in views], axis=0)
        with torch.inference_mode():
            mil = np.mean([torch.sigmoid(head(torch.from_numpy(patches[None]))[0])[0].numpy()
                           for head in self.heads for _, patches in views], axis=0)
        return self.specs["mil"]["weight"] * mil + self.specs["probe"]["weight"] * probe

    def predict(self, images: list[Image.Image]) -> list[dict]:
        probs = np.asarray([self.probabilities(image) for image in images])
        labels = decode(self.decoder, probs, self.config["prior"], self.config["card"])
        return [{"labels": [self.classes[j] for j in np.flatnonzero(row)],
                 "probabilities": {c: round(float(p), 4) for c, p in zip(self.classes, prob)}}
                for row, prob in zip(labels, probs)]


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("images", type=Path, nargs="+", help="image files or directories")
    parser.add_argument("--bundle", type=Path, required=True)
    parser.add_argument("--model", type=Path, required=True, help="DINOv3 ViT-L/16 directory")
    parser.add_argument("--threads", type=int, default=4)
    parser.add_argument("--fp32", action="store_true", help="disable bf16 autocast (slower on AVX512-BF16 CPUs)")
    parser.add_argument("--output", type=Path, help="write JSON lines here instead of stdout")
    args = parser.parse_args()
    torch.set_num_threads(args.threads)
    predictor = V802Predictor(args.bundle, args.model, bf16=not args.fp32)
    paths = image_paths(args.images)
    images = []
    for path in paths:
        with Image.open(path) as image:
            images.append(image.copy())
    lines = [json.dumps({"file": str(path), **result}, ensure_ascii=False)
             for path, result in zip(paths, predictor.predict(images))]
    if args.output:
        args.output.write_text("\n".join(lines) + "\n", encoding="utf-8")
    else:
        print("\n".join(lines))


if __name__ == "__main__":
    main()