v802-classifier / src /extract_features_v802.py
sneo's picture
Add v802 classifier head and source code
59839e3 verified
Raw History Blame Contribute Delete
6.37 kB
#!/usr/bin/env python3
"""Extract native-aspect DINOv3 patch tokens for v802 (resumable).
One forward pass per image at a 16-aligned, aspect-preserving resolution.
Outputs (in --output-dir):
meta.json settings, file order, classes
pooled.npy (N, L, 5, D) f32 per-layer CLS, register mean, patch mean/max/std
patches_l{K}.npy (N, P, D) f16 patch tokens for the layers in --patch-layers
done.npy (N,) bool completion mask for resume
"""
from __future__ import annotations
import argparse
import json
import time
from pathlib import Path
import numpy as np
import torch
from PIL import Image, ImageOps
from transformers import AutoModel
from dataset_utils import load_dataset
MEAN = np.asarray([0.485, 0.456, 0.406], dtype=np.float32)
STD = np.asarray([0.229, 0.224, 0.225], dtype=np.float32)
def preprocess(image: Image.Image, width: int, height: int, mode: str, flip: bool) -> np.ndarray:
image = image.convert("RGB")
if mode == "autocontrast":
gray = ImageOps.autocontrast(image.convert("L"), cutoff=1)
image = Image.merge("RGB", (gray, gray, gray))
elif mode == "equalize":
gray = ImageOps.equalize(image.convert("L"))
image = Image.merge("RGB", (gray, gray, gray))
elif mode != "raw":
raise ValueError(mode)
if flip:
image = ImageOps.mirror(image)
image = image.resize((width, height), Image.Resampling.BICUBIC)
array = np.asarray(image, dtype=np.float32) / 255.0
return ((array - MEAN) / STD).transpose(2, 0, 1)
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--dataset", type=Path, required=True)
parser.add_argument("--model", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--width", type=int, default=640)
parser.add_argument("--height", type=int, default=352)
parser.add_argument("--mode", choices=("raw", "autocontrast", "equalize"), default="raw")
parser.add_argument("--flip", action="store_true")
parser.add_argument("--pooled-layers", type=int, nargs="+", default=[8, 12, 16, 20, 24])
parser.add_argument("--patch-layers", type=int, nargs="*", default=[16, 20, 24])
parser.add_argument("--threads", type=int, default=4)
parser.add_argument("--bf16", action="store_true", help="bfloat16 autocast (x86 AVX512-BF16)")
args = parser.parse_args()
if args.width % 16 or args.height % 16:
raise ValueError("width/height must be multiples of 16")
torch.set_num_threads(args.threads)
dataset = load_dataset(args.dataset)
model = AutoModel.from_pretrained(args.model, local_files_only=True).eval()
registers = int(model.config.num_register_tokens)
depth = int(model.config.num_hidden_layers)
dim = int(model.config.hidden_size)
n = len(dataset.file_names)
patches = (args.width // 16) * (args.height // 16)
out = args.output_dir
out.mkdir(parents=True, exist_ok=True)
meta = {
"dataset": str(args.dataset),
"model": str(args.model),
"width": args.width,
"height": args.height,
"grid": [args.height // 16, args.width // 16],
"mode": args.mode,
"flip": args.flip,
"pooled_layers": args.pooled_layers,
"pooled_stats": ["cls", "register_mean", "patch_mean", "patch_max", "patch_std"],
"patch_layers": args.patch_layers,
"layer_note": f"layer {depth} is the final normalized last_hidden_state; others are raw block outputs",
"bf16": args.bf16,
"file_names": dataset.file_names.tolist(),
"classes": list(dataset.classes),
}
meta_path = out / "meta.json"
if meta_path.exists():
old = json.loads(meta_path.read_text())
if {k: old.get(k) for k in meta} != meta:
raise ValueError("existing output directory has different settings")
else:
meta_path.write_text(json.dumps(meta, indent=2))
def open_array(name: str, shape: tuple[int, ...], dtype) -> np.ndarray:
path = out / name
if path.exists():
return np.load(path, mmap_mode="r+")
return np.lib.format.open_memmap(path, mode="w+", dtype=dtype, shape=shape)
pooled = open_array("pooled.npy", (n, len(args.pooled_layers), 5, dim), np.float32)
patch_arrays = {
layer: open_array(f"patches_l{layer}.npy", (n, patches, dim), np.float16)
for layer in args.patch_layers
}
done_path = out / "done.npy"
done = np.load(done_path) if done_path.exists() else np.zeros(n, dtype=bool)
start = time.time()
processed = 0
for index in np.flatnonzero(~done):
with Image.open(dataset.image_paths[index]) as image:
x = preprocess(image, args.width, args.height, args.mode, args.flip)
with torch.inference_mode(), torch.autocast("cpu", dtype=torch.bfloat16, enabled=args.bf16):
output = model(pixel_values=torch.from_numpy(x)[None], output_hidden_states=True)
for slot, layer in enumerate(args.pooled_layers):
hidden = output.last_hidden_state if layer == depth else output.hidden_states[layer]
hidden = hidden[0].float()
tokens = hidden[1 + registers :]
pooled[index, slot] = torch.stack((
hidden[0],
hidden[1 : 1 + registers].mean(0),
tokens.mean(0),
tokens.amax(0),
tokens.std(0, unbiased=False),
)).numpy()
for layer, array in patch_arrays.items():
hidden = output.last_hidden_state if layer == depth else output.hidden_states[layer]
array[index] = hidden[0, 1 + registers :].float().numpy().astype(np.float16)
done[index] = True
processed += 1
if processed % 20 == 0 or done.all():
pooled.flush()
for array in patch_arrays.values():
array.flush()
np.save(done_path, done)
rate = (time.time() - start) / processed
print(f"{done.sum()}/{n} done, {rate:.2f}s/img, eta {rate * (~done).sum() / 60:.1f} min", flush=True)
pooled.flush()
for array in patch_arrays.values():
array.flush()
np.save(done_path, done)
print("complete", int(done.sum()), "of", n)
if __name__ == "__main__":
main()