#!/usr/bin/env python3 """Extract frozen four-view DINOv3 CLS and patch-mean features.""" from __future__ import annotations import argparse import fcntl import hashlib import json import os import platform import subprocess import tempfile import time from pathlib import Path import numpy as np import torch from PIL import Image, ImageOps from transformers import AutoImageProcessor, AutoModel from dataset_utils import load_dataset FEATURE_SCHEMA = "dinov3_four_view_cls_patchmean_mean_max_v1" def resize_height(image: Image.Image, height: int) -> Image.Image: width = round(image.width * height / image.height) return image.resize((width, height), Image.Resampling.BICUBIC) def letterbox(image: Image.Image, size: int) -> Image.Image: resized = ImageOps.contain(image, (size, size), Image.Resampling.BICUBIC) array = np.asarray(image, dtype=np.float32) background = tuple(np.median(array.reshape(-1, 3), axis=0).astype(np.uint8)) canvas = Image.new("RGB", (size, size), background) canvas.paste(resized, ((size - resized.width) // 2, (size - resized.height) // 2)) return canvas def four_views(image: Image.Image, size: int = 224) -> list[Image.Image]: image = image.convert("RGB") wide = resize_height(image, size) starts = (0, (wide.width - size) // 2, wide.width - size) return [letterbox(image, size)] + [ wide.crop((start, 0, start + size, size)) for start in starts ] def file_sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def atomic_savez(path: Path, **arrays: np.ndarray) -> None: """Write a complete ZIP beside the target, then atomically publish it.""" path.parent.mkdir(parents=True, exist_ok=True) descriptor, temporary = tempfile.mkstemp( prefix=f".{path.name}.", suffix=".tmp.npz", dir=path.parent ) os.close(descriptor) temporary_path = Path(temporary) try: np.savez_compressed(temporary_path, **arrays) os.replace(temporary_path, path) finally: temporary_path.unlink(missing_ok=True) def atomic_write_json(path: Path, value: dict[str, object]) -> None: path.parent.mkdir(parents=True, exist_ok=True) descriptor, temporary = tempfile.mkstemp( prefix=f".{path.name}.", suffix=".tmp", dir=path.parent ) temporary_path = Path(temporary) try: with os.fdopen(descriptor, "w", encoding="utf-8") as handle: json.dump(value, handle, ensure_ascii=False, indent=2) handle.write("\n") handle.flush() os.fsync(handle.fileno()) os.replace(temporary_path, path) finally: temporary_path.unlink(missing_ok=True) def build_cache_identity(dataset: object, model_path: Path) -> tuple[str, dict[str, str], str]: model_files = ["model.safetensors", "config.json", "preprocessor_config.json"] model_digests = { name: file_sha256(model_path / name) for name in model_files if (model_path / name).is_file() } if "model.safetensors" not in model_digests: raise FileNotFoundError(f"missing model weights under {model_path}") source = hashlib.sha256() source.update(FEATURE_SCHEMA.encode()) source.update(json.dumps(dataset.classes, ensure_ascii=False).encode()) source.update(np.asarray(dataset.labels, dtype=np.uint8).tobytes()) for name, path in zip(dataset.file_names, dataset.image_paths): source.update(str(name).encode()) source.update(bytes.fromhex(file_sha256(path))) source_digest = source.hexdigest() identity = hashlib.sha256() identity.update(FEATURE_SCHEMA.encode()) identity.update(json.dumps(model_digests, sort_keys=True).encode()) identity.update(source_digest.encode()) return identity.hexdigest(), model_digests, source_digest def source_digest_prefix(dataset: object, count: int) -> str: source = hashlib.sha256() source.update(FEATURE_SCHEMA.encode()) source.update(json.dumps(dataset.classes, ensure_ascii=False).encode()) source.update(np.asarray(dataset.labels[:count], dtype=np.uint8).tobytes()) for name, path in zip(dataset.file_names[:count], dataset.image_paths[:count]): source.update(str(name).encode()) source.update(bytes.fromhex(file_sha256(path))) return source.hexdigest() def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--dataset", type=Path, required=True) parser.add_argument("--model", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--batch-size", type=int, default=2) parser.add_argument("--threads", type=int, default=4) parser.add_argument("--num-shards", type=int, default=1) parser.add_argument("--shard-index", type=int, default=0) parser.add_argument( "--reuse-features", type=Path, help="Verified prior feature NPZ whose rows are a prefix of this dataset", ) args = parser.parse_args() if args.num_shards < 1 or not 0 <= args.shard_index < args.num_shards: raise ValueError("shard-index must be in [0, num-shards)") dataset = load_dataset(args.dataset) torch.set_num_threads(args.threads) processor = AutoImageProcessor.from_pretrained(args.model, local_files_only=True) model = AutoModel.from_pretrained(args.model, local_files_only=True).eval() register_tokens = int(model.config.num_register_tokens) hidden_size = int(model.config.hidden_size) started = time.time() cache_key, model_digests, source_digest = build_cache_identity(dataset, args.model) parts_dir = args.output.parent / f"{args.output.stem}.parts.{cache_key[:16]}" parts_dir.mkdir(parents=True, exist_ok=True) all_starts = list(range(0, len(dataset.image_paths), args.batch_size)) reused_features: np.ndarray | None = None reused_count = 0 if args.reuse_features is not None: prior_metadata = json.loads(args.reuse_features.with_suffix(".json").read_text(encoding="utf-8")) with np.load(args.reuse_features) as prior: prior_names = prior["file_names"] prior_labels = prior["labels"] prior_classes = prior["classes"] reused_features = prior["features"].astype(np.float32) reused_count = len(prior_names) if prior_metadata.get("feature_type") != FEATURE_SCHEMA: raise ValueError("reused feature schema mismatch") if prior_metadata.get("model_file_sha256") != model_digests: raise ValueError("reused feature backbone mismatch") if not np.array_equal(prior_classes, np.asarray(dataset.classes)): raise ValueError("reused feature class order mismatch") if not np.array_equal(prior_names, dataset.file_names[:reused_count]): raise ValueError("reused feature filenames are not a dataset prefix") if not np.array_equal(prior_labels, dataset.labels[:reused_count]): raise ValueError("reused feature labels mismatch") if prior_metadata.get("source_digest") != source_digest_prefix(dataset, reused_count): raise ValueError("reused feature source bytes mismatch") if reused_features.shape != (reused_count, hidden_size * 4): raise ValueError("reused feature shape mismatch") print(f"Verified {reused_count} reusable feature rows", flush=True) assigned_starts = [ start for batch_index, start in enumerate(all_starts) if batch_index % args.num_shards == args.shard_index ] for start in assigned_starts: paths = dataset.image_paths[start:start + args.batch_size] end = start + len(paths) part_path = parts_dir / f"{start:04d}-{end:04d}.npz" if part_path.is_file(): with np.load(part_path) as part: expected_names = dataset.file_names[start:end] if str(part["cache_key"]) != cache_key: raise ValueError(f"cached feature provenance mismatch: {part_path}") if not np.array_equal(part["file_names"], expected_names): raise ValueError(f"cached feature filename mismatch: {part_path}") print(f"DINOv3 {end}/{len(dataset.image_paths)} (cached)", flush=True) continue if reused_features is not None and end <= reused_count: atomic_savez( part_path, cache_key=np.asarray(cache_key), file_names=dataset.file_names[start:end], features=reused_features[start:end], ) print(f"DINOv3 {end}/{len(dataset.image_paths)} (reused)", flush=True) continue views: list[Image.Image] = [] for path in paths: with Image.open(path) as image: views.extend(four_views(image)) inputs = processor( images=views, return_tensors="pt", do_resize=False, do_center_crop=False ) with torch.inference_mode(): hidden = model(**inputs).last_hidden_state cls = hidden[:, 0] patch_mean = hidden[:, 1 + register_tokens:].mean(dim=1) vectors = torch.cat((cls, patch_mean), dim=1).reshape(len(paths), 4, -1) vectors = torch.cat((vectors.mean(dim=1), vectors.amax(dim=1)), dim=1) vectors = torch.nn.functional.normalize(vectors, dim=1) part_features = vectors.cpu().numpy().astype(np.float32) atomic_savez( part_path, cache_key=np.asarray(cache_key), file_names=dataset.file_names[start:end], features=part_features, ) print(f"DINOv3 {end}/{len(dataset.image_paths)}", flush=True) expected_parts = [ parts_dir / f"{start:04d}-{min(start + args.batch_size, len(dataset.image_paths)):04d}.npz" for start in all_starts ] missing_parts = [path for path in expected_parts if not path.is_file()] if missing_parts: print( f"Shard {args.shard_index}/{args.num_shards} complete; " f"waiting for {len(missing_parts)} parts from other shards.", flush=True ) return lock_path = args.output.with_suffix(args.output.suffix + ".lock") lock_path.parent.mkdir(parents=True, exist_ok=True) with lock_path.open("w") as lock: fcntl.flock(lock, fcntl.LOCK_EX) blocks = [] for start, part_path in zip(all_starts, expected_parts): with np.load(part_path) as part: end = min(start + args.batch_size, len(dataset.image_paths)) if str(part["cache_key"]) != cache_key: raise ValueError(f"cached feature provenance mismatch: {part_path}") if not np.array_equal(part["file_names"], dataset.file_names[start:end]): raise ValueError(f"cached feature filename mismatch: {part_path}") blocks.append(part["features"].astype(np.float32)) features = np.concatenate(blocks) expected = hidden_size * 4 if features.shape != (len(dataset.image_paths), expected): raise ValueError(f"unexpected feature shape {features.shape}, expected (*, {expected})") atomic_savez( args.output, file_names=dataset.file_names, features=features, labels=dataset.labels, classes=np.asarray(dataset.classes), cache_key=np.asarray(cache_key), feature_schema=np.asarray(FEATURE_SCHEMA), source_digest=np.asarray(source_digest), model_file_sha256_json=np.asarray(json.dumps(model_digests, sort_keys=True)), ) try: commit = subprocess.check_output( ["git", "-C", str(args.model), "rev-parse", "HEAD"], text=True ).strip() except (subprocess.CalledProcessError, FileNotFoundError): commit = None metadata = { "feature_type": FEATURE_SCHEMA, "cache_key": cache_key, "source_digest": source_digest, "model_path": str(args.model.resolve()), "model_commit": commit, "model_sha256": model_digests["model.safetensors"], "model_file_sha256": model_digests, "samples": len(dataset.image_paths), "shape": list(features.shape), "hidden_size": hidden_size, "num_register_tokens": register_tokens, "patch_size": int(model.config.patch_size), "batch_size": args.batch_size, "threads": args.threads, "num_shards": args.num_shards, "reused_feature_rows": reused_count, "reused_feature_path": str(args.reuse_features.resolve()) if args.reuse_features else None, "seconds": round(time.time() - started, 3), "python": platform.python_version(), "torch": torch.__version__, } atomic_write_json(args.output.with_suffix(".json"), metadata) print(json.dumps(metadata, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()