papi_cc / scripts /compute_cluster_centers.py
CJR61's picture
Claude Sonnet 4.6
Replace B3 ONNX with EfficientNet-B0 (3-fold CV, orig fold_3)
a5ec00a
Raw History Blame Contribute Delete
4.58 kB
"""
compute_cluster_centers.py
--------------------------
Extract embeddings from all training images, run k-means (k=5),
save cluster centers to resource/cluster_centers.npy.
Usage (from repo root):
python scripts/compute_cluster_centers.py
Requirements: onnxruntime, opencv-python, numpy, scikit-learn
"""
import os
import sys
import numpy as np
import cv2
from pathlib import Path
# ── Config ────────────────────────────────────────────────────────────────────
REPO_ROOT = Path(__file__).parent.parent
DATA_DIR = Path("/mnt/d/pig/data") # training images root
MODEL_PATH = REPO_ROOT / "resource" / "model_b0_balanced_v2_emb.onnx"
OUT_PATH = REPO_ROOT / "resource" / "cluster_centers.npy"
K = 5
IMG_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".tif"}
# ──────────────────────────────────────────────────────────────────────────────
def preprocess(image_path: str) -> np.ndarray:
img = cv2.imread(image_path)
if img is None:
return None
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
# Match inference preprocessing: Resize(256) + CenterCrop(224)
h, w = img.shape[:2]
if h < w:
new_h, new_w = 256, int(w * 256 / h)
else:
new_h, new_w = int(h * 256 / w), 256
img = cv2.resize(img, (new_w, new_h))
y0 = (new_h - 224) // 2
x0 = (new_w - 224) // 2
img = img[y0:y0+224, x0:x0+224]
img = img / 255.0
mean = np.array([0.485, 0.456, 0.406])
std = np.array([0.229, 0.224, 0.225])
img = (img - mean) / std
img = np.transpose(img, (2, 0, 1))
return np.expand_dims(img, 0).astype(np.float32)
def extract_embedding(sess, image_path: str) -> np.ndarray | None:
x = preprocess(image_path)
if x is None:
return None
inp_name = sess.get_inputs()[0].name
out_map = {o.name: i for i, o in enumerate(sess.get_outputs())}
if "embedding" not in out_map:
raise KeyError(f"Model has no 'embedding' output. Outputs: {list(out_map)}")
outs = sess.run(None, {inp_name: x})
return outs[out_map["embedding"]].squeeze().astype(np.float32)
def collect_images(root: Path) -> list[str]:
paths = []
for p in root.rglob("*"):
if p.suffix.lower() in IMG_EXTS:
paths.append(str(p))
return sorted(paths)
def main():
import onnxruntime as ort
from sklearn.cluster import KMeans
print(f"Model : {MODEL_PATH}")
print(f"Data : {DATA_DIR}")
print(f"Output : {OUT_PATH}")
print(f"k : {K}")
print()
sess = ort.InferenceSession(str(MODEL_PATH),
providers=["CUDAExecutionProvider", "CPUExecutionProvider"])
images = collect_images(DATA_DIR)
print(f"Found {len(images)} images β€” extracting embeddings...")
embeddings, failed = [], []
for i, path in enumerate(images, 1):
emb = extract_embedding(sess, path)
if emb is not None:
embeddings.append(emb)
else:
failed.append(path)
if i % 100 == 0 or i == len(images):
print(f" {i}/{len(images)} ok={len(embeddings)} fail={len(failed)}", end="\r")
print(f"\nExtracted {len(embeddings)} embeddings ({len(failed)} failed)")
if failed:
print("Failed files:")
for f in failed[:5]:
print(f" {f}")
X = np.stack(embeddings) # (N, 1536)
print(f"\nRunning k-means (k={K}) on {X.shape} ...")
km = KMeans(n_clusters=K, random_state=42, n_init=10)
km.fit(X)
centers = km.cluster_centers_.astype(np.float32) # (K, 1536)
# L2-normalise so cosine similarity == dot product
norms = np.linalg.norm(centers, axis=1, keepdims=True)
centers = centers / (norms + 1e-8)
np.save(str(OUT_PATH), centers)
print(f"Saved {centers.shape} cluster centers β†’ {OUT_PATH}")
# Quick sanity: show avg max-sim of training images to centers
sims = X / (np.linalg.norm(X, axis=1, keepdims=True) + 1e-8)
max_sims = (sims @ centers.T).max(axis=1)
print(f"\nSanity check (training set max-sim to nearest center):")
print(f" mean={max_sims.mean():.3f} min={max_sims.min():.3f} "
f"p5={np.percentile(max_sims, 5):.3f} p10={np.percentile(max_sims, 10):.3f}")
print("Done.")
if __name__ == "__main__":
main()