File size: 2,420 Bytes
d667566 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | """Break down out-of-fold errors by cause and by box size.
This is the script that decided the current priority: it separates "the model
never proposed this box" from "the model proposed it but the coordinates were
too loose", and reports the recall ceiling that perfect box regression would
reach.
modal volume get pmdm-ckpt /oof/fold0_convnext_tiny.npz /tmp/oof.npz
PMDM_DATA=Task1/PackagingMaterialDifferenceMiningDataset \
.venv/bin/python scripts/error_analysis.py /tmp/oof.npz
"""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
import numpy as np # noqa: E402
from pmdm.dataset import load_gt # noqa: E402
from pmdm.metric import iou_matrix # noqa: E402
SIZE_BINS = [(0, 12, "<12px"), (12, 24, "12-24px"), (24, 10 ** 9, ">24px")]
def main(npz_path: str) -> None:
z = np.load(npz_path)
gt = load_gt()
keys = sorted({k.split("__")[0] for k in z.files})
best_iou, sizes = [], []
n_candidates = 0
for key in keys:
idx = int(key.split("_")[-1])
boxes = z[f"{key}__boxes"]
g = gt[idx]
n_candidates += len(boxes)
m = iou_matrix(boxes, g) if len(boxes) and len(g) else np.zeros((len(boxes), len(g)))
for j in range(len(g)):
best_iou.append(float(m[:, j].max()) if m.size else 0.0)
sizes.append(max(g[j][2] - g[j][0], g[j][3] - g[j][1]))
best_iou, sizes = np.array(best_iou), np.array(sizes)
print(f"pairs: {len(keys)}, ground-truth boxes: {len(best_iou)}, "
f"candidates per image: {n_candidates / max(1, len(keys)):.1f}")
print(f"matched at IoU >= 0.5: {int((best_iou >= 0.5).sum())}")
print(f"proposed but IoU < 0.5: {int(((best_iou > 0) & (best_iou < 0.5)).sum())}")
print(f"never proposed (IoU == 0): {int((best_iou == 0).sum())}")
print(f"recall ceiling with perfect boxes: {float((best_iou > 0).mean()):.4f}")
print("\nby box size (longest side):")
for lo, hi, name in SIZE_BINS:
m = (sizes >= lo) & (sizes < hi)
if not m.sum():
continue
print(f" {name:9s} n={int(m.sum()):4d} "
f"matched={float((best_iou[m] >= 0.5).mean()):.3f} "
f"never_proposed={int((best_iou[m] == 0).sum())}")
if __name__ == "__main__":
main(sys.argv[1] if len(sys.argv) > 1 else "/tmp/oof.npz")
|