scripts/error_analysis.py

"""Break down out-of-fold errors by cause and by box size.

This is the script that decided the current priority: it separates "the model
never proposed this box" from "the model proposed it but the coordinates were
too loose", and reports the recall ceiling that perfect box regression would
reach.

    modal volume get pmdm-ckpt /oof/fold0_convnext_tiny.npz /tmp/oof.npz
    PMDM_DATA=Task1/PackagingMaterialDifferenceMiningDataset \
    .venv/bin/python scripts/error_analysis.py /tmp/oof.npz
"""
from __future__ import annotations

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))

import numpy as np  # noqa: E402

from pmdm.dataset import load_gt  # noqa: E402
from pmdm.metric import iou_matrix  # noqa: E402

SIZE_BINS = [(0, 12, "<12px"), (12, 24, "12-24px"), (24, 10 ** 9, ">24px")]


def main(npz_path: str) -> None:
    z = np.load(npz_path)
    gt = load_gt()
    keys = sorted({k.split("__")[0] for k in z.files})

    best_iou, sizes = [], []
    n_candidates = 0
    for key in keys:
        idx = int(key.split("_")[-1])
        boxes = z[f"{key}__boxes"]
        g = gt[idx]
        n_candidates += len(boxes)
        m = iou_matrix(boxes, g) if len(boxes) and len(g) else np.zeros((len(boxes), len(g)))
        for j in range(len(g)):
            best_iou.append(float(m[:, j].max()) if m.size else 0.0)
            sizes.append(max(g[j][2] - g[j][0], g[j][3] - g[j][1]))

    best_iou, sizes = np.array(best_iou), np.array(sizes)
    print(f"pairs: {len(keys)}, ground-truth boxes: {len(best_iou)}, "
          f"candidates per image: {n_candidates / max(1, len(keys)):.1f}")
    print(f"matched at IoU >= 0.5:        {int((best_iou >= 0.5).sum())}")
    print(f"proposed but IoU < 0.5:       {int(((best_iou > 0) & (best_iou < 0.5)).sum())}")
    print(f"never proposed (IoU == 0):    {int((best_iou == 0).sum())}")
    print(f"recall ceiling with perfect boxes: {float((best_iou > 0).mean()):.4f}")

    print("\nby box size (longest side):")
    for lo, hi, name in SIZE_BINS:
        m = (sizes >= lo) & (sizes < hi)
        if not m.sum():
            continue
        print(f"  {name:9s} n={int(m.sum()):4d}  "
              f"matched={float((best_iou[m] >= 0.5).mean()):.3f}  "
              f"never_proposed={int((best_iou[m] == 0).sum())}")


if __name__ == "__main__":
    main(sys.argv[1] if len(sys.argv) > 1 else "/tmp/oof.npz")