#!/usr/bin/env python3 """Window accuracy on REAL indoor photos: the number window verification never had. ``window_eval/labels.jsonl`` holds 51 real room photos from the two reviewed real-photo sets (never trained on): 34 with windows traced as polygons (percent of width/height), 17 without -- 15 plain negatives and 2 whose only window is curtained, kept as ignore regions so they score false windows elsewhere in the frame. ``ignore`` polygons (glazed doors, glass balustrades, curtained glass) count neither way. Labels are a first pass traced by eye on a 5 % grid; their ``status`` says so, and a human check should come before any number here is published. Metrics per post-processing variant (eval_geometry's variants, same cached network pass): window IoU / precision / recall (pixels, ignore excluded); window object P/R/F1 (components, IoU >= 0.5, and the looser 0.3 since hand-traced polygons are coarse); false-window rate = share of window-free photos with any predicted window blob >= 0.5 % of the frame. python eval_windows_indoor.py --checkpoint artifacts/candidates/C2-cal.pt [--out benchmark/records/...json] """ from __future__ import annotations import argparse import json from pathlib import Path import cv2 import numpy as np import torch from scipy import ndimage import eval_geometry as eg import window_negatives as wn ROOT = Path(__file__).resolve().parent LABELS = ROOT / "window_eval" / "labels.jsonl" VARIANTS = {k: eg.VARIANTS[k] for k in ("C B + flip TTA", "D C + window verify", "H D + snap openings to edges")} # J = D + look-alike / frame gate (window_negatives.py); needs the ADE20K maps, passed in via a closure per image. GATED = "J D + look-alike/frame gate" ADE_CACHE = ROOT / ".geom_cache" def ade_cached(rgb, stem): f = ADE_CACHE / f"ade_indoor_{stem}.npz" if f.is_file(): z = np.load(f) return z["win"].astype(np.float32), z["look"].astype(np.float32) win, look = wn.ade_maps(rgb) ADE_CACHE.mkdir(exist_ok=True) np.savez_compressed(f, win=win.astype(np.float16), look=look.astype(np.float16)) return win, look MIN_BLOB = 0.005 def rasterise(polys, shape): h, w = shape m = np.zeros(shape, np.uint8) for p in polys: cv2.fillPoly(m, [np.round(np.asarray(p, np.float64) * [w / 100.0, h / 100.0]).astype(np.int32)], 1) return m.astype(bool) def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--checkpoint", type=Path, required=True) ap.add_argument("--out", type=Path) ap.add_argument("--split", choices=("all", "dev", "test"), default="all", help="dev = even rows (fit the gate), test = odd rows (report it); all = every row") ap.add_argument("--margin", type=float, default=wn.LOOKALIKE_MARGIN) ap.add_argument("--min-frame", type=float, default=wn.MIN_FRAME) a = ap.parse_args() from train import load_checkpoint_model device = torch.device("cpu") model, checkpoint = load_checkpoint_model(a.checkpoint, device) model.eval() rows = [json.loads(line) for line in LABELS.open()] if a.split != "all": rows = rows[(0 if a.split == "dev" else 1)::2] variants = {**VARIANTS, GATED: None} acc = {k: {"i": 0, "u": 0, "p": 0, "g": 0, "tp5": 0, "tp3": 0, "np": 0, "ng": 0, "neg": 0, "neg_fp": 0} for k in variants} for n, r in enumerate(rows, 1): bgr = cv2.imread(str(ROOT / r["image"])) rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB) shape = rgb.shape[:2] gt, ignore = rasterise(r["windows"], shape), rasterise(r["ignore"], shape) probs = eg.cached_forward(model, checkpoint, a.checkpoint.stem, "indoor", {"image": Path(r["image"]).name}, rgb, device) ade = ade_cached(rgb, Path(r["image"]).stem) for name, fn in variants.items(): if fn is None: cls, _ = wn.filter_windows(VARIANTS["D C + window verify"](probs, shape, rgb), rgb, ade, a.margin, a.min_frame) else: cls = fn(probs, shape, rgb) pred = (cls == eg.WINDOW) & ~ignore s = acc[name] s["i"] += int((pred & gt).sum()); s["u"] += int((pred | gt).sum()) s["p"] += int(pred.sum()); s["g"] += int(gt.sum()) for iou, key in ((0.5, "tp5"), (0.3, "tp3")): tp, npred, ngt = eg.object_f1(pred, gt, iou) s[key] += tp s["np"] += npred; s["ng"] += ngt if not r["windows"]: lab, k = ndimage.label(pred) sizes = ndimage.sum(pred, lab, range(1, k + 1)) if k else [] s["neg"] += 1 s["neg_fp"] += int(any(sz >= MIN_BLOB * pred.size for sz in sizes)) if n % 10 == 0: print(f"{n}/{len(rows)}", flush=True) f = lambda x, y: round(x / y, 4) if y else None res = {"checkpoint": a.checkpoint.name, "split": a.split, "margin": a.margin, "min_frame": a.min_frame, "images": len(rows), "with_windows": sum(1 for r in rows if r["windows"]), "labels_status": rows[0].get("status"), "variants": {}} for name, s in acc.items(): out = {"window_iou": f(s["i"], s["u"]), "window_precision": f(s["i"], s["p"]), "window_recall": f(s["i"], s["g"]), "false_window_rate_on_window_free_photos": f(s["neg_fp"], s["neg"])} for key, label in (("tp5", "objects@0.5"), ("tp3", "objects@0.3")): p, rc = f(s[key], s["np"]), f(s[key], s["ng"]) out[label] = {"precision": p, "recall": rc, "f1": round(2 * p * rc / (p + rc), 4) if p and rc else 0.0, "predicted": s["np"], "labelled": s["ng"]} res["variants"][name] = out print(f"{name:32s} IoU {out['window_iou']} P {out['window_precision']} R {out['window_recall']} | " f"obj@0.5 F1 {out['objects@0.5']['f1']} obj@0.3 F1 {out['objects@0.3']['f1']} | " f"false windows {out['false_window_rate_on_window_free_photos']}") if a.out: a.out.write_text(json.dumps(res, indent=1)) if __name__ == "__main__": main()