painting-vision-robotics-kit / eval_windows_indoor.py
constructelligence's picture
Update code: real-machine G-code adapter, commissioning gate, machine_job runner, pump watchdog firmware, twin completion gate, realism and masking modules (card and weights unchanged)
4eecd03 verified
Raw History Blame Contribute Delete
6.14 kB
#!/usr/bin/env python3
"""Window accuracy on REAL indoor photos: the number window verification never had.
``window_eval/labels.jsonl`` holds 51 real room photos from the two reviewed real-photo sets (never trained on):
34 with windows traced as polygons (percent of width/height), 17 without -- 15 plain negatives and 2 whose only window
is curtained, kept as ignore regions so they score false windows elsewhere in the frame. ``ignore`` polygons
(glazed doors, glass balustrades, curtained glass) count neither way. Labels are a first pass traced by eye on a 5 %
grid; their ``status`` says so, and a human check should come before any number here is published.
Metrics per post-processing variant (eval_geometry's variants, same cached network pass):
window IoU / precision / recall (pixels, ignore excluded); window object P/R/F1 (components, IoU >= 0.5, and the
looser 0.3 since hand-traced polygons are coarse); false-window rate = share of window-free photos with any
predicted window blob >= 0.5 % of the frame.
python eval_windows_indoor.py --checkpoint artifacts/candidates/C2-cal.pt [--out benchmark/records/...json]
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import cv2
import numpy as np
import torch
from scipy import ndimage
import eval_geometry as eg
import window_negatives as wn
ROOT = Path(__file__).resolve().parent
LABELS = ROOT / "window_eval" / "labels.jsonl"
VARIANTS = {k: eg.VARIANTS[k] for k in ("C B + flip TTA", "D C + window verify", "H D + snap openings to edges")}
# J = D + look-alike / frame gate (window_negatives.py); needs the ADE20K maps, passed in via a closure per image.
GATED = "J D + look-alike/frame gate"
ADE_CACHE = ROOT / ".geom_cache"
def ade_cached(rgb, stem):
f = ADE_CACHE / f"ade_indoor_{stem}.npz"
if f.is_file():
z = np.load(f)
return z["win"].astype(np.float32), z["look"].astype(np.float32)
win, look = wn.ade_maps(rgb)
ADE_CACHE.mkdir(exist_ok=True)
np.savez_compressed(f, win=win.astype(np.float16), look=look.astype(np.float16))
return win, look
MIN_BLOB = 0.005
def rasterise(polys, shape):
h, w = shape
m = np.zeros(shape, np.uint8)
for p in polys:
cv2.fillPoly(m, [np.round(np.asarray(p, np.float64) * [w / 100.0, h / 100.0]).astype(np.int32)], 1)
return m.astype(bool)
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--checkpoint", type=Path, required=True)
ap.add_argument("--out", type=Path)
ap.add_argument("--split", choices=("all", "dev", "test"), default="all",
help="dev = even rows (fit the gate), test = odd rows (report it); all = every row")
ap.add_argument("--margin", type=float, default=wn.LOOKALIKE_MARGIN)
ap.add_argument("--min-frame", type=float, default=wn.MIN_FRAME)
a = ap.parse_args()
from train import load_checkpoint_model
device = torch.device("cpu")
model, checkpoint = load_checkpoint_model(a.checkpoint, device)
model.eval()
rows = [json.loads(line) for line in LABELS.open()]
if a.split != "all":
rows = rows[(0 if a.split == "dev" else 1)::2]
variants = {**VARIANTS, GATED: None}
acc = {k: {"i": 0, "u": 0, "p": 0, "g": 0, "tp5": 0, "tp3": 0, "np": 0, "ng": 0, "neg": 0, "neg_fp": 0} for k in variants}
for n, r in enumerate(rows, 1):
bgr = cv2.imread(str(ROOT / r["image"]))
rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
shape = rgb.shape[:2]
gt, ignore = rasterise(r["windows"], shape), rasterise(r["ignore"], shape)
probs = eg.cached_forward(model, checkpoint, a.checkpoint.stem, "indoor", {"image": Path(r["image"]).name}, rgb, device)
ade = ade_cached(rgb, Path(r["image"]).stem)
for name, fn in variants.items():
if fn is None:
cls, _ = wn.filter_windows(VARIANTS["D C + window verify"](probs, shape, rgb), rgb, ade, a.margin, a.min_frame)
else:
cls = fn(probs, shape, rgb)
pred = (cls == eg.WINDOW) & ~ignore
s = acc[name]
s["i"] += int((pred & gt).sum()); s["u"] += int((pred | gt).sum())
s["p"] += int(pred.sum()); s["g"] += int(gt.sum())
for iou, key in ((0.5, "tp5"), (0.3, "tp3")):
tp, npred, ngt = eg.object_f1(pred, gt, iou)
s[key] += tp
s["np"] += npred; s["ng"] += ngt
if not r["windows"]:
lab, k = ndimage.label(pred)
sizes = ndimage.sum(pred, lab, range(1, k + 1)) if k else []
s["neg"] += 1
s["neg_fp"] += int(any(sz >= MIN_BLOB * pred.size for sz in sizes))
if n % 10 == 0:
print(f"{n}/{len(rows)}", flush=True)
f = lambda x, y: round(x / y, 4) if y else None
res = {"checkpoint": a.checkpoint.name, "split": a.split, "margin": a.margin, "min_frame": a.min_frame, "images": len(rows), "with_windows": sum(1 for r in rows if r["windows"]),
"labels_status": rows[0].get("status"), "variants": {}}
for name, s in acc.items():
out = {"window_iou": f(s["i"], s["u"]), "window_precision": f(s["i"], s["p"]), "window_recall": f(s["i"], s["g"]),
"false_window_rate_on_window_free_photos": f(s["neg_fp"], s["neg"])}
for key, label in (("tp5", "objects@0.5"), ("tp3", "objects@0.3")):
p, rc = f(s[key], s["np"]), f(s[key], s["ng"])
out[label] = {"precision": p, "recall": rc, "f1": round(2 * p * rc / (p + rc), 4) if p and rc else 0.0,
"predicted": s["np"], "labelled": s["ng"]}
res["variants"][name] = out
print(f"{name:32s} IoU {out['window_iou']} P {out['window_precision']} R {out['window_recall']} | "
f"obj@0.5 F1 {out['objects@0.5']['f1']} obj@0.3 F1 {out['objects@0.3']['f1']} | "
f"false windows {out['false_window_rate_on_window_free_photos']}")
if a.out:
a.out.write_text(json.dumps(res, indent=1))
if __name__ == "__main__":
main()