#!/usr/bin/env python3 """Region-level test of damage detection: does the robot get one task per real patch? Pixel IoU says little about what a robot does with damage. It needs each peeling patch or rust stain found **as one region** that covers the whole patch (the edge still has to be scraped), and no regions on sound walls. Against the pixel masks in ``condition_masks//`` this reports, per method: * ``region_recall`` - share of true patches (>= ``MIN_AREA_PX``) whose pixels a prediction covers at least half of; * ``region_precision`` - share of predicted regions with at least half their pixels on true damage (3 px tolerance); * ``fragments_per_found_patch`` - predicted regions touching each found patch (1.0 is ideal: one patch, one tool task); * ``pixel_recall``/``pixel_iou`` - how much of the damage the regions cover; * ``sound_image_region_rate`` - share of damage-free images given any region, and the mean regions per such image, separately for real photos. Methods: ``argmax`` is the old rule (confident argmax pixels, speckle removed, connected components); ``regions:low=..,gap=..`` is ``regions.damage_regions``. Everything runs at the model's resolution. Tune on ``val``; report ``test``. Usage:: python3 eval_regions.py --checkpoint artifacts/best.pt --data real_data --split val --report regions_val.json """ import argparse import csv import json from pathlib import Path import numpy as np from scipy import ndimage from regions import MIN_AREA_PX, damage_regions LOWS = (None, 0.4, "off") # None = default offset below high; "off" = no growth (low = high) GAPS = (0, 8, 12, 20) TOLERANCE_PX = 3 def components(mask, min_area=1): labels, count = ndimage.label(mask, structure=np.ones((3, 3), bool)) if not count: return labels, [] sizes = np.bincount(labels.ravel(), minlength=count + 1) return labels, [i for i in range(1, count + 1) if sizes[i] >= min_area] def argmax_mask(probability, wall, high): """The old rule: per-pixel argmax over sound/paint failure/rust, confident, speckle removed.""" damage = np.asarray(probability, np.float32) sound = 1.0 - damage.sum(axis=0) winner = np.argmax(np.concatenate([sound[None], damage]), axis=0) confidence = np.max(np.concatenate([sound[None], damage]), axis=0) out = np.zeros_like(wall) for index in (1, 2): out |= ndimage.binary_opening((winner == index) & (confidence >= high) & wall, iterations=1) return out def score_image(pred, truth, wall): """Counts for one image; ``truth`` is a bool damage mask on the wall.""" pred, truth = pred & wall, truth & wall p_labels, p_ids = components(pred, MIN_AREA_PX) t_labels, t_ids = components(truth, MIN_AREA_PX) kept = np.isin(p_labels, p_ids) near_truth = ndimage.binary_dilation(truth, iterations=TOLERANCE_PX) if truth.any() else truth correct = sum(near_truth[p_labels == i].mean() >= 0.5 for i in p_ids) found, fragments = 0, [] for i in t_ids: patch = t_labels == i if kept[patch].mean() >= 0.5: found += 1 fragments.append(len(set(np.unique(p_labels[patch & kept])) - {0})) return {"pred_regions": len(p_ids), "pred_correct": int(correct), "true_patches": len(t_ids), "found": found, "fragments": fragments, "tp_px": int((kept & truth).sum()), "pred_px": int(kept.sum()), "true_px": int(truth.sum())} def summarise(per_image): def total(key): return sum(r[key] for r in per_image) damaged = [r for r in per_image if r["true_px"] > 0] sound = [r for r in per_image if r["true_px"] == 0] sound_real = [r for r in sound if r["real"]] frags = [f for r in per_image for f in r["fragments"]] def rate(group): return round(float(np.mean([r["pred_regions"] > 0 for r in group])), 4) if group else None union = total("pred_px") + total("true_px") - total("tp_px") return { "damaged_images": len(damaged), "sound_images": len(sound), "sound_real_images": len(sound_real), "true_patches": total("true_patches"), "region_recall": round(total("found") / total("true_patches"), 4) if total("true_patches") else None, "region_precision": (round(sum(r["pred_correct"] for r in damaged) / sum(r["pred_regions"] for r in damaged), 4) if sum(r["pred_regions"] for r in damaged) else None), "fragments_per_found_patch": round(float(np.mean(frags)), 3) if frags else None, "pixel_recall": round(total("tp_px") / total("true_px"), 4) if total("true_px") else None, "pixel_iou": round(total("tp_px") / union, 4) if union else None, "sound_image_region_rate": rate(sound), "sound_real_image_region_rate": rate(sound_real), "regions_per_sound_real_image": (round(float(np.mean([r["pred_regions"] for r in sound_real])), 3) if sound_real else None), } def methods(high): yield "argmax", lambda prob, wall: argmax_mask(prob, wall, high) for low in LOWS: for gap in GAPS: low_value = high if low == "off" else low name = f"regions:low={'off' if low == 'off' else (low or 'default')},gap={gap}" yield name, (lambda prob, wall, lv=low_value, g=gap: damage_regions(prob, wall, high=high, low=lv, merge_gap_px=g, describe=False)[1]) def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--checkpoint", type=Path, required=True) ap.add_argument("--data", type=Path, default=Path("real_data")) ap.add_argument("--split", default="val") ap.add_argument("--limit", type=int) ap.add_argument("--report", type=Path) args = ap.parse_args() import cv2 import torch from PIL import Image from predict import analyze, build_parser, condition_thresholds from train import load_checkpoint_model device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model, checkpoint = load_checkpoint_model(args.checkpoint, device) predict_args = build_parser().parse_args(["unused.jpg"]) predict_args.measure = False high, _, source = condition_thresholds(predict_args, checkpoint) meta = {} if (args.data / "metadata.csv").is_file(): meta = {r["image"]: r for r in csv.DictReader((args.data / "metadata.csv").open()) if r["split"] == args.split} cond_dir = args.data / "condition_masks" / args.split images = sorted(p for p in (args.data / args.split / "images").glob("*") if (cond_dir / (p.stem + ".png")).is_file()) per_method = {name: [] for name, _ in methods(high)} for path in images[:args.limit]: bgr = cv2.imread(str(path)) if bgr is None: continue _, masks = analyze(bgr, model, checkpoint, device, predict_args) prob = masks["condition_probability_small"] if prob is None: raise SystemExit(f"{args.checkpoint} has no condition head") size = (prob.shape[2], prob.shape[1]) wall = cv2.resize(masks["known_wall"].astype(np.uint8), size, interpolation=cv2.INTER_NEAREST) > 0 truth_full = np.asarray(Image.open(cond_dir / (path.stem + ".png"))) truth = cv2.resize(np.isin(truth_full, (1, 2)).astype(np.uint8), size, interpolation=cv2.INTER_NEAREST) > 0 real = (meta.get(path.name) or {}).get("source", "") != "synthetic" for name, method in methods(high): per_method[name].append({**score_image(method(prob, wall), truth, wall), "real": real}) print(json.dumps({"image": path.name, "real": real, "true_px": per_method["argmax"][-1]["true_px"], "argmax_regions": per_method["argmax"][-1]["pred_regions"]}), flush=True) report = {"checkpoint": str(args.checkpoint), "split": args.split, "high": high, "thresholds_from": source, "min_area_px": MIN_AREA_PX, "methods": {name: summarise(rows) for name, rows in per_method.items()}} print(json.dumps(report, indent=2)) if args.report: args.report.write_text(json.dumps(report, indent=2) + "\n") if __name__ == "__main__": main()