painting-vision-robotics-kit / eval_regions.py
constructelligence's picture
Upload eval_regions.py with huggingface_hub
93b7c15 verified
Raw History Blame Contribute Delete
8.24 kB
#!/usr/bin/env python3
"""Region-level test of damage detection: does the robot get one task per real patch?
Pixel IoU says little about what a robot does with damage. It needs each peeling
patch or rust stain found **as one region** that covers the whole patch (the edge
still has to be scraped), and no regions on sound walls. Against the pixel masks
in ``condition_masks/<split>/`` this reports, per method:
* ``region_recall`` - share of true patches (>= ``MIN_AREA_PX``) whose pixels a
prediction covers at least half of;
* ``region_precision`` - share of predicted regions with at least half their
pixels on true damage (3 px tolerance);
* ``fragments_per_found_patch`` - predicted regions touching each found patch
(1.0 is ideal: one patch, one tool task);
* ``pixel_recall``/``pixel_iou`` - how much of the damage the regions cover;
* ``sound_image_region_rate`` - share of damage-free images given any region,
and the mean regions per such image, separately for real photos.
Methods: ``argmax`` is the old rule (confident argmax pixels, speckle removed,
connected components); ``regions:low=..,gap=..`` is ``regions.damage_regions``.
Everything runs at the model's resolution. Tune on ``val``; report ``test``.
Usage::
python3 eval_regions.py --checkpoint artifacts/best.pt --data real_data --split val --report regions_val.json
"""
import argparse
import csv
import json
from pathlib import Path
import numpy as np
from scipy import ndimage
from regions import MIN_AREA_PX, damage_regions
LOWS = (None, 0.4, "off") # None = default offset below high; "off" = no growth (low = high)
GAPS = (0, 8, 12, 20)
TOLERANCE_PX = 3
def components(mask, min_area=1):
labels, count = ndimage.label(mask, structure=np.ones((3, 3), bool))
if not count:
return labels, []
sizes = np.bincount(labels.ravel(), minlength=count + 1)
return labels, [i for i in range(1, count + 1) if sizes[i] >= min_area]
def argmax_mask(probability, wall, high):
"""The old rule: per-pixel argmax over sound/paint failure/rust, confident, speckle removed."""
damage = np.asarray(probability, np.float32)
sound = 1.0 - damage.sum(axis=0)
winner = np.argmax(np.concatenate([sound[None], damage]), axis=0)
confidence = np.max(np.concatenate([sound[None], damage]), axis=0)
out = np.zeros_like(wall)
for index in (1, 2):
out |= ndimage.binary_opening((winner == index) & (confidence >= high) & wall, iterations=1)
return out
def score_image(pred, truth, wall):
"""Counts for one image; ``truth`` is a bool damage mask on the wall."""
pred, truth = pred & wall, truth & wall
p_labels, p_ids = components(pred, MIN_AREA_PX)
t_labels, t_ids = components(truth, MIN_AREA_PX)
kept = np.isin(p_labels, p_ids)
near_truth = ndimage.binary_dilation(truth, iterations=TOLERANCE_PX) if truth.any() else truth
correct = sum(near_truth[p_labels == i].mean() >= 0.5 for i in p_ids)
found, fragments = 0, []
for i in t_ids:
patch = t_labels == i
if kept[patch].mean() >= 0.5:
found += 1
fragments.append(len(set(np.unique(p_labels[patch & kept])) - {0}))
return {"pred_regions": len(p_ids), "pred_correct": int(correct), "true_patches": len(t_ids),
"found": found, "fragments": fragments, "tp_px": int((kept & truth).sum()),
"pred_px": int(kept.sum()), "true_px": int(truth.sum())}
def summarise(per_image):
def total(key):
return sum(r[key] for r in per_image)
damaged = [r for r in per_image if r["true_px"] > 0]
sound = [r for r in per_image if r["true_px"] == 0]
sound_real = [r for r in sound if r["real"]]
frags = [f for r in per_image for f in r["fragments"]]
def rate(group):
return round(float(np.mean([r["pred_regions"] > 0 for r in group])), 4) if group else None
union = total("pred_px") + total("true_px") - total("tp_px")
return {
"damaged_images": len(damaged), "sound_images": len(sound), "sound_real_images": len(sound_real),
"true_patches": total("true_patches"),
"region_recall": round(total("found") / total("true_patches"), 4) if total("true_patches") else None,
"region_precision": (round(sum(r["pred_correct"] for r in damaged) / sum(r["pred_regions"] for r in damaged), 4)
if sum(r["pred_regions"] for r in damaged) else None),
"fragments_per_found_patch": round(float(np.mean(frags)), 3) if frags else None,
"pixel_recall": round(total("tp_px") / total("true_px"), 4) if total("true_px") else None,
"pixel_iou": round(total("tp_px") / union, 4) if union else None,
"sound_image_region_rate": rate(sound),
"sound_real_image_region_rate": rate(sound_real),
"regions_per_sound_real_image": (round(float(np.mean([r["pred_regions"] for r in sound_real])), 3)
if sound_real else None),
}
def methods(high):
yield "argmax", lambda prob, wall: argmax_mask(prob, wall, high)
for low in LOWS:
for gap in GAPS:
low_value = high if low == "off" else low
name = f"regions:low={'off' if low == 'off' else (low or 'default')},gap={gap}"
yield name, (lambda prob, wall, lv=low_value, g=gap:
damage_regions(prob, wall, high=high, low=lv, merge_gap_px=g, describe=False)[1])
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--checkpoint", type=Path, required=True)
ap.add_argument("--data", type=Path, default=Path("real_data"))
ap.add_argument("--split", default="val")
ap.add_argument("--limit", type=int)
ap.add_argument("--report", type=Path)
args = ap.parse_args()
import cv2
import torch
from PIL import Image
from predict import analyze, build_parser, condition_thresholds
from train import load_checkpoint_model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model, checkpoint = load_checkpoint_model(args.checkpoint, device)
predict_args = build_parser().parse_args(["unused.jpg"])
predict_args.measure = False
high, _, source = condition_thresholds(predict_args, checkpoint)
meta = {}
if (args.data / "metadata.csv").is_file():
meta = {r["image"]: r for r in csv.DictReader((args.data / "metadata.csv").open()) if r["split"] == args.split}
cond_dir = args.data / "condition_masks" / args.split
images = sorted(p for p in (args.data / args.split / "images").glob("*") if (cond_dir / (p.stem + ".png")).is_file())
per_method = {name: [] for name, _ in methods(high)}
for path in images[:args.limit]:
bgr = cv2.imread(str(path))
if bgr is None:
continue
_, masks = analyze(bgr, model, checkpoint, device, predict_args)
prob = masks["condition_probability_small"]
if prob is None:
raise SystemExit(f"{args.checkpoint} has no condition head")
size = (prob.shape[2], prob.shape[1])
wall = cv2.resize(masks["known_wall"].astype(np.uint8), size, interpolation=cv2.INTER_NEAREST) > 0
truth_full = np.asarray(Image.open(cond_dir / (path.stem + ".png")))
truth = cv2.resize(np.isin(truth_full, (1, 2)).astype(np.uint8), size, interpolation=cv2.INTER_NEAREST) > 0
real = (meta.get(path.name) or {}).get("source", "") != "synthetic"
for name, method in methods(high):
per_method[name].append({**score_image(method(prob, wall), truth, wall), "real": real})
print(json.dumps({"image": path.name, "real": real, "true_px": per_method["argmax"][-1]["true_px"],
"argmax_regions": per_method["argmax"][-1]["pred_regions"]}), flush=True)
report = {"checkpoint": str(args.checkpoint), "split": args.split, "high": high, "thresholds_from": source,
"min_area_px": MIN_AREA_PX, "methods": {name: summarise(rows) for name, rows in per_method.items()}}
print(json.dumps(report, indent=2))
if args.report:
args.report.write_text(json.dumps(report, indent=2) + "\n")
if __name__ == "__main__":
main()