Image Segmentation
PyTorch
English
painting-vision-robotics-kit
semantic-segmentation
robotics
edge-ai
construction-ai
autonomous-painting
wall-painting-robot
paint-coverage-estimation
building-facade
drywall
skirting-detection
window-detection
lidar
depth-validation
segformer
deeplabv3
Eval Results (legacy)
Download eval_regions.py from constructelligence/painting-vision-robotics-kit: direct link, hf CLI and curl.
- Browser
- Download file 8.24 kB
-
https://huggingface.co/constructelligence/painting-vision-robotics-kit/resolve/main/eval_regions.py
- Command line
-
hf download hf://constructelligence/painting-vision-robotics-kit/eval_regions.py
-
curl -L -o eval_regions.py https://huggingface.co/constructelligence/painting-vision-robotics-kit/resolve/main/eval_regions.py
8.24 kB
| #!/usr/bin/env python3 | |
| """Region-level test of damage detection: does the robot get one task per real patch? | |
| Pixel IoU says little about what a robot does with damage. It needs each peeling | |
| patch or rust stain found **as one region** that covers the whole patch (the edge | |
| still has to be scraped), and no regions on sound walls. Against the pixel masks | |
| in ``condition_masks/<split>/`` this reports, per method: | |
| * ``region_recall`` - share of true patches (>= ``MIN_AREA_PX``) whose pixels a | |
| prediction covers at least half of; | |
| * ``region_precision`` - share of predicted regions with at least half their | |
| pixels on true damage (3 px tolerance); | |
| * ``fragments_per_found_patch`` - predicted regions touching each found patch | |
| (1.0 is ideal: one patch, one tool task); | |
| * ``pixel_recall``/``pixel_iou`` - how much of the damage the regions cover; | |
| * ``sound_image_region_rate`` - share of damage-free images given any region, | |
| and the mean regions per such image, separately for real photos. | |
| Methods: ``argmax`` is the old rule (confident argmax pixels, speckle removed, | |
| connected components); ``regions:low=..,gap=..`` is ``regions.damage_regions``. | |
| Everything runs at the model's resolution. Tune on ``val``; report ``test``. | |
| Usage:: | |
| python3 eval_regions.py --checkpoint artifacts/best.pt --data real_data --split val --report regions_val.json | |
| """ | |
| import argparse | |
| import csv | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| from scipy import ndimage | |
| from regions import MIN_AREA_PX, damage_regions | |
| LOWS = (None, 0.4, "off") # None = default offset below high; "off" = no growth (low = high) | |
| GAPS = (0, 8, 12, 20) | |
| TOLERANCE_PX = 3 | |
| def components(mask, min_area=1): | |
| labels, count = ndimage.label(mask, structure=np.ones((3, 3), bool)) | |
| if not count: | |
| return labels, [] | |
| sizes = np.bincount(labels.ravel(), minlength=count + 1) | |
| return labels, [i for i in range(1, count + 1) if sizes[i] >= min_area] | |
| def argmax_mask(probability, wall, high): | |
| """The old rule: per-pixel argmax over sound/paint failure/rust, confident, speckle removed.""" | |
| damage = np.asarray(probability, np.float32) | |
| sound = 1.0 - damage.sum(axis=0) | |
| winner = np.argmax(np.concatenate([sound[None], damage]), axis=0) | |
| confidence = np.max(np.concatenate([sound[None], damage]), axis=0) | |
| out = np.zeros_like(wall) | |
| for index in (1, 2): | |
| out |= ndimage.binary_opening((winner == index) & (confidence >= high) & wall, iterations=1) | |
| return out | |
| def score_image(pred, truth, wall): | |
| """Counts for one image; ``truth`` is a bool damage mask on the wall.""" | |
| pred, truth = pred & wall, truth & wall | |
| p_labels, p_ids = components(pred, MIN_AREA_PX) | |
| t_labels, t_ids = components(truth, MIN_AREA_PX) | |
| kept = np.isin(p_labels, p_ids) | |
| near_truth = ndimage.binary_dilation(truth, iterations=TOLERANCE_PX) if truth.any() else truth | |
| correct = sum(near_truth[p_labels == i].mean() >= 0.5 for i in p_ids) | |
| found, fragments = 0, [] | |
| for i in t_ids: | |
| patch = t_labels == i | |
| if kept[patch].mean() >= 0.5: | |
| found += 1 | |
| fragments.append(len(set(np.unique(p_labels[patch & kept])) - {0})) | |
| return {"pred_regions": len(p_ids), "pred_correct": int(correct), "true_patches": len(t_ids), | |
| "found": found, "fragments": fragments, "tp_px": int((kept & truth).sum()), | |
| "pred_px": int(kept.sum()), "true_px": int(truth.sum())} | |
| def summarise(per_image): | |
| def total(key): | |
| return sum(r[key] for r in per_image) | |
| damaged = [r for r in per_image if r["true_px"] > 0] | |
| sound = [r for r in per_image if r["true_px"] == 0] | |
| sound_real = [r for r in sound if r["real"]] | |
| frags = [f for r in per_image for f in r["fragments"]] | |
| def rate(group): | |
| return round(float(np.mean([r["pred_regions"] > 0 for r in group])), 4) if group else None | |
| union = total("pred_px") + total("true_px") - total("tp_px") | |
| return { | |
| "damaged_images": len(damaged), "sound_images": len(sound), "sound_real_images": len(sound_real), | |
| "true_patches": total("true_patches"), | |
| "region_recall": round(total("found") / total("true_patches"), 4) if total("true_patches") else None, | |
| "region_precision": (round(sum(r["pred_correct"] for r in damaged) / sum(r["pred_regions"] for r in damaged), 4) | |
| if sum(r["pred_regions"] for r in damaged) else None), | |
| "fragments_per_found_patch": round(float(np.mean(frags)), 3) if frags else None, | |
| "pixel_recall": round(total("tp_px") / total("true_px"), 4) if total("true_px") else None, | |
| "pixel_iou": round(total("tp_px") / union, 4) if union else None, | |
| "sound_image_region_rate": rate(sound), | |
| "sound_real_image_region_rate": rate(sound_real), | |
| "regions_per_sound_real_image": (round(float(np.mean([r["pred_regions"] for r in sound_real])), 3) | |
| if sound_real else None), | |
| } | |
| def methods(high): | |
| yield "argmax", lambda prob, wall: argmax_mask(prob, wall, high) | |
| for low in LOWS: | |
| for gap in GAPS: | |
| low_value = high if low == "off" else low | |
| name = f"regions:low={'off' if low == 'off' else (low or 'default')},gap={gap}" | |
| yield name, (lambda prob, wall, lv=low_value, g=gap: | |
| damage_regions(prob, wall, high=high, low=lv, merge_gap_px=g, describe=False)[1]) | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("--checkpoint", type=Path, required=True) | |
| ap.add_argument("--data", type=Path, default=Path("real_data")) | |
| ap.add_argument("--split", default="val") | |
| ap.add_argument("--limit", type=int) | |
| ap.add_argument("--report", type=Path) | |
| args = ap.parse_args() | |
| import cv2 | |
| import torch | |
| from PIL import Image | |
| from predict import analyze, build_parser, condition_thresholds | |
| from train import load_checkpoint_model | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| model, checkpoint = load_checkpoint_model(args.checkpoint, device) | |
| predict_args = build_parser().parse_args(["unused.jpg"]) | |
| predict_args.measure = False | |
| high, _, source = condition_thresholds(predict_args, checkpoint) | |
| meta = {} | |
| if (args.data / "metadata.csv").is_file(): | |
| meta = {r["image"]: r for r in csv.DictReader((args.data / "metadata.csv").open()) if r["split"] == args.split} | |
| cond_dir = args.data / "condition_masks" / args.split | |
| images = sorted(p for p in (args.data / args.split / "images").glob("*") if (cond_dir / (p.stem + ".png")).is_file()) | |
| per_method = {name: [] for name, _ in methods(high)} | |
| for path in images[:args.limit]: | |
| bgr = cv2.imread(str(path)) | |
| if bgr is None: | |
| continue | |
| _, masks = analyze(bgr, model, checkpoint, device, predict_args) | |
| prob = masks["condition_probability_small"] | |
| if prob is None: | |
| raise SystemExit(f"{args.checkpoint} has no condition head") | |
| size = (prob.shape[2], prob.shape[1]) | |
| wall = cv2.resize(masks["known_wall"].astype(np.uint8), size, interpolation=cv2.INTER_NEAREST) > 0 | |
| truth_full = np.asarray(Image.open(cond_dir / (path.stem + ".png"))) | |
| truth = cv2.resize(np.isin(truth_full, (1, 2)).astype(np.uint8), size, interpolation=cv2.INTER_NEAREST) > 0 | |
| real = (meta.get(path.name) or {}).get("source", "") != "synthetic" | |
| for name, method in methods(high): | |
| per_method[name].append({**score_image(method(prob, wall), truth, wall), "real": real}) | |
| print(json.dumps({"image": path.name, "real": real, "true_px": per_method["argmax"][-1]["true_px"], | |
| "argmax_regions": per_method["argmax"][-1]["pred_regions"]}), flush=True) | |
| report = {"checkpoint": str(args.checkpoint), "split": args.split, "high": high, "thresholds_from": source, | |
| "min_area_px": MIN_AREA_PX, "methods": {name: summarise(rows) for name, rows in per_method.items()}} | |
| print(json.dumps(report, indent=2)) | |
| if args.report: | |
| args.report.write_text(json.dumps(report, indent=2) + "\n") | |
| if __name__ == "__main__": | |
| main() | |