Image Segmentation
PyTorch
English
painting-vision-robotics-kit
semantic-segmentation
robotics
edge-ai
construction-ai
autonomous-painting
wall-painting-robot
paint-coverage-estimation
building-facade
drywall
skirting-detection
window-detection
lidar
depth-validation
segformer
deeplabv3
Eval Results (legacy)
Download calibrate_condition.py from constructelligence/painting-vision-robotics-kit: direct link, hf CLI and curl.
- Browser
- Download file 8.79 kB
-
https://huggingface.co/constructelligence/painting-vision-robotics-kit/resolve/main/calibrate_condition.py
- Command line
-
hf download hf://constructelligence/painting-vision-robotics-kit/calibrate_condition.py
-
curl -L -o calibrate_condition.py https://huggingface.co/constructelligence/painting-vision-robotics-kit/resolve/main/calibrate_condition.py
8.79 kB
| #!/usr/bin/env python3 | |
| """Fit the surface-condition operating point (confidence, wall share) for one checkpoint. | |
| The condition head learned damage from synthetic walls and, at the fixed defaults | |
| (confidence 0.5 on 0.5% of the wall), calls paint failure or rust on most *real* | |
| sound photos. A detector that cries wolf on every frame gets switched off, so this | |
| picks the pair ``(min_confidence, min_fraction)`` that keeps the false-alarm rate on | |
| real sound walls under a target while finding as much damage as possible. | |
| * **negatives** - validation images whose condition mask has no damage. When the | |
| metadata marks some as real photos, only those set the false-alarm rate | |
| (synthetic sound walls are too easy); synthetic negatives are still reported. | |
| * **positives** - validation images with paint failure or rust in their mask | |
| (synthetic today: real damage photos are kept back as the test set, | |
| ``surface_eval_worse/``, and must not be used here). | |
| Every image is run once; a per-image curve of the largest single-type defect share | |
| at each confidence scores the whole grid (``condition_cues.defect_fraction_curve``), | |
| using exactly the rule ``predict.py`` applies. ``--write`` stores the chosen point | |
| in the checkpoint as ``condition_operating_point``, which ``predict.py`` uses unless | |
| ``--min-condition-confidence``/``--min-defect-fraction`` are given. | |
| Usage:: | |
| python3 calibrate_condition.py --checkpoint artifacts/best.pt --data real_data --report cond_cal.json | |
| python3 calibrate_condition.py --checkpoint artifacts/best.pt --data real_data --write | |
| """ | |
| import argparse | |
| import csv | |
| import json | |
| import os | |
| from pathlib import Path | |
| import numpy as np | |
| CONFIDENCES = (0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.98) | |
| FRACTIONS = (0.005, 0.01, 0.02, 0.03, 0.05, 0.08, 0.12, 0.2) | |
| MAX_FALSE_ALARM = 0.10 | |
| MIN_DEFECT_PX = 32 # damage in the mask smaller than this does not make a positive | |
| def operating_points(rows, confidences=CONFIDENCES, fractions=FRACTIONS): | |
| """Recall on positives and false-alarm rates on negatives at every ``(confidence, fraction)``. | |
| ``rows``: dicts with ``positive`` (bool), ``real`` (bool) and ``curve`` ({confidence: share}). | |
| """ | |
| pos = [r for r in rows if r["positive"]] | |
| neg = [r for r in rows if not r["positive"]] | |
| real_neg = [r for r in neg if r["real"]] | |
| synth_neg = [r for r in neg if not r["real"]] | |
| def rate(group, c, f): | |
| return round(float(np.mean([r["curve"][c] >= f for r in group])), 4) if group else None | |
| points = [] | |
| for c in confidences: | |
| for f in fractions: | |
| points.append({"min_confidence": c, "min_fraction": f, "recall": rate(pos, c, f), | |
| "false_alarm_real": rate(real_neg, c, f), "false_alarm_synthetic": rate(synth_neg, c, f), | |
| "false_alarm": rate(real_neg or neg, c, f)}) | |
| return {"positives": len(pos), "negatives_real": len(real_neg), "negatives_synthetic": len(synth_neg), | |
| "false_alarm_set": "real" if real_neg else "synthetic", "points": points} | |
| def choose_operating_point(curve, max_false_alarm=MAX_FALSE_ALARM): | |
| """Most recall with false alarms <= target; ties go to fewer false alarms, then the looser setting. | |
| Returns the chosen point plus ``reason``, or ``{"min_confidence": None, ...}`` | |
| when nothing qualifies (the caller should then not write a calibration). | |
| """ | |
| if not curve["positives"] or curve["false_alarm_set"] is None: | |
| return {"min_confidence": None, "min_fraction": None, "reason": "need both damaged and sound validation images"} | |
| qualifying = [p for p in curve["points"] if p["false_alarm"] is not None and p["false_alarm"] <= max_false_alarm] | |
| if not qualifying: | |
| return {"min_confidence": None, "min_fraction": None, | |
| "reason": f"no grid point keeps false alarms on {curve['false_alarm_set']} sound walls " | |
| f"<= {max_false_alarm}"} | |
| best = max(qualifying, key=lambda p: (p["recall"], -p["false_alarm"], -p["min_confidence"], -p["min_fraction"])) | |
| return {**best, "max_false_alarm": max_false_alarm, | |
| "reason": f"highest recall with false alarms <= {max_false_alarm} on " | |
| f"{curve['false_alarm_set']} sound validation walls"} | |
| def is_real(meta_row): | |
| source = (meta_row or {}).get("source", "") | |
| return source != "synthetic" | |
| def collect_rows(model, checkpoint, device, data, split, predict_args, limit=None): | |
| import cv2 | |
| from PIL import Image | |
| from condition_cues import defect_fraction_curve | |
| from predict import analyze | |
| metadata = {} | |
| meta_path = data / "metadata.csv" | |
| if meta_path.is_file(): | |
| metadata = {r["image"]: r for r in csv.DictReader(meta_path.open()) if r.get("split") == split} | |
| cond_dir = data / "condition_masks" / split | |
| images = sorted(p for p in (data / split / "images").glob("*") if (cond_dir / (p.stem + ".png")).is_file()) | |
| rows = [] | |
| for path in images[:limit]: | |
| condition = np.asarray(Image.open(cond_dir / (path.stem + ".png"))) | |
| defect_px = int(np.isin(condition, (1, 2)).sum()) | |
| if 0 < defect_px < MIN_DEFECT_PX: | |
| continue # too little damage to call either way | |
| bgr = cv2.imread(str(path)) | |
| if bgr is None: | |
| continue | |
| _, masks = analyze(bgr, model, checkpoint, device, predict_args) | |
| curve = defect_fraction_curve(masks["known_wall"], masks["condition_class"], masks["condition_confidence"], | |
| CONFIDENCES) | |
| rows.append({"image": path.name, "positive": defect_px > 0, "real": is_real(metadata.get(path.name)), | |
| "defect_px": defect_px, "curve": curve}) | |
| print(json.dumps({**rows[-1], "curve": {str(k): round(v, 5) for k, v in curve.items()}}), flush=True) | |
| return rows | |
| def write_into_checkpoint(path, point, curve): | |
| import torch | |
| checkpoint = torch.load(path, map_location="cpu", weights_only=True) | |
| checkpoint["condition_operating_point"] = { | |
| "min_confidence": point["min_confidence"], "min_fraction": point["min_fraction"], | |
| "recall": point["recall"], "false_alarm": point["false_alarm"], | |
| "false_alarm_set": curve["false_alarm_set"], "max_false_alarm": point["max_false_alarm"], | |
| "positives": curve["positives"], "negatives_real": curve["negatives_real"], | |
| "negatives_synthetic": curve["negatives_synthetic"]} | |
| tmp = path.with_name(path.name + ".tmp") | |
| torch.save(checkpoint, tmp) | |
| os.replace(tmp, path) # never leave a half-written checkpoint behind | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("--checkpoint", type=Path, required=True) | |
| ap.add_argument("--data", type=Path, default=Path("real_data")) | |
| ap.add_argument("--split", default="val") | |
| ap.add_argument("--max-false-alarm", type=float, default=MAX_FALSE_ALARM) | |
| ap.add_argument("--limit", type=int, help="score only the first N images (smoke runs)") | |
| ap.add_argument("--report", type=Path, help="write the operating curve and choice here") | |
| ap.add_argument("--write", action="store_true", help="store the chosen point in the checkpoint") | |
| args = ap.parse_args() | |
| if args.split == "test": | |
| ap.error("calibrate on val, never test") | |
| import torch | |
| from predict import build_parser | |
| from train import load_checkpoint_model | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| model, checkpoint = load_checkpoint_model(args.checkpoint, device) | |
| if not checkpoint.get("condition_classes"): | |
| raise SystemExit(f"{args.checkpoint} has no condition head") | |
| predict_args = build_parser().parse_args(["unused.jpg"]) | |
| predict_args.measure = False | |
| rows = collect_rows(model, checkpoint, device, args.data, args.split, predict_args, args.limit) | |
| curve = operating_points(rows) | |
| choice = choose_operating_point(curve, args.max_false_alarm) | |
| default = next(p for p in curve["points"] if p["min_confidence"] == 0.5 and p["min_fraction"] == 0.005) | |
| report = {"checkpoint": str(args.checkpoint), "data": str(args.data), "split": args.split, | |
| "uncalibrated_default": default, "chosen": choice, **curve} | |
| print(json.dumps({k: v for k, v in report.items() if k != "points"}, indent=2)) | |
| if args.report: | |
| args.report.write_text(json.dumps(report, indent=2) + "\n") | |
| if args.write: | |
| if choice["min_confidence"] is None: | |
| raise SystemExit(f"not written: {choice['reason']}") | |
| write_into_checkpoint(args.checkpoint, choice, curve) | |
| print(f"wrote condition_operating_point into {args.checkpoint}") | |
| if __name__ == "__main__": | |
| main() | |