Image Segmentation
PyTorch
English
painting-vision-robotics-kit
semantic-segmentation
robotics
edge-ai
construction-ai
autonomous-painting
wall-painting-robot
paint-coverage-estimation
building-facade
drywall
skirting-detection
window-detection
lidar
depth-validation
segformer
deeplabv3
Eval Results (legacy)
Upload calibrate_condition.py with huggingface_hub
Browse files- calibrate_condition.py +175 -0
calibrate_condition.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Fit the surface-condition operating point (confidence, wall share) for one checkpoint.
|
| 3 |
+
|
| 4 |
+
The condition head learned damage from synthetic walls and, at the fixed defaults
|
| 5 |
+
(confidence 0.5 on 0.5% of the wall), calls paint failure or rust on most *real*
|
| 6 |
+
sound photos. A detector that cries wolf on every frame gets switched off, so this
|
| 7 |
+
picks the pair ``(min_confidence, min_fraction)`` that keeps the false-alarm rate on
|
| 8 |
+
real sound walls under a target while finding as much damage as possible.
|
| 9 |
+
|
| 10 |
+
* **negatives** - validation images whose condition mask has no damage. When the
|
| 11 |
+
metadata marks some as real photos, only those set the false-alarm rate
|
| 12 |
+
(synthetic sound walls are too easy); synthetic negatives are still reported.
|
| 13 |
+
* **positives** - validation images with paint failure or rust in their mask
|
| 14 |
+
(synthetic today: real damage photos are kept back as the test set,
|
| 15 |
+
``surface_eval_worse/``, and must not be used here).
|
| 16 |
+
|
| 17 |
+
Every image is run once; a per-image curve of the largest single-type defect share
|
| 18 |
+
at each confidence scores the whole grid (``condition_cues.defect_fraction_curve``),
|
| 19 |
+
using exactly the rule ``predict.py`` applies. ``--write`` stores the chosen point
|
| 20 |
+
in the checkpoint as ``condition_operating_point``, which ``predict.py`` uses unless
|
| 21 |
+
``--min-condition-confidence``/``--min-defect-fraction`` are given.
|
| 22 |
+
|
| 23 |
+
Usage::
|
| 24 |
+
|
| 25 |
+
python3 calibrate_condition.py --checkpoint artifacts/best.pt --data real_data --report cond_cal.json
|
| 26 |
+
python3 calibrate_condition.py --checkpoint artifacts/best.pt --data real_data --write
|
| 27 |
+
"""
|
| 28 |
+
import argparse
|
| 29 |
+
import csv
|
| 30 |
+
import json
|
| 31 |
+
import os
|
| 32 |
+
from pathlib import Path
|
| 33 |
+
|
| 34 |
+
import numpy as np
|
| 35 |
+
|
| 36 |
+
CONFIDENCES = (0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.98)
|
| 37 |
+
FRACTIONS = (0.005, 0.01, 0.02, 0.03, 0.05, 0.08, 0.12, 0.2)
|
| 38 |
+
MAX_FALSE_ALARM = 0.10
|
| 39 |
+
MIN_DEFECT_PX = 32 # damage in the mask smaller than this does not make a positive
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def operating_points(rows, confidences=CONFIDENCES, fractions=FRACTIONS):
|
| 43 |
+
"""Recall on positives and false-alarm rates on negatives at every ``(confidence, fraction)``.
|
| 44 |
+
|
| 45 |
+
``rows``: dicts with ``positive`` (bool), ``real`` (bool) and ``curve`` ({confidence: share}).
|
| 46 |
+
"""
|
| 47 |
+
pos = [r for r in rows if r["positive"]]
|
| 48 |
+
neg = [r for r in rows if not r["positive"]]
|
| 49 |
+
real_neg = [r for r in neg if r["real"]]
|
| 50 |
+
synth_neg = [r for r in neg if not r["real"]]
|
| 51 |
+
|
| 52 |
+
def rate(group, c, f):
|
| 53 |
+
return round(float(np.mean([r["curve"][c] >= f for r in group])), 4) if group else None
|
| 54 |
+
|
| 55 |
+
points = []
|
| 56 |
+
for c in confidences:
|
| 57 |
+
for f in fractions:
|
| 58 |
+
points.append({"min_confidence": c, "min_fraction": f, "recall": rate(pos, c, f),
|
| 59 |
+
"false_alarm_real": rate(real_neg, c, f), "false_alarm_synthetic": rate(synth_neg, c, f),
|
| 60 |
+
"false_alarm": rate(real_neg or neg, c, f)})
|
| 61 |
+
return {"positives": len(pos), "negatives_real": len(real_neg), "negatives_synthetic": len(synth_neg),
|
| 62 |
+
"false_alarm_set": "real" if real_neg else "synthetic", "points": points}
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def choose_operating_point(curve, max_false_alarm=MAX_FALSE_ALARM):
|
| 66 |
+
"""Most recall with false alarms <= target; ties go to fewer false alarms, then the looser setting.
|
| 67 |
+
|
| 68 |
+
Returns the chosen point plus ``reason``, or ``{"min_confidence": None, ...}``
|
| 69 |
+
when nothing qualifies (the caller should then not write a calibration).
|
| 70 |
+
"""
|
| 71 |
+
if not curve["positives"] or curve["false_alarm_set"] is None:
|
| 72 |
+
return {"min_confidence": None, "min_fraction": None, "reason": "need both damaged and sound validation images"}
|
| 73 |
+
qualifying = [p for p in curve["points"] if p["false_alarm"] is not None and p["false_alarm"] <= max_false_alarm]
|
| 74 |
+
if not qualifying:
|
| 75 |
+
return {"min_confidence": None, "min_fraction": None,
|
| 76 |
+
"reason": f"no grid point keeps false alarms on {curve['false_alarm_set']} sound walls "
|
| 77 |
+
f"<= {max_false_alarm}"}
|
| 78 |
+
best = max(qualifying, key=lambda p: (p["recall"], -p["false_alarm"], -p["min_confidence"], -p["min_fraction"]))
|
| 79 |
+
return {**best, "max_false_alarm": max_false_alarm,
|
| 80 |
+
"reason": f"highest recall with false alarms <= {max_false_alarm} on "
|
| 81 |
+
f"{curve['false_alarm_set']} sound validation walls"}
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def is_real(meta_row):
|
| 85 |
+
source = (meta_row or {}).get("source", "")
|
| 86 |
+
return source != "synthetic"
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def collect_rows(model, checkpoint, device, data, split, predict_args, limit=None):
|
| 90 |
+
import cv2
|
| 91 |
+
from PIL import Image
|
| 92 |
+
|
| 93 |
+
from condition_cues import defect_fraction_curve
|
| 94 |
+
from predict import analyze
|
| 95 |
+
|
| 96 |
+
metadata = {}
|
| 97 |
+
meta_path = data / "metadata.csv"
|
| 98 |
+
if meta_path.is_file():
|
| 99 |
+
metadata = {r["image"]: r for r in csv.DictReader(meta_path.open()) if r.get("split") == split}
|
| 100 |
+
cond_dir = data / "condition_masks" / split
|
| 101 |
+
images = sorted(p for p in (data / split / "images").glob("*") if (cond_dir / (p.stem + ".png")).is_file())
|
| 102 |
+
rows = []
|
| 103 |
+
for path in images[:limit]:
|
| 104 |
+
condition = np.asarray(Image.open(cond_dir / (path.stem + ".png")))
|
| 105 |
+
defect_px = int(np.isin(condition, (1, 2)).sum())
|
| 106 |
+
if 0 < defect_px < MIN_DEFECT_PX:
|
| 107 |
+
continue # too little damage to call either way
|
| 108 |
+
bgr = cv2.imread(str(path))
|
| 109 |
+
if bgr is None:
|
| 110 |
+
continue
|
| 111 |
+
_, masks = analyze(bgr, model, checkpoint, device, predict_args)
|
| 112 |
+
curve = defect_fraction_curve(masks["known_wall"], masks["condition_class"], masks["condition_confidence"],
|
| 113 |
+
CONFIDENCES)
|
| 114 |
+
rows.append({"image": path.name, "positive": defect_px > 0, "real": is_real(metadata.get(path.name)),
|
| 115 |
+
"defect_px": defect_px, "curve": curve})
|
| 116 |
+
print(json.dumps({**rows[-1], "curve": {str(k): round(v, 5) for k, v in curve.items()}}), flush=True)
|
| 117 |
+
return rows
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def write_into_checkpoint(path, point, curve):
|
| 121 |
+
import torch
|
| 122 |
+
checkpoint = torch.load(path, map_location="cpu", weights_only=True)
|
| 123 |
+
checkpoint["condition_operating_point"] = {
|
| 124 |
+
"min_confidence": point["min_confidence"], "min_fraction": point["min_fraction"],
|
| 125 |
+
"recall": point["recall"], "false_alarm": point["false_alarm"],
|
| 126 |
+
"false_alarm_set": curve["false_alarm_set"], "max_false_alarm": point["max_false_alarm"],
|
| 127 |
+
"positives": curve["positives"], "negatives_real": curve["negatives_real"],
|
| 128 |
+
"negatives_synthetic": curve["negatives_synthetic"]}
|
| 129 |
+
tmp = path.with_name(path.name + ".tmp")
|
| 130 |
+
torch.save(checkpoint, tmp)
|
| 131 |
+
os.replace(tmp, path) # never leave a half-written checkpoint behind
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def main():
|
| 135 |
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
| 136 |
+
ap.add_argument("--checkpoint", type=Path, required=True)
|
| 137 |
+
ap.add_argument("--data", type=Path, default=Path("real_data"))
|
| 138 |
+
ap.add_argument("--split", default="val")
|
| 139 |
+
ap.add_argument("--max-false-alarm", type=float, default=MAX_FALSE_ALARM)
|
| 140 |
+
ap.add_argument("--limit", type=int, help="score only the first N images (smoke runs)")
|
| 141 |
+
ap.add_argument("--report", type=Path, help="write the operating curve and choice here")
|
| 142 |
+
ap.add_argument("--write", action="store_true", help="store the chosen point in the checkpoint")
|
| 143 |
+
args = ap.parse_args()
|
| 144 |
+
if args.split == "test":
|
| 145 |
+
ap.error("calibrate on val, never test")
|
| 146 |
+
|
| 147 |
+
import torch
|
| 148 |
+
|
| 149 |
+
from predict import build_parser
|
| 150 |
+
from train import load_checkpoint_model
|
| 151 |
+
|
| 152 |
+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 153 |
+
model, checkpoint = load_checkpoint_model(args.checkpoint, device)
|
| 154 |
+
if not checkpoint.get("condition_classes"):
|
| 155 |
+
raise SystemExit(f"{args.checkpoint} has no condition head")
|
| 156 |
+
predict_args = build_parser().parse_args(["unused.jpg"])
|
| 157 |
+
predict_args.measure = False
|
| 158 |
+
rows = collect_rows(model, checkpoint, device, args.data, args.split, predict_args, args.limit)
|
| 159 |
+
curve = operating_points(rows)
|
| 160 |
+
choice = choose_operating_point(curve, args.max_false_alarm)
|
| 161 |
+
default = next(p for p in curve["points"] if p["min_confidence"] == 0.5 and p["min_fraction"] == 0.005)
|
| 162 |
+
report = {"checkpoint": str(args.checkpoint), "data": str(args.data), "split": args.split,
|
| 163 |
+
"uncalibrated_default": default, "chosen": choice, **curve}
|
| 164 |
+
print(json.dumps({k: v for k, v in report.items() if k != "points"}, indent=2))
|
| 165 |
+
if args.report:
|
| 166 |
+
args.report.write_text(json.dumps(report, indent=2) + "\n")
|
| 167 |
+
if args.write:
|
| 168 |
+
if choice["min_confidence"] is None:
|
| 169 |
+
raise SystemExit(f"not written: {choice['reason']}")
|
| 170 |
+
write_into_checkpoint(args.checkpoint, choice, curve)
|
| 171 |
+
print(f"wrote condition_operating_point into {args.checkpoint}")
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
if __name__ == "__main__":
|
| 175 |
+
main()
|