painting-vision-robotics-kit / calibrate_condition.py
constructelligence's picture
Upload calibrate_condition.py with huggingface_hub
1c1e00f verified
Raw History Blame Contribute Delete
8.79 kB
#!/usr/bin/env python3
"""Fit the surface-condition operating point (confidence, wall share) for one checkpoint.
The condition head learned damage from synthetic walls and, at the fixed defaults
(confidence 0.5 on 0.5% of the wall), calls paint failure or rust on most *real*
sound photos. A detector that cries wolf on every frame gets switched off, so this
picks the pair ``(min_confidence, min_fraction)`` that keeps the false-alarm rate on
real sound walls under a target while finding as much damage as possible.
* **negatives** - validation images whose condition mask has no damage. When the
metadata marks some as real photos, only those set the false-alarm rate
(synthetic sound walls are too easy); synthetic negatives are still reported.
* **positives** - validation images with paint failure or rust in their mask
(synthetic today: real damage photos are kept back as the test set,
``surface_eval_worse/``, and must not be used here).
Every image is run once; a per-image curve of the largest single-type defect share
at each confidence scores the whole grid (``condition_cues.defect_fraction_curve``),
using exactly the rule ``predict.py`` applies. ``--write`` stores the chosen point
in the checkpoint as ``condition_operating_point``, which ``predict.py`` uses unless
``--min-condition-confidence``/``--min-defect-fraction`` are given.
Usage::
python3 calibrate_condition.py --checkpoint artifacts/best.pt --data real_data --report cond_cal.json
python3 calibrate_condition.py --checkpoint artifacts/best.pt --data real_data --write
"""
import argparse
import csv
import json
import os
from pathlib import Path
import numpy as np
CONFIDENCES = (0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.98)
FRACTIONS = (0.005, 0.01, 0.02, 0.03, 0.05, 0.08, 0.12, 0.2)
MAX_FALSE_ALARM = 0.10
MIN_DEFECT_PX = 32 # damage in the mask smaller than this does not make a positive
def operating_points(rows, confidences=CONFIDENCES, fractions=FRACTIONS):
"""Recall on positives and false-alarm rates on negatives at every ``(confidence, fraction)``.
``rows``: dicts with ``positive`` (bool), ``real`` (bool) and ``curve`` ({confidence: share}).
"""
pos = [r for r in rows if r["positive"]]
neg = [r for r in rows if not r["positive"]]
real_neg = [r for r in neg if r["real"]]
synth_neg = [r for r in neg if not r["real"]]
def rate(group, c, f):
return round(float(np.mean([r["curve"][c] >= f for r in group])), 4) if group else None
points = []
for c in confidences:
for f in fractions:
points.append({"min_confidence": c, "min_fraction": f, "recall": rate(pos, c, f),
"false_alarm_real": rate(real_neg, c, f), "false_alarm_synthetic": rate(synth_neg, c, f),
"false_alarm": rate(real_neg or neg, c, f)})
return {"positives": len(pos), "negatives_real": len(real_neg), "negatives_synthetic": len(synth_neg),
"false_alarm_set": "real" if real_neg else "synthetic", "points": points}
def choose_operating_point(curve, max_false_alarm=MAX_FALSE_ALARM):
"""Most recall with false alarms <= target; ties go to fewer false alarms, then the looser setting.
Returns the chosen point plus ``reason``, or ``{"min_confidence": None, ...}``
when nothing qualifies (the caller should then not write a calibration).
"""
if not curve["positives"] or curve["false_alarm_set"] is None:
return {"min_confidence": None, "min_fraction": None, "reason": "need both damaged and sound validation images"}
qualifying = [p for p in curve["points"] if p["false_alarm"] is not None and p["false_alarm"] <= max_false_alarm]
if not qualifying:
return {"min_confidence": None, "min_fraction": None,
"reason": f"no grid point keeps false alarms on {curve['false_alarm_set']} sound walls "
f"<= {max_false_alarm}"}
best = max(qualifying, key=lambda p: (p["recall"], -p["false_alarm"], -p["min_confidence"], -p["min_fraction"]))
return {**best, "max_false_alarm": max_false_alarm,
"reason": f"highest recall with false alarms <= {max_false_alarm} on "
f"{curve['false_alarm_set']} sound validation walls"}
def is_real(meta_row):
source = (meta_row or {}).get("source", "")
return source != "synthetic"
def collect_rows(model, checkpoint, device, data, split, predict_args, limit=None):
import cv2
from PIL import Image
from condition_cues import defect_fraction_curve
from predict import analyze
metadata = {}
meta_path = data / "metadata.csv"
if meta_path.is_file():
metadata = {r["image"]: r for r in csv.DictReader(meta_path.open()) if r.get("split") == split}
cond_dir = data / "condition_masks" / split
images = sorted(p for p in (data / split / "images").glob("*") if (cond_dir / (p.stem + ".png")).is_file())
rows = []
for path in images[:limit]:
condition = np.asarray(Image.open(cond_dir / (path.stem + ".png")))
defect_px = int(np.isin(condition, (1, 2)).sum())
if 0 < defect_px < MIN_DEFECT_PX:
continue # too little damage to call either way
bgr = cv2.imread(str(path))
if bgr is None:
continue
_, masks = analyze(bgr, model, checkpoint, device, predict_args)
curve = defect_fraction_curve(masks["known_wall"], masks["condition_class"], masks["condition_confidence"],
CONFIDENCES)
rows.append({"image": path.name, "positive": defect_px > 0, "real": is_real(metadata.get(path.name)),
"defect_px": defect_px, "curve": curve})
print(json.dumps({**rows[-1], "curve": {str(k): round(v, 5) for k, v in curve.items()}}), flush=True)
return rows
def write_into_checkpoint(path, point, curve):
import torch
checkpoint = torch.load(path, map_location="cpu", weights_only=True)
checkpoint["condition_operating_point"] = {
"min_confidence": point["min_confidence"], "min_fraction": point["min_fraction"],
"recall": point["recall"], "false_alarm": point["false_alarm"],
"false_alarm_set": curve["false_alarm_set"], "max_false_alarm": point["max_false_alarm"],
"positives": curve["positives"], "negatives_real": curve["negatives_real"],
"negatives_synthetic": curve["negatives_synthetic"]}
tmp = path.with_name(path.name + ".tmp")
torch.save(checkpoint, tmp)
os.replace(tmp, path) # never leave a half-written checkpoint behind
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--checkpoint", type=Path, required=True)
ap.add_argument("--data", type=Path, default=Path("real_data"))
ap.add_argument("--split", default="val")
ap.add_argument("--max-false-alarm", type=float, default=MAX_FALSE_ALARM)
ap.add_argument("--limit", type=int, help="score only the first N images (smoke runs)")
ap.add_argument("--report", type=Path, help="write the operating curve and choice here")
ap.add_argument("--write", action="store_true", help="store the chosen point in the checkpoint")
args = ap.parse_args()
if args.split == "test":
ap.error("calibrate on val, never test")
import torch
from predict import build_parser
from train import load_checkpoint_model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model, checkpoint = load_checkpoint_model(args.checkpoint, device)
if not checkpoint.get("condition_classes"):
raise SystemExit(f"{args.checkpoint} has no condition head")
predict_args = build_parser().parse_args(["unused.jpg"])
predict_args.measure = False
rows = collect_rows(model, checkpoint, device, args.data, args.split, predict_args, args.limit)
curve = operating_points(rows)
choice = choose_operating_point(curve, args.max_false_alarm)
default = next(p for p in curve["points"] if p["min_confidence"] == 0.5 and p["min_fraction"] == 0.005)
report = {"checkpoint": str(args.checkpoint), "data": str(args.data), "split": args.split,
"uncalibrated_default": default, "chosen": choice, **curve}
print(json.dumps({k: v for k, v in report.items() if k != "points"}, indent=2))
if args.report:
args.report.write_text(json.dumps(report, indent=2) + "\n")
if args.write:
if choice["min_confidence"] is None:
raise SystemExit(f"not written: {choice['reason']}")
write_into_checkpoint(args.checkpoint, choice, curve)
print(f"wrote condition_operating_point into {args.checkpoint}")
if __name__ == "__main__":
main()