#!/usr/bin/env python3 """Fit the surface-condition operating point (confidence, wall share) for one checkpoint. The condition head learned damage from synthetic walls and, at the fixed defaults (confidence 0.5 on 0.5% of the wall), calls paint failure or rust on most *real* sound photos. A detector that cries wolf on every frame gets switched off, so this picks the pair ``(min_confidence, min_fraction)`` that keeps the false-alarm rate on real sound walls under a target while finding as much damage as possible. * **negatives** - validation images whose condition mask has no damage. When the metadata marks some as real photos, only those set the false-alarm rate (synthetic sound walls are too easy); synthetic negatives are still reported. * **positives** - validation images with paint failure or rust in their mask (synthetic today: real damage photos are kept back as the test set, ``surface_eval_worse/``, and must not be used here). Every image is run once; a per-image curve of the largest single-type defect share at each confidence scores the whole grid (``condition_cues.defect_fraction_curve``), using exactly the rule ``predict.py`` applies. ``--write`` stores the chosen point in the checkpoint as ``condition_operating_point``, which ``predict.py`` uses unless ``--min-condition-confidence``/``--min-defect-fraction`` are given. Usage:: python3 calibrate_condition.py --checkpoint artifacts/best.pt --data real_data --report cond_cal.json python3 calibrate_condition.py --checkpoint artifacts/best.pt --data real_data --write """ import argparse import csv import json import os from pathlib import Path import numpy as np CONFIDENCES = (0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.98) FRACTIONS = (0.005, 0.01, 0.02, 0.03, 0.05, 0.08, 0.12, 0.2) MAX_FALSE_ALARM = 0.10 MIN_DEFECT_PX = 32 # damage in the mask smaller than this does not make a positive def operating_points(rows, confidences=CONFIDENCES, fractions=FRACTIONS): """Recall on positives and false-alarm rates on negatives at every ``(confidence, fraction)``. ``rows``: dicts with ``positive`` (bool), ``real`` (bool) and ``curve`` ({confidence: share}). """ pos = [r for r in rows if r["positive"]] neg = [r for r in rows if not r["positive"]] real_neg = [r for r in neg if r["real"]] synth_neg = [r for r in neg if not r["real"]] def rate(group, c, f): return round(float(np.mean([r["curve"][c] >= f for r in group])), 4) if group else None points = [] for c in confidences: for f in fractions: points.append({"min_confidence": c, "min_fraction": f, "recall": rate(pos, c, f), "false_alarm_real": rate(real_neg, c, f), "false_alarm_synthetic": rate(synth_neg, c, f), "false_alarm": rate(real_neg or neg, c, f)}) return {"positives": len(pos), "negatives_real": len(real_neg), "negatives_synthetic": len(synth_neg), "false_alarm_set": "real" if real_neg else "synthetic", "points": points} def choose_operating_point(curve, max_false_alarm=MAX_FALSE_ALARM): """Most recall with false alarms <= target; ties go to fewer false alarms, then the looser setting. Returns the chosen point plus ``reason``, or ``{"min_confidence": None, ...}`` when nothing qualifies (the caller should then not write a calibration). """ if not curve["positives"] or curve["false_alarm_set"] is None: return {"min_confidence": None, "min_fraction": None, "reason": "need both damaged and sound validation images"} qualifying = [p for p in curve["points"] if p["false_alarm"] is not None and p["false_alarm"] <= max_false_alarm] if not qualifying: return {"min_confidence": None, "min_fraction": None, "reason": f"no grid point keeps false alarms on {curve['false_alarm_set']} sound walls " f"<= {max_false_alarm}"} best = max(qualifying, key=lambda p: (p["recall"], -p["false_alarm"], -p["min_confidence"], -p["min_fraction"])) return {**best, "max_false_alarm": max_false_alarm, "reason": f"highest recall with false alarms <= {max_false_alarm} on " f"{curve['false_alarm_set']} sound validation walls"} def is_real(meta_row): source = (meta_row or {}).get("source", "") return source != "synthetic" def collect_rows(model, checkpoint, device, data, split, predict_args, limit=None): import cv2 from PIL import Image from condition_cues import defect_fraction_curve from predict import analyze metadata = {} meta_path = data / "metadata.csv" if meta_path.is_file(): metadata = {r["image"]: r for r in csv.DictReader(meta_path.open()) if r.get("split") == split} cond_dir = data / "condition_masks" / split images = sorted(p for p in (data / split / "images").glob("*") if (cond_dir / (p.stem + ".png")).is_file()) rows = [] for path in images[:limit]: condition = np.asarray(Image.open(cond_dir / (path.stem + ".png"))) defect_px = int(np.isin(condition, (1, 2)).sum()) if 0 < defect_px < MIN_DEFECT_PX: continue # too little damage to call either way bgr = cv2.imread(str(path)) if bgr is None: continue _, masks = analyze(bgr, model, checkpoint, device, predict_args) curve = defect_fraction_curve(masks["known_wall"], masks["condition_class"], masks["condition_confidence"], CONFIDENCES) rows.append({"image": path.name, "positive": defect_px > 0, "real": is_real(metadata.get(path.name)), "defect_px": defect_px, "curve": curve}) print(json.dumps({**rows[-1], "curve": {str(k): round(v, 5) for k, v in curve.items()}}), flush=True) return rows def write_into_checkpoint(path, point, curve): import torch checkpoint = torch.load(path, map_location="cpu", weights_only=True) checkpoint["condition_operating_point"] = { "min_confidence": point["min_confidence"], "min_fraction": point["min_fraction"], "recall": point["recall"], "false_alarm": point["false_alarm"], "false_alarm_set": curve["false_alarm_set"], "max_false_alarm": point["max_false_alarm"], "positives": curve["positives"], "negatives_real": curve["negatives_real"], "negatives_synthetic": curve["negatives_synthetic"]} tmp = path.with_name(path.name + ".tmp") torch.save(checkpoint, tmp) os.replace(tmp, path) # never leave a half-written checkpoint behind def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--checkpoint", type=Path, required=True) ap.add_argument("--data", type=Path, default=Path("real_data")) ap.add_argument("--split", default="val") ap.add_argument("--max-false-alarm", type=float, default=MAX_FALSE_ALARM) ap.add_argument("--limit", type=int, help="score only the first N images (smoke runs)") ap.add_argument("--report", type=Path, help="write the operating curve and choice here") ap.add_argument("--write", action="store_true", help="store the chosen point in the checkpoint") args = ap.parse_args() if args.split == "test": ap.error("calibrate on val, never test") import torch from predict import build_parser from train import load_checkpoint_model device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model, checkpoint = load_checkpoint_model(args.checkpoint, device) if not checkpoint.get("condition_classes"): raise SystemExit(f"{args.checkpoint} has no condition head") predict_args = build_parser().parse_args(["unused.jpg"]) predict_args.measure = False rows = collect_rows(model, checkpoint, device, args.data, args.split, predict_args, args.limit) curve = operating_points(rows) choice = choose_operating_point(curve, args.max_false_alarm) default = next(p for p in curve["points"] if p["min_confidence"] == 0.5 and p["min_fraction"] == 0.005) report = {"checkpoint": str(args.checkpoint), "data": str(args.data), "split": args.split, "uncalibrated_default": default, "chosen": choice, **curve} print(json.dumps({k: v for k, v in report.items() if k != "points"}, indent=2)) if args.report: args.report.write_text(json.dumps(report, indent=2) + "\n") if args.write: if choice["min_confidence"] is None: raise SystemExit(f"not written: {choice['reason']}") write_into_checkpoint(args.checkpoint, choice, curve) print(f"wrote condition_operating_point into {args.checkpoint}") if __name__ == "__main__": main()