constructelligence commited on
Commit
1c1e00f
·
verified ·
1 Parent(s): 04c7da2

Upload calibrate_condition.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. calibrate_condition.py +175 -0
calibrate_condition.py ADDED
@@ -0,0 +1,175 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Fit the surface-condition operating point (confidence, wall share) for one checkpoint.
3
+
4
+ The condition head learned damage from synthetic walls and, at the fixed defaults
5
+ (confidence 0.5 on 0.5% of the wall), calls paint failure or rust on most *real*
6
+ sound photos. A detector that cries wolf on every frame gets switched off, so this
7
+ picks the pair ``(min_confidence, min_fraction)`` that keeps the false-alarm rate on
8
+ real sound walls under a target while finding as much damage as possible.
9
+
10
+ * **negatives** - validation images whose condition mask has no damage. When the
11
+ metadata marks some as real photos, only those set the false-alarm rate
12
+ (synthetic sound walls are too easy); synthetic negatives are still reported.
13
+ * **positives** - validation images with paint failure or rust in their mask
14
+ (synthetic today: real damage photos are kept back as the test set,
15
+ ``surface_eval_worse/``, and must not be used here).
16
+
17
+ Every image is run once; a per-image curve of the largest single-type defect share
18
+ at each confidence scores the whole grid (``condition_cues.defect_fraction_curve``),
19
+ using exactly the rule ``predict.py`` applies. ``--write`` stores the chosen point
20
+ in the checkpoint as ``condition_operating_point``, which ``predict.py`` uses unless
21
+ ``--min-condition-confidence``/``--min-defect-fraction`` are given.
22
+
23
+ Usage::
24
+
25
+ python3 calibrate_condition.py --checkpoint artifacts/best.pt --data real_data --report cond_cal.json
26
+ python3 calibrate_condition.py --checkpoint artifacts/best.pt --data real_data --write
27
+ """
28
+ import argparse
29
+ import csv
30
+ import json
31
+ import os
32
+ from pathlib import Path
33
+
34
+ import numpy as np
35
+
36
+ CONFIDENCES = (0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.98)
37
+ FRACTIONS = (0.005, 0.01, 0.02, 0.03, 0.05, 0.08, 0.12, 0.2)
38
+ MAX_FALSE_ALARM = 0.10
39
+ MIN_DEFECT_PX = 32 # damage in the mask smaller than this does not make a positive
40
+
41
+
42
+ def operating_points(rows, confidences=CONFIDENCES, fractions=FRACTIONS):
43
+ """Recall on positives and false-alarm rates on negatives at every ``(confidence, fraction)``.
44
+
45
+ ``rows``: dicts with ``positive`` (bool), ``real`` (bool) and ``curve`` ({confidence: share}).
46
+ """
47
+ pos = [r for r in rows if r["positive"]]
48
+ neg = [r for r in rows if not r["positive"]]
49
+ real_neg = [r for r in neg if r["real"]]
50
+ synth_neg = [r for r in neg if not r["real"]]
51
+
52
+ def rate(group, c, f):
53
+ return round(float(np.mean([r["curve"][c] >= f for r in group])), 4) if group else None
54
+
55
+ points = []
56
+ for c in confidences:
57
+ for f in fractions:
58
+ points.append({"min_confidence": c, "min_fraction": f, "recall": rate(pos, c, f),
59
+ "false_alarm_real": rate(real_neg, c, f), "false_alarm_synthetic": rate(synth_neg, c, f),
60
+ "false_alarm": rate(real_neg or neg, c, f)})
61
+ return {"positives": len(pos), "negatives_real": len(real_neg), "negatives_synthetic": len(synth_neg),
62
+ "false_alarm_set": "real" if real_neg else "synthetic", "points": points}
63
+
64
+
65
+ def choose_operating_point(curve, max_false_alarm=MAX_FALSE_ALARM):
66
+ """Most recall with false alarms <= target; ties go to fewer false alarms, then the looser setting.
67
+
68
+ Returns the chosen point plus ``reason``, or ``{"min_confidence": None, ...}``
69
+ when nothing qualifies (the caller should then not write a calibration).
70
+ """
71
+ if not curve["positives"] or curve["false_alarm_set"] is None:
72
+ return {"min_confidence": None, "min_fraction": None, "reason": "need both damaged and sound validation images"}
73
+ qualifying = [p for p in curve["points"] if p["false_alarm"] is not None and p["false_alarm"] <= max_false_alarm]
74
+ if not qualifying:
75
+ return {"min_confidence": None, "min_fraction": None,
76
+ "reason": f"no grid point keeps false alarms on {curve['false_alarm_set']} sound walls "
77
+ f"<= {max_false_alarm}"}
78
+ best = max(qualifying, key=lambda p: (p["recall"], -p["false_alarm"], -p["min_confidence"], -p["min_fraction"]))
79
+ return {**best, "max_false_alarm": max_false_alarm,
80
+ "reason": f"highest recall with false alarms <= {max_false_alarm} on "
81
+ f"{curve['false_alarm_set']} sound validation walls"}
82
+
83
+
84
+ def is_real(meta_row):
85
+ source = (meta_row or {}).get("source", "")
86
+ return source != "synthetic"
87
+
88
+
89
+ def collect_rows(model, checkpoint, device, data, split, predict_args, limit=None):
90
+ import cv2
91
+ from PIL import Image
92
+
93
+ from condition_cues import defect_fraction_curve
94
+ from predict import analyze
95
+
96
+ metadata = {}
97
+ meta_path = data / "metadata.csv"
98
+ if meta_path.is_file():
99
+ metadata = {r["image"]: r for r in csv.DictReader(meta_path.open()) if r.get("split") == split}
100
+ cond_dir = data / "condition_masks" / split
101
+ images = sorted(p for p in (data / split / "images").glob("*") if (cond_dir / (p.stem + ".png")).is_file())
102
+ rows = []
103
+ for path in images[:limit]:
104
+ condition = np.asarray(Image.open(cond_dir / (path.stem + ".png")))
105
+ defect_px = int(np.isin(condition, (1, 2)).sum())
106
+ if 0 < defect_px < MIN_DEFECT_PX:
107
+ continue # too little damage to call either way
108
+ bgr = cv2.imread(str(path))
109
+ if bgr is None:
110
+ continue
111
+ _, masks = analyze(bgr, model, checkpoint, device, predict_args)
112
+ curve = defect_fraction_curve(masks["known_wall"], masks["condition_class"], masks["condition_confidence"],
113
+ CONFIDENCES)
114
+ rows.append({"image": path.name, "positive": defect_px > 0, "real": is_real(metadata.get(path.name)),
115
+ "defect_px": defect_px, "curve": curve})
116
+ print(json.dumps({**rows[-1], "curve": {str(k): round(v, 5) for k, v in curve.items()}}), flush=True)
117
+ return rows
118
+
119
+
120
+ def write_into_checkpoint(path, point, curve):
121
+ import torch
122
+ checkpoint = torch.load(path, map_location="cpu", weights_only=True)
123
+ checkpoint["condition_operating_point"] = {
124
+ "min_confidence": point["min_confidence"], "min_fraction": point["min_fraction"],
125
+ "recall": point["recall"], "false_alarm": point["false_alarm"],
126
+ "false_alarm_set": curve["false_alarm_set"], "max_false_alarm": point["max_false_alarm"],
127
+ "positives": curve["positives"], "negatives_real": curve["negatives_real"],
128
+ "negatives_synthetic": curve["negatives_synthetic"]}
129
+ tmp = path.with_name(path.name + ".tmp")
130
+ torch.save(checkpoint, tmp)
131
+ os.replace(tmp, path) # never leave a half-written checkpoint behind
132
+
133
+
134
+ def main():
135
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
136
+ ap.add_argument("--checkpoint", type=Path, required=True)
137
+ ap.add_argument("--data", type=Path, default=Path("real_data"))
138
+ ap.add_argument("--split", default="val")
139
+ ap.add_argument("--max-false-alarm", type=float, default=MAX_FALSE_ALARM)
140
+ ap.add_argument("--limit", type=int, help="score only the first N images (smoke runs)")
141
+ ap.add_argument("--report", type=Path, help="write the operating curve and choice here")
142
+ ap.add_argument("--write", action="store_true", help="store the chosen point in the checkpoint")
143
+ args = ap.parse_args()
144
+ if args.split == "test":
145
+ ap.error("calibrate on val, never test")
146
+
147
+ import torch
148
+
149
+ from predict import build_parser
150
+ from train import load_checkpoint_model
151
+
152
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
153
+ model, checkpoint = load_checkpoint_model(args.checkpoint, device)
154
+ if not checkpoint.get("condition_classes"):
155
+ raise SystemExit(f"{args.checkpoint} has no condition head")
156
+ predict_args = build_parser().parse_args(["unused.jpg"])
157
+ predict_args.measure = False
158
+ rows = collect_rows(model, checkpoint, device, args.data, args.split, predict_args, args.limit)
159
+ curve = operating_points(rows)
160
+ choice = choose_operating_point(curve, args.max_false_alarm)
161
+ default = next(p for p in curve["points"] if p["min_confidence"] == 0.5 and p["min_fraction"] == 0.005)
162
+ report = {"checkpoint": str(args.checkpoint), "data": str(args.data), "split": args.split,
163
+ "uncalibrated_default": default, "chosen": choice, **curve}
164
+ print(json.dumps({k: v for k, v in report.items() if k != "points"}, indent=2))
165
+ if args.report:
166
+ args.report.write_text(json.dumps(report, indent=2) + "\n")
167
+ if args.write:
168
+ if choice["min_confidence"] is None:
169
+ raise SystemExit(f"not written: {choice['reason']}")
170
+ write_into_checkpoint(args.checkpoint, choice, curve)
171
+ print(f"wrote condition_operating_point into {args.checkpoint}")
172
+
173
+
174
+ if __name__ == "__main__":
175
+ main()