#!/usr/bin/env python3 """Does test-time augmentation (flip, multi-scale) help the wall model on held-out photos? Same data, same metric, only the inference differs. python3 eval_tta.py --checkpoint artifacts/candidates/C2-cal.pt --data real_data --split test --out runs_tta_test.json Reports per-class IoU, mIoU over classes present, window recall/precision (the safety-relevant class) and painted-coverage MAE for each mode, plus a bootstrap over images for the mIoU difference to the plain model, so a gain inside the noise is not reported as a gain.""" import argparse import json import time import numpy as np import torch import torch.nn.functional as F from benchmark import confusion from label_schema import CLASSES from models import DEFAULT_ARCH, build_from_checkpoint from train import WallDataset MODES = {"plain": [(1.0, False)], "flip": [(1.0, False), (1.0, True)], "flip+ms": [(s, f) for s in (0.75, 1.0, 1.25) for f in (False, True)], "ms": [(s, False) for s in (0.75, 1.0, 1.25)]} def predict(model, x, mode): n_c = len(CLASSES) acc = None for scale, flip in MODES[mode]: xi = x if scale != 1.0: h, w = x.shape[-2:] xi = F.interpolate(x, size=(int(round(h * scale / 32) * 32), int(round(w * scale / 32) * 32)), mode="bilinear", align_corners=False) if flip: xi = torch.flip(xi, dims=[3]) out = model(xi)["semantic"] if flip: out = torch.flip(out, dims=[3]) if out.shape[-2:] != x.shape[-2:]: out = F.interpolate(out, size=x.shape[-2:], mode="bilinear", align_corners=False) p = out.softmax(1) acc = p if acc is None else acc + p return (acc / len(MODES[mode])).argmax(1) def miou(cm): inter = np.diag(cm).astype(float) union = cm.sum(0) + cm.sum(1) - np.diag(cm) present = cm.sum(1) > 0 iou = np.where(union > 0, inter / np.maximum(union, 1), np.nan) return float(np.nanmean(iou[present])), iou def main(): ap = argparse.ArgumentParser() ap.add_argument("--checkpoint", required=True) ap.add_argument("--data", default="real_data") ap.add_argument("--split", default="test") ap.add_argument("--size", type=int, default=640) ap.add_argument("--limit", type=int) ap.add_argument("--modes", default="plain,flip,ms,flip+ms") ap.add_argument("--out") a = ap.parse_args() ck = torch.load(a.checkpoint, map_location="cpu", weights_only=True) model = build_from_checkpoint(ck, len(CLASSES)).eval() ds = WallDataset(a.data, a.split, a.size) n = min(len(ds), a.limit or len(ds)) modes = a.modes.split(",") per_image = {m: [] for m in modes} cms = {m: np.zeros((len(CLASSES),) * 2, np.int64) for m in modes} t0 = time.time() with torch.inference_mode(): for i in range(n): x, y, *_ = ds[i] x, y = x.unsqueeze(0), y.numpy() for m in modes: p = predict(model, x, m)[0].numpy() c = confusion(p, y, len(CLASSES)) cms[m] += c per_image[m].append(c) if i % 5 == 0: print(f"{i}/{n} {time.time() - t0:.0f}s", flush=True) rng = np.random.default_rng(0) res = {"checkpoint": a.checkpoint, "split": a.split, "images": n, "modes": {}} base = np.stack(per_image["plain"]) if "plain" in per_image else None for m in modes: mi, iou = miou(cms[m]) cm = cms[m] w = CLASSES.index("window") if "window" in CLASSES else None row = {"mIoU": round(mi, 4), "iou": {c: (None if np.isnan(v) else round(float(v), 4)) for c, v in zip(CLASSES, iou)}} if w is not None and cm[w].sum() and cm[:, w].sum(): row["window_recall"] = round(float(cm[w, w] / cm[w].sum()), 4) row["window_precision"] = round(float(cm[w, w] / cm[:, w].sum()), 4) if base is not None and m != "plain": cur = np.stack(per_image[m]); diffs = [] for _ in range(400): idx = rng.integers(0, n, n) diffs.append(miou(cur[idx].sum(0))[0] - miou(base[idx].sum(0))[0]) row["mIoU_gain_vs_plain"] = round(float(np.mean(diffs)), 4) row["gain_95ci"] = [round(float(np.percentile(diffs, 2.5)), 4), round(float(np.percentile(diffs, 97.5)), 4)] res["modes"][m] = row print(m, json.dumps(row), flush=True) if a.out: json.dump(res, open(a.out, "w"), indent=1) if __name__ == "__main__": main()