constructelligence's picture
Update code: real-machine G-code adapter, commissioning gate, machine_job runner, pump watchdog firmware, twin completion gate, realism and masking modules (card and weights unchanged)
4eecd03 verified
Raw History Blame Contribute Delete
4.52 kB
#!/usr/bin/env python3
"""Does test-time augmentation (flip, multi-scale) help the wall model on held-out photos? Same data, same metric, only the inference differs.
python3 eval_tta.py --checkpoint artifacts/candidates/C2-cal.pt --data real_data --split test --out runs_tta_test.json
Reports per-class IoU, mIoU over classes present, window recall/precision (the safety-relevant class) and painted-coverage MAE for each mode, plus
a bootstrap over images for the mIoU difference to the plain model, so a gain inside the noise is not reported as a gain."""
import argparse
import json
import time
import numpy as np
import torch
import torch.nn.functional as F
from benchmark import confusion
from label_schema import CLASSES
from models import DEFAULT_ARCH, build_from_checkpoint
from train import WallDataset
MODES = {"plain": [(1.0, False)], "flip": [(1.0, False), (1.0, True)],
"flip+ms": [(s, f) for s in (0.75, 1.0, 1.25) for f in (False, True)], "ms": [(s, False) for s in (0.75, 1.0, 1.25)]}
def predict(model, x, mode):
n_c = len(CLASSES)
acc = None
for scale, flip in MODES[mode]:
xi = x
if scale != 1.0:
h, w = x.shape[-2:]
xi = F.interpolate(x, size=(int(round(h * scale / 32) * 32), int(round(w * scale / 32) * 32)), mode="bilinear", align_corners=False)
if flip:
xi = torch.flip(xi, dims=[3])
out = model(xi)["semantic"]
if flip:
out = torch.flip(out, dims=[3])
if out.shape[-2:] != x.shape[-2:]:
out = F.interpolate(out, size=x.shape[-2:], mode="bilinear", align_corners=False)
p = out.softmax(1)
acc = p if acc is None else acc + p
return (acc / len(MODES[mode])).argmax(1)
def miou(cm):
inter = np.diag(cm).astype(float)
union = cm.sum(0) + cm.sum(1) - np.diag(cm)
present = cm.sum(1) > 0
iou = np.where(union > 0, inter / np.maximum(union, 1), np.nan)
return float(np.nanmean(iou[present])), iou
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--checkpoint", required=True)
ap.add_argument("--data", default="real_data")
ap.add_argument("--split", default="test")
ap.add_argument("--size", type=int, default=640)
ap.add_argument("--limit", type=int)
ap.add_argument("--modes", default="plain,flip,ms,flip+ms")
ap.add_argument("--out")
a = ap.parse_args()
ck = torch.load(a.checkpoint, map_location="cpu", weights_only=True)
model = build_from_checkpoint(ck, len(CLASSES)).eval()
ds = WallDataset(a.data, a.split, a.size)
n = min(len(ds), a.limit or len(ds))
modes = a.modes.split(",")
per_image = {m: [] for m in modes}
cms = {m: np.zeros((len(CLASSES),) * 2, np.int64) for m in modes}
t0 = time.time()
with torch.inference_mode():
for i in range(n):
x, y, *_ = ds[i]
x, y = x.unsqueeze(0), y.numpy()
for m in modes:
p = predict(model, x, m)[0].numpy()
c = confusion(p, y, len(CLASSES))
cms[m] += c
per_image[m].append(c)
if i % 5 == 0:
print(f"{i}/{n} {time.time() - t0:.0f}s", flush=True)
rng = np.random.default_rng(0)
res = {"checkpoint": a.checkpoint, "split": a.split, "images": n, "modes": {}}
base = np.stack(per_image["plain"]) if "plain" in per_image else None
for m in modes:
mi, iou = miou(cms[m])
cm = cms[m]
w = CLASSES.index("window") if "window" in CLASSES else None
row = {"mIoU": round(mi, 4), "iou": {c: (None if np.isnan(v) else round(float(v), 4)) for c, v in zip(CLASSES, iou)}}
if w is not None and cm[w].sum() and cm[:, w].sum():
row["window_recall"] = round(float(cm[w, w] / cm[w].sum()), 4)
row["window_precision"] = round(float(cm[w, w] / cm[:, w].sum()), 4)
if base is not None and m != "plain":
cur = np.stack(per_image[m]); diffs = []
for _ in range(400):
idx = rng.integers(0, n, n)
diffs.append(miou(cur[idx].sum(0))[0] - miou(base[idx].sum(0))[0])
row["mIoU_gain_vs_plain"] = round(float(np.mean(diffs)), 4)
row["gain_95ci"] = [round(float(np.percentile(diffs, 2.5)), 4), round(float(np.percentile(diffs, 97.5)), 4)]
res["modes"][m] = row
print(m, json.dumps(row), flush=True)
if a.out:
json.dump(res, open(a.out, "w"), indent=1)
if __name__ == "__main__":
main()