"""Compare the old centre-crop with the letterbox on real non-square photos. There are no labels for these photos here, so the reference is the model's own prediction, and the numbers are *differences*, read against yardsticks: crop vs crop-run-twice the ONNX export samples a little input noise per call crop vs mirrored run a benign test-time variation (the mirror average) crop vs letterbox the thing being tested, on the shared central region edge band vs inner does reflected padding hurt next to the padding? Depth is also compared after a least-squares scale+shift (its scale is relative). docker compose run --rm train bash -c "pip install -q onnxruntime huggingface_hub && python space/test_letterbox.py" """ import glob import os import sys import numpy as np import onnxruntime as ort import torch import torch.nn.functional as F from huggingface_hub import hf_hub_download from PIL import Image HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) from framing import crop_to_box, letterbox, resize_maps # noqa: E402 SIZE = 448 OUT = os.path.join(os.path.dirname(HERE), "_lb_out") os.makedirs(OUT, exist_ok=True) so = ort.SessionOptions() so.log_severity_level = 3 so.intra_op_num_threads = 4 SESS = ort.InferenceSession( hf_hub_download("singam96/ShadeNet-5-35M", "onnx/shadenet5_mix_int4_int8.onnx"), so, providers=["CPUExecutionProvider"]) def tensor(img): a = np.asarray(img.convert("RGB"), dtype=np.float32) / 255.0 return torch.from_numpy(a.transpose(2, 0, 1)[None] * 2 - 1) def run(x): return torch.from_numpy(SESS.run(None, {"input_rgb": np.ascontiguousarray(x.numpy())})[0]) def centre_crop(img): # the old behaviour w, h = img.size s = max(SIZE / w, SIZE / h) nw, nh = int(round(w * s)), int(round(h * s)) im = img.convert("RGB").resize((nw, nh), Image.BICUBIC) l, t = (nw - SIZE) // 2, (nh - SIZE) // 2 return im.crop((l, t, l + SIZE, t + SIZE)) def mirrored(x): m = run(x.flip(-1)).flip(-1).clone() m[:, 4] = -m[:, 4] return m def ssi(a, b): """Depth a aligned to b by least-squares scale+shift.""" A = torch.stack([a.flatten(), torch.ones(a.numel())], 1) sol = torch.linalg.lstsq(A, b.flatten()[:, None]).solution return (A @ sol).reshape(a.shape) def diffs(a, b): return (float((a[:, 0:3] - b[:, 0:3]).abs().mean()), float((a[:, 3:4] - b[:, 3:4]).abs().mean()), float((ssi(a[:, 3:4], b[:, 3:4]) - b[:, 3:4]).abs().mean()), float((a[:, 4:7] - b[:, 4:7]).abs().mean())) def tile(x01, ts): if x01.shape[1] == 1: x01 = x01.expand(-1, 3, -1, -1) return Image.fromarray((x01[0].permute(1, 2, 0).clamp(0, 1).numpy() * 255).round().astype("uint8")).resize(ts, Image.BICUBIC) rows = [] for f in sorted(glob.glob(os.path.join(os.path.dirname(HERE), "dist", "hf_bundle", "assets", "examples", "*.jpg"))): img = Image.open(f).convert("RGB") w, h = img.size xc = tensor(centre_crop(img)) mc, mc2, mm = run(xc), run(xc), mirrored(xc) sq, box = letterbox(img, SIZE) ml = crop_to_box(run(tensor(sq)), box) # [1,7,nh,nw] real photo only left, top, nw, nh = box # shared region: what the crop saw, in the letterbox photo's coordinates if w >= h: c0, c1 = int(round(nw * (w - h) / (2 * w))), int(round(nw * (w + h) / (2 * w))) shared = ml[:, :, :, c0:c1] edge = torch.zeros(nh, c1 - c0, dtype=torch.bool) k = max(1, int(0.125 * nh)); edge[:k] = True; edge[-k:] = True else: r0, r1 = int(round(nh * (h - w) / (2 * h))), int(round(nh * (h + w) / (2 * h))) shared = ml[:, :, r0:r1, :] edge = torch.zeros(r1 - r0, nw, dtype=torch.bool) k = max(1, int(0.125 * nw)); edge[:, :k] = True; edge[:, -k:] = True shared = F.interpolate(shared, size=(SIZE, SIZE), mode="bicubic", align_corners=False).clamp(-1, 1) edge_up = F.interpolate(edge[None, None].float(), size=(SIZE, SIZE), mode="nearest")[0, 0] > 0.5 d_noise, d_mirror, d_lb = diffs(mc, mc2), diffs(mc, mm), diffs(shared, mc) inner = (~edge_up)[None, None] e_in = float(((shared - mc).abs() * inner).sum() / (inner.sum() * 7)) e_edge = float(((shared - mc).abs() * edge_up[None, None]).sum() / (edge_up.sum() * 7)) rows.append((os.path.basename(f)[:10], f"{w}x{h}", d_noise, d_mirror, d_lb, e_edge, e_in)) # picture: letterboxed input + its maps (padding cropped off) over the crop's maps ts = (nw, nh) top_row = [Image.fromarray(np.asarray(sq)[top:top + nh, left:left + nw])] + [tile((ml[:, a:b] + 1) / 2, ts) for a, b in ((0, 3), (3, 4), (4, 7))] s = 360 / max(nw, nh) cw, ch = int(nw * s), int(nh * s) canvas = Image.new("RGB", (4 * cw, ch + 360), (24, 24, 24)) for i, t in enumerate(top_row): canvas.paste(t.resize((cw, ch)), (i * cw, 0)) canvas.paste(centre_crop(img).resize((360, 360)), (0, ch)) for i, (a, b) in enumerate(((0, 3), (3, 4), (4, 7))): canvas.paste(tile((mc[:, a:b] + 1) / 2, (360, 360)), ((i + 1) * 360, ch)) canvas.save(os.path.join(OUT, os.path.basename(f).replace(".jpg", ".png"))) print(f"{'image':11s} {'size':8s} | mean |diff| over 7 maps: alb/depth/depth-aligned/normal") hdr = ["noise (run twice)", "crop vs mirrored", "crop vs letterbox"] for r in rows: print(f"{r[0]:11s} {r[1]:8s} | " + " | ".join(f"{n}: " + "/".join(f"{v:.3f}" for v in d) for n, d in zip(hdr, r[2:5]))) mean = lambda i: np.mean([np.array(r[i]) for r in rows], axis=0) print("\nMEAN over photos (alb/depth/depth-aligned/normal)") for n, i in zip(hdr, (2, 3, 4)): print(f" {n:20s}", "/".join(f"{v:.4f}" for v in mean(i))) print(f"\nletterbox vs crop, mean over all 7 channels: edge band {np.mean([r[5] for r in rows]):.4f} inner {np.mean([r[6] for r in rows]):.4f}") print("pictures in", OUT)