"""Auditable, matched image stacks. This never synthesizes comparison imagery.""" from __future__ import annotations import argparse import hashlib import html import json import math from pathlib import Path import numpy as np from PIL import Image, ImageDraw, ImageFont def image_metrics(reference, candidate): a, b = np.asarray(reference.convert("RGB")), np.asarray(candidate.convert("RGB")) if a.shape != b.shape: raise ValueError(f"Cannot score different image sizes: {a.shape} vs {b.shape}") diff = a.astype(np.float64) - b.astype(np.float64) mse = float(np.mean(diff**2)) result = { "pixel_identical": bool(np.array_equal(a, b)), "mae_8bit": float(np.mean(abs(diff))), "max_error_8bit": int(np.max(abs(diff))), "psnr_db": None if mse == 0 else 20 * math.log10(255 / math.sqrt(mse)), } try: from skimage.metrics import structural_similarity result["ssim_rgb"] = float( structural_similarity(a, b, channel_axis=-1, data_range=255) ) except ImportError: result["ssim_rgb"] = None return result def font(size): for path in ( "/System/Library/Fonts/Supplemental/Arial.ttf", "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", ): if Path(path).exists(): return ImageFont.truetype(path, size) return ImageFont.load_default(size=size) def _verify_identity(reference_path, candidate_path): ra, rb = reference_path.with_suffix(".json"), candidate_path.with_suffix(".json") if not ra.exists() or not rb.exists(): raise ValueError( "Every comparison image must have a sibling inference JSON record" ) a, b = json.loads(ra.read_text()), json.loads(rb.read_text()) for key in ( "prompt", "seed", "size", "canvas", "rng", "reference_images", "sigmas", ): if a.get(key) != b.get(key): raise ValueError(f"Unmatched {key} for {candidate_path}") if a.get("reference_images") and a.get("reference_sha256") != b.get( "reference_sha256" ): raise ValueError(f"Unmatched reference image contents for {candidate_path}") if b.get("validation_scope") == "Same final latent; decoder only": expected = a.get("source_trace_sha256") if expected is None: trace = reference_path.with_suffix(".trace.safetensors") if trace.exists(): expected = hashlib.sha256(trace.read_bytes()).hexdigest() if expected is None or expected != b.get("source_trace_sha256"): raise ValueError(f"Unmatched final latent provenance for {candidate_path}") return a, b def make_comparison(reference, candidates, destination, *, labels=None, crop=None): reference, destination = Path(reference), Path(destination) destination.mkdir(parents=True, exist_ok=True) paths = [reference, *map(Path, candidates)] labels = labels or [p.stem for p in paths] if len(labels) != len(paths): raise ValueError("One label per image is required, including the reference") images = [Image.open(p).convert("RGB") for p in paths] records = [json.loads(reference.with_suffix(".json").read_text())] for p in paths[1:]: _, row = _verify_identity(reference, p) records.append(row) metrics = [image_metrics(images[0], im) for im in images] iw, ih = images[0].size if crop is None: side = min(512, iw, ih) crop = ((iw - side) // 2, (ih - side) // 2, (iw + side) // 2, (ih + side) // 2) if len(crop) != 4 or not ( 0 <= crop[0] < crop[2] <= iw and 0 <= crop[1] < crop[3] <= ih ): raise ValueError("Crop must lie inside the native-resolution image") preview_width = min(768, iw) preview_height = round(ih * preview_width / iw) label_height = 84 full = Image.new( "RGB", (preview_width, len(paths) * (preview_height + label_height)), "#10141c" ) detail_w, detail_h = crop[2] - crop[0], crop[3] - crop[1] details = Image.new( "RGB", (detail_w, len(paths) * (detail_h + label_height)), "#10141c" ) heatmap = Image.new( "RGB", (preview_width, len(paths) * (preview_height + label_height)), "#10141c" ) source = np.asarray(images[0]).astype(np.int16) rows = [] for i, (p, im, label, metric, record) in enumerate( zip(paths, images, labels, metrics, records, strict=False) ): psnr = ( "infinity" if metric["psnr_db"] is None else f"{metric['psnr_db']:.2f} dB" ) ss = ( "unavailable" if metric["ssim_rgb"] is None else f"{metric['ssim_rgb']:.5f}" ) seconds = record.get("timings", {}).get("total_s") caption = f"PSNR {psnr} SSIM {ss} MAE {metric['mae_8bit']:.3f}/255" if record.get("steps"): caption += f" {len(record['steps'])} passes" if seconds is not None: sample = ( "warm" if record.get("dit_was_resident") else "load+run" if record.get("dit_was_resident") is False else "recorded" ) caption += f" {sample} {seconds:.2f}s" elif record.get("timing_scope") and any( k.startswith("step_") for k in record.get("timings", {}) ): core = sum(v for k, v in record["timings"].items() if k.startswith("step_")) caption += f" DiT {core:.2f}s" elif record.get("timing_scope") and "decode_s" in record.get("timings", {}): caption += f" decoder {record['timings']['decode_s']:.2f}s" for canvas, stride in ( (full, preview_height + label_height), (details, detail_h + label_height), (heatmap, preview_height + label_height), ): draw = ImageDraw.Draw(canvas) draw.text((12, i * stride + 10), label, font=font(19), fill="white") draw.text((12, i * stride + 43), caption, font=font(13), fill="#b9c8dc") full.paste( im.resize((preview_width, preview_height), Image.Resampling.LANCZOS), (0, i * (preview_height + label_height) + label_height), ) details.paste(im.crop(crop), (0, i * (detail_h + label_height) + label_height)) error = np.minimum( 255, np.abs(np.asarray(im).astype(np.int16) - source) * 8 ).astype(np.uint8) heatmap.paste( Image.fromarray(error).resize((preview_width, preview_height)), (0, i * (preview_height + label_height) + label_height), ) rows.append( { "label": label, "image": str(p.resolve()), "png_sha256": hashlib.sha256(p.read_bytes()).hexdigest(), **metric, "timings": record.get("timings"), "steps": record.get("steps"), "timing_scope": record.get("timing_scope", "Fresh end-to-end request"), } ) full.save(destination / "stack.png") details.save(destination / "details.png") heatmap.save(destination / "difference-x8.png") # Side-by-side rows make small losses much easier to inspect than scrolling # a tall stack. Keep the original stacks and exact native-resolution crops. columns = min(3, len(paths)) grid_rows = math.ceil(len(paths) / columns) grid_w = min(512, iw) grid_h = round(ih * grid_w / iw) for filename, panels, pw, ph in ( ( "grid.png", [im.resize((grid_w, grid_h), Image.Resampling.LANCZOS) for im in images], grid_w, grid_h, ), ("details-grid.png", [im.crop(crop) for im in images], detail_w, detail_h), ): grid = Image.new( "RGB", (columns * pw, grid_rows * (ph + label_height)), "#10141c" ) draw = ImageDraw.Draw(grid) for i, (panel, label, metric) in enumerate( zip(panels, labels, metrics, strict=False) ): gx, gy = (i % columns) * pw, (i // columns) * (ph + label_height) draw.text((gx + 12, gy + 10), label, font=font(18), fill="white") ss = "n/a" if metric["ssim_rgb"] is None else f"{metric['ssim_rgb']:.5f}" draw.text( (gx + 12, gy + 42), f"SSIM {ss} MAE {metric['mae_8bit']:.3f}/255", font=font(14), fill="#b9c8dc", ) grid.paste(panel, (gx, gy + label_height)) grid.save(destination / filename) scope = records[0].get("validation_scope", "Complete image inference") if any( r.get("validation_scope") == "Same final latent; decoder only" for r in records ): scope = "Same final latent; decoder-only comparison. Source-generation and decoder-only timings have different scopes." if records[0].get("source_record"): scope = "Cross-backend comparison: hardware and VAE quality settings differ. Timings do not isolate a port speedup." report = { "validation_scope": scope, "reference": str(reference.resolve()), "native_size": [iw, ih], "crop_xyxy": list(crop), "heatmap_gain": 8, "metric_definition": { "psnr": "Full-resolution 8-bit RGB MSE, data range 255; null means infinity", "ssim": "scikit-image RGB structural_similarity, channel_axis=-1, data_range=255, default 7x7 window", "mae": "Full-resolution mean absolute 8-bit RGB error", }, "rows": rows, "interpretation": "Pixel similarity to the same seeded trajectory, not a measure of prompt adherence or perceptual quality.", } (destination / "metrics.json").write_text(json.dumps(report, indent=2) + "\n") title = html.escape(reference.stem) page = f""" {title} — H3 MLX precision comparison

{title}: matched precision comparison

Same prompt, seed, noise generator, size and edit references. Each row names its configuration. Fewer steps are a separate approximation. Full frames are resized for display; detail crops use identical native pixels.

{html.escape(scope)}

PSNR/SSIM describe similarity to the reference trajectory. They do not establish visual quality or prompt accuracy. Raw measurements.

Side-by-side frames

Matched images side by side

Side-by-side native detail

Identical native-resolution crop coordinates

Full frames

1:1 detail crops

Absolute RGB difference ×8

""" (destination / "index.html").write_text(page) return report def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument("--reference", required=True) p.add_argument("--candidate", action="append", required=True) p.add_argument("--label", action="append", help="One per image: reference first") p.add_argument("--crop", type=int, nargs=4) p.add_argument("--output", required=True) args = p.parse_args() make_comparison( args.reference, args.candidate, args.output, labels=args.label, crop=args.crop ) if __name__ == "__main__": main()