devin-lai's picture
Release v0.4.0: selective MLX model downloads
3da88e3 verified
Raw History Blame Contribute Delete
11.9 kB
"""Auditable, matched image stacks. This never synthesizes comparison imagery."""
from __future__ import annotations
import argparse
import hashlib
import html
import json
import math
from pathlib import Path
import numpy as np
from PIL import Image, ImageDraw, ImageFont
def image_metrics(reference, candidate):
a, b = np.asarray(reference.convert("RGB")), np.asarray(candidate.convert("RGB"))
if a.shape != b.shape:
raise ValueError(f"Cannot score different image sizes: {a.shape} vs {b.shape}")
diff = a.astype(np.float64) - b.astype(np.float64)
mse = float(np.mean(diff**2))
result = {
"pixel_identical": bool(np.array_equal(a, b)),
"mae_8bit": float(np.mean(abs(diff))),
"max_error_8bit": int(np.max(abs(diff))),
"psnr_db": None if mse == 0 else 20 * math.log10(255 / math.sqrt(mse)),
}
try:
from skimage.metrics import structural_similarity
result["ssim_rgb"] = float(
structural_similarity(a, b, channel_axis=-1, data_range=255)
)
except ImportError:
result["ssim_rgb"] = None
return result
def font(size):
for path in (
"/System/Library/Fonts/Supplemental/Arial.ttf",
"/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
):
if Path(path).exists():
return ImageFont.truetype(path, size)
return ImageFont.load_default(size=size)
def _verify_identity(reference_path, candidate_path):
ra, rb = reference_path.with_suffix(".json"), candidate_path.with_suffix(".json")
if not ra.exists() or not rb.exists():
raise ValueError(
"Every comparison image must have a sibling inference JSON record"
)
a, b = json.loads(ra.read_text()), json.loads(rb.read_text())
for key in (
"prompt",
"seed",
"size",
"canvas",
"rng",
"reference_images",
"sigmas",
):
if a.get(key) != b.get(key):
raise ValueError(f"Unmatched {key} for {candidate_path}")
if a.get("reference_images") and a.get("reference_sha256") != b.get(
"reference_sha256"
):
raise ValueError(f"Unmatched reference image contents for {candidate_path}")
if b.get("validation_scope") == "Same final latent; decoder only":
expected = a.get("source_trace_sha256")
if expected is None:
trace = reference_path.with_suffix(".trace.safetensors")
if trace.exists():
expected = hashlib.sha256(trace.read_bytes()).hexdigest()
if expected is None or expected != b.get("source_trace_sha256"):
raise ValueError(f"Unmatched final latent provenance for {candidate_path}")
return a, b
def make_comparison(reference, candidates, destination, *, labels=None, crop=None):
reference, destination = Path(reference), Path(destination)
destination.mkdir(parents=True, exist_ok=True)
paths = [reference, *map(Path, candidates)]
labels = labels or [p.stem for p in paths]
if len(labels) != len(paths):
raise ValueError("One label per image is required, including the reference")
images = [Image.open(p).convert("RGB") for p in paths]
records = [json.loads(reference.with_suffix(".json").read_text())]
for p in paths[1:]:
_, row = _verify_identity(reference, p)
records.append(row)
metrics = [image_metrics(images[0], im) for im in images]
iw, ih = images[0].size
if crop is None:
side = min(512, iw, ih)
crop = ((iw - side) // 2, (ih - side) // 2, (iw + side) // 2, (ih + side) // 2)
if len(crop) != 4 or not (
0 <= crop[0] < crop[2] <= iw and 0 <= crop[1] < crop[3] <= ih
):
raise ValueError("Crop must lie inside the native-resolution image")
preview_width = min(768, iw)
preview_height = round(ih * preview_width / iw)
label_height = 84
full = Image.new(
"RGB", (preview_width, len(paths) * (preview_height + label_height)), "#10141c"
)
detail_w, detail_h = crop[2] - crop[0], crop[3] - crop[1]
details = Image.new(
"RGB", (detail_w, len(paths) * (detail_h + label_height)), "#10141c"
)
heatmap = Image.new(
"RGB", (preview_width, len(paths) * (preview_height + label_height)), "#10141c"
)
source = np.asarray(images[0]).astype(np.int16)
rows = []
for i, (p, im, label, metric, record) in enumerate(
zip(paths, images, labels, metrics, records, strict=False)
):
psnr = (
"infinity" if metric["psnr_db"] is None else f"{metric['psnr_db']:.2f} dB"
)
ss = (
"unavailable" if metric["ssim_rgb"] is None else f"{metric['ssim_rgb']:.5f}"
)
seconds = record.get("timings", {}).get("total_s")
caption = f"PSNR {psnr} SSIM {ss} MAE {metric['mae_8bit']:.3f}/255"
if record.get("steps"):
caption += f" {len(record['steps'])} passes"
if seconds is not None:
sample = (
"warm"
if record.get("dit_was_resident")
else "load+run"
if record.get("dit_was_resident") is False
else "recorded"
)
caption += f" {sample} {seconds:.2f}s"
elif record.get("timing_scope") and any(
k.startswith("step_") for k in record.get("timings", {})
):
core = sum(v for k, v in record["timings"].items() if k.startswith("step_"))
caption += f" DiT {core:.2f}s"
elif record.get("timing_scope") and "decode_s" in record.get("timings", {}):
caption += f" decoder {record['timings']['decode_s']:.2f}s"
for canvas, stride in (
(full, preview_height + label_height),
(details, detail_h + label_height),
(heatmap, preview_height + label_height),
):
draw = ImageDraw.Draw(canvas)
draw.text((12, i * stride + 10), label, font=font(19), fill="white")
draw.text((12, i * stride + 43), caption, font=font(13), fill="#b9c8dc")
full.paste(
im.resize((preview_width, preview_height), Image.Resampling.LANCZOS),
(0, i * (preview_height + label_height) + label_height),
)
details.paste(im.crop(crop), (0, i * (detail_h + label_height) + label_height))
error = np.minimum(
255, np.abs(np.asarray(im).astype(np.int16) - source) * 8
).astype(np.uint8)
heatmap.paste(
Image.fromarray(error).resize((preview_width, preview_height)),
(0, i * (preview_height + label_height) + label_height),
)
rows.append(
{
"label": label,
"image": str(p.resolve()),
"png_sha256": hashlib.sha256(p.read_bytes()).hexdigest(),
**metric,
"timings": record.get("timings"),
"steps": record.get("steps"),
"timing_scope": record.get("timing_scope", "Fresh end-to-end request"),
}
)
full.save(destination / "stack.png")
details.save(destination / "details.png")
heatmap.save(destination / "difference-x8.png")
# Side-by-side rows make small losses much easier to inspect than scrolling
# a tall stack. Keep the original stacks and exact native-resolution crops.
columns = min(3, len(paths))
grid_rows = math.ceil(len(paths) / columns)
grid_w = min(512, iw)
grid_h = round(ih * grid_w / iw)
for filename, panels, pw, ph in (
(
"grid.png",
[im.resize((grid_w, grid_h), Image.Resampling.LANCZOS) for im in images],
grid_w,
grid_h,
),
("details-grid.png", [im.crop(crop) for im in images], detail_w, detail_h),
):
grid = Image.new(
"RGB", (columns * pw, grid_rows * (ph + label_height)), "#10141c"
)
draw = ImageDraw.Draw(grid)
for i, (panel, label, metric) in enumerate(
zip(panels, labels, metrics, strict=False)
):
gx, gy = (i % columns) * pw, (i // columns) * (ph + label_height)
draw.text((gx + 12, gy + 10), label, font=font(18), fill="white")
ss = "n/a" if metric["ssim_rgb"] is None else f"{metric['ssim_rgb']:.5f}"
draw.text(
(gx + 12, gy + 42),
f"SSIM {ss} MAE {metric['mae_8bit']:.3f}/255",
font=font(14),
fill="#b9c8dc",
)
grid.paste(panel, (gx, gy + label_height))
grid.save(destination / filename)
scope = records[0].get("validation_scope", "Complete image inference")
if any(
r.get("validation_scope") == "Same final latent; decoder only" for r in records
):
scope = "Same final latent; decoder-only comparison. Source-generation and decoder-only timings have different scopes."
if records[0].get("source_record"):
scope = "Cross-backend comparison: hardware and VAE quality settings differ. Timings do not isolate a port speedup."
report = {
"validation_scope": scope,
"reference": str(reference.resolve()),
"native_size": [iw, ih],
"crop_xyxy": list(crop),
"heatmap_gain": 8,
"metric_definition": {
"psnr": "Full-resolution 8-bit RGB MSE, data range 255; null means infinity",
"ssim": "scikit-image RGB structural_similarity, channel_axis=-1, data_range=255, default 7x7 window",
"mae": "Full-resolution mean absolute 8-bit RGB error",
},
"rows": rows,
"interpretation": "Pixel similarity to the same seeded trajectory, not a measure of prompt adherence or perceptual quality.",
}
(destination / "metrics.json").write_text(json.dumps(report, indent=2) + "\n")
title = html.escape(reference.stem)
page = f"""<!doctype html><html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width">
<title>{title} — H3 MLX precision comparison</title><style>body{{background:#10141c;color:#edf3ff;font:16px system-ui;max-width:1500px;margin:40px auto;padding:0 24px}} img{{max-width:100%;height:auto}} .columns{{display:flex;gap:24px;align-items:flex-start}} .columns>div{{flex:1}} a{{color:#91c8ff}}</style>
<h1>{title}: matched precision comparison</h1><p>Same prompt, seed, noise generator, size and edit references. Each row names its configuration. Fewer steps are a separate approximation. Full frames are resized for display; detail crops use identical native pixels.</p>
<p>{html.escape(scope)}</p>
<p>PSNR/SSIM describe similarity to the reference trajectory. They do not establish visual quality or prompt accuracy. <a href="metrics.json">Raw measurements</a>.</p>
<h2>Side-by-side frames</h2><a href="grid.png"><img src="grid.png" alt="Matched images side by side"></a>
<h2>Side-by-side native detail</h2><a href="details-grid.png"><img src="details-grid.png" alt="Identical native-resolution crop coordinates"></a>
<div class="columns"><div><h2>Full frames</h2><img src="stack.png"></div><div><h2>1:1 detail crops</h2><img src="details.png"></div><div><h2>Absolute RGB difference ×8</h2><img src="difference-x8.png"></div></div></html>"""
(destination / "index.html").write_text(page)
return report
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--reference", required=True)
p.add_argument("--candidate", action="append", required=True)
p.add_argument("--label", action="append", help="One per image: reference first")
p.add_argument("--crop", type=int, nargs=4)
p.add_argument("--output", required=True)
args = p.parse_args()
make_comparison(
args.reference, args.candidate, args.output, labels=args.label, crop=args.crop
)
if __name__ == "__main__":
main()