"""Compare an FP16 SDXL-family checkpoint with a quantized copy using STOCK ComfyUI. Runs standard SDXL ComfyUI verification recipes on an unmodified ComfyUI v0.45.0 install. For calibration it also renders a BF16 copy of the same FP16 weights: BF16 is treated as "no visible loss", so the quantized file should stay close to the BF16-vs-FP16 distance. Usage: python verify_sdxl_quant.py --mode base|lightning|turbo|refiner --fp16 F --quant Q --out DIR [--comfy-path DIR] [--base-fp16 B] [--steps N] [--size 1024] [--seeds 3] """ import argparse import gc import json import os import sys import time DEFAULT_COMFY = os.environ.get("COMFY_PATH", "./ComfyUI") def _bootstrap_comfy(comfy_path: str): root = os.path.abspath(comfy_path) sys.path.insert(0, root) os.chdir(root) global np, torch, Image, structural_similarity, peak_signal_noise_ratio global mm, QuantizedTensor, comfy import numpy as np import torch from PIL import Image from skimage.metrics import structural_similarity, peak_signal_noise_ratio import comfy.model_management as mm import comfy.sample import comfy.samplers import comfy.sd import comfy.utils from comfy.quant_ops import QuantizedTensor PROMPTS = [ "Close-up portrait photo of an elderly fisherman with weathered skin and a grey beard, soft window light, 85mm", "A misty alpine lake at sunrise, pine forest, snow-capped mountains reflected in the water, landscape photography", "A vintage neon sign that says \"OPEN 24 HOURS\" on a rainy city street at night, reflections on wet asphalt", "Macro photo of a red fox's fur and amber eye, extremely detailed, shallow depth of field", "Isometric illustration of a cozy wizard's library with floating books and warm candlelight, digital art", "A gourmet burger with melted cheese and sesame bun on a wooden board, food photography, studio lighting", "Futuristic glass skyscraper in a desert city, golden hour, architectural photography, wide angle", "Watercolor painting of a hummingbird drinking from a purple flower, loose brush strokes, white paper", ] SEEDS = [42, 1337, 987654] def load(path, dtype=None): opts = {"dtype": dtype} if dtype is not None else {} model, clip, vae, _ = comfy.sd.load_checkpoint_guess_config( path, output_vae=True, output_clip=True, embedding_directory=None, model_options=opts) return model, clip, vae def count_quant(model): q = t = 0 for _, m in model.model.diffusion_model.named_modules(): w = getattr(m, "weight", None) if isinstance(w, torch.Tensor) and w.ndim >= 2: t += 1 if isinstance(w, QuantizedTensor) or getattr(m, "quant_format", None): q += 1 return q, t def encode(clip, text): tokens = clip.tokenize(text) return clip.encode_from_tokens_scheduled(tokens) def encode_refiner(clip, text, ascore, w, h): tokens = clip.tokenize(text) cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True) return [[cond, {"pooled_output": pooled, "aesthetic_score": ascore, "width": w, "height": h}]] def empty_latent(model, size): lat = torch.zeros([1, 4, size // 8, size // 8]) return comfy.sample.fix_empty_latent_channels(model, lat) def ksample(model, pos, neg, latent, seed, steps, cfg, scheduler, start=None, last=None, disable_noise=False, force_full=False): if disable_noise: noise = torch.zeros(latent.size(), dtype=latent.dtype, layout=latent.layout, device="cpu") else: noise = comfy.sample.prepare_noise(latent, seed) return comfy.sample.sample(model, noise, steps, cfg, "euler", scheduler, pos, neg, latent, denoise=1.0, disable_noise=disable_noise, start_step=start, last_step=last, force_full_denoise=force_full, seed=seed) def turbo_sample(model, pos, neg, latent, seed, steps): timesteps = torch.flip(torch.arange(1, 11) * 100 - 1, (0,))[:steps] sigmas = model.get_model_object("model_sampling").sigma(timesteps) sigmas = torch.cat([sigmas, sigmas.new_zeros([1])]) sampler = comfy.samplers.sampler_object("euler_ancestral") noise = comfy.sample.prepare_noise(latent, seed) return comfy.sample.sample_custom(model, noise, 1.0, sampler, sigmas, pos, neg, latent, seed=seed) def run_set(path, args, dtype=None, base_stage=None): """Render every prompt/seed. Returns ({key: latent}, vae, timing_dict).""" model, clip, vae = load(path, dtype) q, t = count_quant(model) print(f"[load] {os.path.basename(path)} dtype={dtype} quantized_layers={q}/{t}", flush=True) out = {} times = [] t0 = time.time() for pi, prompt in enumerate(PROMPTS): if args.mode == "refiner": pos = encode_refiner(clip, prompt, 6.0, args.size, args.size) neg = encode_refiner(clip, "", 2.5, args.size, args.size) else: pos, neg = encode(clip, prompt), encode(clip, "") for seed in SEEDS[: args.seeds]: key = f"p{pi}_s{seed}" ts = time.perf_counter() if args.mode == "base": lat = ksample(model, pos, neg, empty_latent(model, args.size), seed, args.steps, 6.0, "normal") elif args.mode == "lightning": lat = ksample(model, pos, neg, empty_latent(model, args.size), seed, 4, 1.0, "sgm_uniform") elif args.mode == "turbo": lat = turbo_sample(model, pos, neg, empty_latent(model, args.size), seed, args.steps) else: # refiner stage on top of a fixed FP16 base stage split = args.steps * 4 // 5 lat = ksample(model, pos, neg, base_stage[key], seed, args.steps, 6.0, "normal", start=split, last=args.steps, disable_noise=True, force_full=True) torch.cuda.synchronize() if torch.cuda.is_available() else None times.append(time.perf_counter() - ts) out[key] = lat.cpu() total_s = time.time() - t0 warm = times[1:] if len(times) > 1 else times warm_mean = float(np.mean(warm)) if warm else 0.0 print(f"[time] {len(out)} renders in {total_s:.1f}s (warm_mean={warm_mean:.2f}s/img)", flush=True) del model, clip return out, vae, {"quantized_layers": q, "total_layers": t, "total_s": total_s, "warm_mean_s": warm_mean} def base_stage_latents(base_path, args): model, clip, _ = load(base_path) split = args.steps * 4 // 5 out = {} for pi, prompt in enumerate(PROMPTS): pos, neg = encode(clip, prompt), encode(clip, "") for seed in SEEDS[: args.seeds]: out[f"p{pi}_s{seed}"] = ksample(model, pos, neg, empty_latent(model, args.size), seed, args.steps, 6.0, "normal", start=0, last=split).cpu() del model, clip free() return out def free(): mm.unload_all_models() gc.collect() mm.soft_empty_cache() def decode(vae, lat): with torch.inference_mode(): img = vae.decode(lat) return img[0].detach().float().clamp(0, 1).cpu().numpy() def cosine(a, b): a, b = a.flatten().double(), b.flatten().double() return float(torch.nn.functional.cosine_similarity(a, b, dim=0)) def main(): ap = argparse.ArgumentParser() ap.add_argument("--mode", required=True, choices=["base", "lightning", "turbo", "refiner"]) ap.add_argument("--fp16", required=True) ap.add_argument("--quant", required=True) ap.add_argument("--comfy-path", default=DEFAULT_COMFY) ap.add_argument("--base-fp16", default=None, help="SDXL Base, required for --mode refiner") ap.add_argument("--out", required=True) ap.add_argument("--cache-ref", default=None, help="Cache FP16/BF16 reference latents here across runs") ap.add_argument("--steps", type=int, default=None) ap.add_argument("--size", type=int, default=1024) ap.add_argument("--seeds", type=int, default=3) ap.add_argument("--no-bf16", action="store_true") args = ap.parse_args() _bootstrap_comfy(args.comfy_path) args.steps = args.steps or {"base": 50, "lightning": 4, "turbo": 2, "refiner": 40}[args.mode] os.makedirs(args.out, exist_ok=True) cache_file = os.path.join(args.cache_ref, "ref_cache.pt") if args.cache_ref else None if cache_file and os.path.isfile(cache_file): cached = torch.load(cache_file, map_location="cpu", weights_only=False) base_stage = cached.get("base_stage") ref, t_ref = cached["ref"], cached["t_ref"] bf16, t_bf16 = cached.get("bf16"), cached.get("t_bf16") _, _, vae = load(args.fp16) free() else: base_stage = base_stage_latents(args.base_fp16, args) if args.mode == "refiner" else None ref, vae, t_ref = run_set(args.fp16, args, base_stage=base_stage) free() bf16, t_bf16 = None, None if not args.no_bf16: bf16, _, t_bf16 = run_set(args.fp16, args, dtype=torch.bfloat16, base_stage=base_stage) free() if cache_file: os.makedirs(args.cache_ref, exist_ok=True) torch.save({"base_stage": base_stage, "ref": ref, "t_ref": t_ref, "bf16": bf16, "t_bf16": t_bf16}, cache_file) quant, _, t_quant = run_set(args.quant, args, base_stage=base_stage) free() rows = [] for key in ref: a = decode(vae, ref[key]) b = decode(vae, quant[key]) row = {"key": key, "q_cos": float(cosine(ref[key], quant[key])), "q_ssim": float(structural_similarity(a, b, channel_axis=2, data_range=1.0)), "q_psnr": float(peak_signal_noise_ratio(a, b, data_range=1.0))} panels = [a, b] if bf16 is not None: c = decode(vae, bf16[key]) row.update(b_cos=float(cosine(ref[key], bf16[key])), b_ssim=float(structural_similarity(a, c, channel_axis=2, data_range=1.0)), b_psnr=float(peak_signal_noise_ratio(a, c, data_range=1.0))) panels.append(c) rows.append(row) grid = np.concatenate(panels, axis=1) Image.fromarray((grid * 255).round().astype(np.uint8)).save(os.path.join(args.out, f"{key}.png")) print(json.dumps({k: (round(v, 5) if isinstance(v, float) else v) for k, v in row.items()}), flush=True) def stats(name): vals = [r[name] for r in rows if name in r] return {"mean": float(np.mean(vals)), "min": float(np.min(vals))} if vals else None summary = {k: stats(k) for k in ("q_cos", "q_ssim", "q_psnr", "b_cos", "b_ssim", "b_psnr")} summary.update(mode=args.mode, fp16=args.fp16, quant=args.quant, steps=args.steps, size=args.size, renders=len(rows), fp16_bytes=os.path.getsize(args.fp16), quant_bytes=os.path.getsize(args.quant), timing={"fp16": t_ref, "bf16": t_bf16, "quant": t_quant}) with open(os.path.join(args.out, "summary.json"), "w") as f: json.dump({"summary": summary, "rows": rows}, f, indent=1) print("SUMMARY " + json.dumps(summary), flush=True) if __name__ == "__main__": main()