SDXL-Lightning-ConvRot / scripts /verify_sdxl_quant.py
SUSSYEATING's picture
Initialize SDXL-Lightning-ConvRot docs, metrics, comparisons, scripts, and license
0809b1b verified
Raw History Blame Contribute Delete
11.4 kB
"""Compare an FP16 SDXL-family checkpoint with a quantized copy using STOCK ComfyUI.
Runs standard SDXL ComfyUI verification recipes on an unmodified ComfyUI v0.45.0 install. For calibration it also
renders a BF16 copy of the same FP16 weights: BF16 is treated as "no visible loss", so the
quantized file should stay close to the BF16-vs-FP16 distance.
Usage:
python verify_sdxl_quant.py --mode base|lightning|turbo|refiner --fp16 F --quant Q --out DIR
[--comfy-path DIR] [--base-fp16 B] [--steps N] [--size 1024] [--seeds 3]
"""
import argparse
import gc
import json
import os
import sys
import time
DEFAULT_COMFY = os.environ.get("COMFY_PATH", "./ComfyUI")
def _bootstrap_comfy(comfy_path: str):
root = os.path.abspath(comfy_path)
sys.path.insert(0, root)
os.chdir(root)
global np, torch, Image, structural_similarity, peak_signal_noise_ratio
global mm, QuantizedTensor, comfy
import numpy as np
import torch
from PIL import Image
from skimage.metrics import structural_similarity, peak_signal_noise_ratio
import comfy.model_management as mm
import comfy.sample
import comfy.samplers
import comfy.sd
import comfy.utils
from comfy.quant_ops import QuantizedTensor
PROMPTS = [
"Close-up portrait photo of an elderly fisherman with weathered skin and a grey beard, soft window light, 85mm",
"A misty alpine lake at sunrise, pine forest, snow-capped mountains reflected in the water, landscape photography",
"A vintage neon sign that says \"OPEN 24 HOURS\" on a rainy city street at night, reflections on wet asphalt",
"Macro photo of a red fox's fur and amber eye, extremely detailed, shallow depth of field",
"Isometric illustration of a cozy wizard's library with floating books and warm candlelight, digital art",
"A gourmet burger with melted cheese and sesame bun on a wooden board, food photography, studio lighting",
"Futuristic glass skyscraper in a desert city, golden hour, architectural photography, wide angle",
"Watercolor painting of a hummingbird drinking from a purple flower, loose brush strokes, white paper",
]
SEEDS = [42, 1337, 987654]
def load(path, dtype=None):
opts = {"dtype": dtype} if dtype is not None else {}
model, clip, vae, _ = comfy.sd.load_checkpoint_guess_config(
path, output_vae=True, output_clip=True, embedding_directory=None, model_options=opts)
return model, clip, vae
def count_quant(model):
q = t = 0
for _, m in model.model.diffusion_model.named_modules():
w = getattr(m, "weight", None)
if isinstance(w, torch.Tensor) and w.ndim >= 2:
t += 1
if isinstance(w, QuantizedTensor) or getattr(m, "quant_format", None):
q += 1
return q, t
def encode(clip, text):
tokens = clip.tokenize(text)
return clip.encode_from_tokens_scheduled(tokens)
def encode_refiner(clip, text, ascore, w, h):
tokens = clip.tokenize(text)
cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True)
return [[cond, {"pooled_output": pooled, "aesthetic_score": ascore, "width": w, "height": h}]]
def empty_latent(model, size):
lat = torch.zeros([1, 4, size // 8, size // 8])
return comfy.sample.fix_empty_latent_channels(model, lat)
def ksample(model, pos, neg, latent, seed, steps, cfg, scheduler, start=None, last=None,
disable_noise=False, force_full=False):
if disable_noise:
noise = torch.zeros(latent.size(), dtype=latent.dtype, layout=latent.layout, device="cpu")
else:
noise = comfy.sample.prepare_noise(latent, seed)
return comfy.sample.sample(model, noise, steps, cfg, "euler", scheduler, pos, neg, latent,
denoise=1.0, disable_noise=disable_noise, start_step=start,
last_step=last, force_full_denoise=force_full, seed=seed)
def turbo_sample(model, pos, neg, latent, seed, steps):
timesteps = torch.flip(torch.arange(1, 11) * 100 - 1, (0,))[:steps]
sigmas = model.get_model_object("model_sampling").sigma(timesteps)
sigmas = torch.cat([sigmas, sigmas.new_zeros([1])])
sampler = comfy.samplers.sampler_object("euler_ancestral")
noise = comfy.sample.prepare_noise(latent, seed)
return comfy.sample.sample_custom(model, noise, 1.0, sampler, sigmas, pos, neg, latent, seed=seed)
def run_set(path, args, dtype=None, base_stage=None):
"""Render every prompt/seed. Returns ({key: latent}, vae, timing_dict)."""
model, clip, vae = load(path, dtype)
q, t = count_quant(model)
print(f"[load] {os.path.basename(path)} dtype={dtype} quantized_layers={q}/{t}", flush=True)
out = {}
times = []
t0 = time.time()
for pi, prompt in enumerate(PROMPTS):
if args.mode == "refiner":
pos = encode_refiner(clip, prompt, 6.0, args.size, args.size)
neg = encode_refiner(clip, "", 2.5, args.size, args.size)
else:
pos, neg = encode(clip, prompt), encode(clip, "")
for seed in SEEDS[: args.seeds]:
key = f"p{pi}_s{seed}"
ts = time.perf_counter()
if args.mode == "base":
lat = ksample(model, pos, neg, empty_latent(model, args.size), seed, args.steps, 6.0, "normal")
elif args.mode == "lightning":
lat = ksample(model, pos, neg, empty_latent(model, args.size), seed, 4, 1.0, "sgm_uniform")
elif args.mode == "turbo":
lat = turbo_sample(model, pos, neg, empty_latent(model, args.size), seed, args.steps)
else: # refiner stage on top of a fixed FP16 base stage
split = args.steps * 4 // 5
lat = ksample(model, pos, neg, base_stage[key], seed, args.steps, 6.0, "normal",
start=split, last=args.steps, disable_noise=True, force_full=True)
torch.cuda.synchronize() if torch.cuda.is_available() else None
times.append(time.perf_counter() - ts)
out[key] = lat.cpu()
total_s = time.time() - t0
warm = times[1:] if len(times) > 1 else times
warm_mean = float(np.mean(warm)) if warm else 0.0
print(f"[time] {len(out)} renders in {total_s:.1f}s (warm_mean={warm_mean:.2f}s/img)", flush=True)
del model, clip
return out, vae, {"quantized_layers": q, "total_layers": t, "total_s": total_s, "warm_mean_s": warm_mean}
def base_stage_latents(base_path, args):
model, clip, _ = load(base_path)
split = args.steps * 4 // 5
out = {}
for pi, prompt in enumerate(PROMPTS):
pos, neg = encode(clip, prompt), encode(clip, "")
for seed in SEEDS[: args.seeds]:
out[f"p{pi}_s{seed}"] = ksample(model, pos, neg, empty_latent(model, args.size), seed,
args.steps, 6.0, "normal", start=0, last=split).cpu()
del model, clip
free()
return out
def free():
mm.unload_all_models()
gc.collect()
mm.soft_empty_cache()
def decode(vae, lat):
with torch.inference_mode():
img = vae.decode(lat)
return img[0].detach().float().clamp(0, 1).cpu().numpy()
def cosine(a, b):
a, b = a.flatten().double(), b.flatten().double()
return float(torch.nn.functional.cosine_similarity(a, b, dim=0))
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--mode", required=True, choices=["base", "lightning", "turbo", "refiner"])
ap.add_argument("--fp16", required=True)
ap.add_argument("--quant", required=True)
ap.add_argument("--comfy-path", default=DEFAULT_COMFY)
ap.add_argument("--base-fp16", default=None, help="SDXL Base, required for --mode refiner")
ap.add_argument("--out", required=True)
ap.add_argument("--cache-ref", default=None, help="Cache FP16/BF16 reference latents here across runs")
ap.add_argument("--steps", type=int, default=None)
ap.add_argument("--size", type=int, default=1024)
ap.add_argument("--seeds", type=int, default=3)
ap.add_argument("--no-bf16", action="store_true")
args = ap.parse_args()
_bootstrap_comfy(args.comfy_path)
args.steps = args.steps or {"base": 50, "lightning": 4, "turbo": 2, "refiner": 40}[args.mode]
os.makedirs(args.out, exist_ok=True)
cache_file = os.path.join(args.cache_ref, "ref_cache.pt") if args.cache_ref else None
if cache_file and os.path.isfile(cache_file):
cached = torch.load(cache_file, map_location="cpu", weights_only=False)
base_stage = cached.get("base_stage")
ref, t_ref = cached["ref"], cached["t_ref"]
bf16, t_bf16 = cached.get("bf16"), cached.get("t_bf16")
_, _, vae = load(args.fp16)
free()
else:
base_stage = base_stage_latents(args.base_fp16, args) if args.mode == "refiner" else None
ref, vae, t_ref = run_set(args.fp16, args, base_stage=base_stage)
free()
bf16, t_bf16 = None, None
if not args.no_bf16:
bf16, _, t_bf16 = run_set(args.fp16, args, dtype=torch.bfloat16, base_stage=base_stage)
free()
if cache_file:
os.makedirs(args.cache_ref, exist_ok=True)
torch.save({"base_stage": base_stage, "ref": ref, "t_ref": t_ref, "bf16": bf16, "t_bf16": t_bf16}, cache_file)
quant, _, t_quant = run_set(args.quant, args, base_stage=base_stage)
free()
rows = []
for key in ref:
a = decode(vae, ref[key])
b = decode(vae, quant[key])
row = {"key": key,
"q_cos": float(cosine(ref[key], quant[key])),
"q_ssim": float(structural_similarity(a, b, channel_axis=2, data_range=1.0)),
"q_psnr": float(peak_signal_noise_ratio(a, b, data_range=1.0))}
panels = [a, b]
if bf16 is not None:
c = decode(vae, bf16[key])
row.update(b_cos=float(cosine(ref[key], bf16[key])),
b_ssim=float(structural_similarity(a, c, channel_axis=2, data_range=1.0)),
b_psnr=float(peak_signal_noise_ratio(a, c, data_range=1.0)))
panels.append(c)
rows.append(row)
grid = np.concatenate(panels, axis=1)
Image.fromarray((grid * 255).round().astype(np.uint8)).save(os.path.join(args.out, f"{key}.png"))
print(json.dumps({k: (round(v, 5) if isinstance(v, float) else v) for k, v in row.items()}), flush=True)
def stats(name):
vals = [r[name] for r in rows if name in r]
return {"mean": float(np.mean(vals)), "min": float(np.min(vals))} if vals else None
summary = {k: stats(k) for k in ("q_cos", "q_ssim", "q_psnr", "b_cos", "b_ssim", "b_psnr")}
summary.update(mode=args.mode, fp16=args.fp16, quant=args.quant, steps=args.steps,
size=args.size, renders=len(rows),
fp16_bytes=os.path.getsize(args.fp16), quant_bytes=os.path.getsize(args.quant),
timing={"fp16": t_ref, "bf16": t_bf16, "quant": t_quant})
with open(os.path.join(args.out, "summary.json"), "w") as f:
json.dump({"summary": summary, "rows": rows}, f, indent=1)
print("SUMMARY " + json.dumps(summary), flush=True)
if __name__ == "__main__":
main()