"""Evaluate GLM-Image ConvRot INT8 / INT6 / INT4 checkpoints against BF16 baseline (end-to-end + DiT-isolated).""" import argparse import gc import json import os import time import numpy as np import torch from PIL import Image from skimage.metrics import peak_signal_noise_ratio, structural_similarity from glm_image_convrot import load_glm_image_convrot BASE_MODEL = "/content/models/GLM-Image" BF16_DIR = "/content/work/glm_bf16" EVAL_CASES = [ ( "t2i_cabin_s42", "A cozy wooden cabin beside a calm alpine lake at golden hour, warm glowing windows, pine trees reflected in crystal clear water, photorealistic, sharp detail", 1024, 1024, 42, ), ( "t2i_poster_s1337", "A vintage travel poster for 'ALPS EXPRESS' with bold cream typography at the top, a red steam train crossing a stone viaduct over snowy mountains, and small text 'SWITZERLAND 1964' at the bottom, crisp retro print illustration", 1024, 1024, 1337, ), ( "t2i_watch_s777", "Macro studio photograph of a luxury mechanical wristwatch on dark slate stone, brushed steel case, sapphire crystal catching soft rim light, crisp numerals and gears", 1024, 1024, 777, ), ] def img_metrics(ref_img: Image.Image, cand_img: Image.Image) -> dict: a = np.asarray(ref_img.convert("RGB"), dtype=np.float32) / 255.0 b = np.asarray(cand_img.convert("RGB"), dtype=np.float32) / 255.0 cos = float(np.dot(a.ravel(), b.ravel()) / (np.linalg.norm(a.ravel()) * np.linalg.norm(b.ravel()) + 1e-12)) ssim = float(structural_similarity(a, b, channel_axis=2, data_range=1.0)) psnr = float(peak_signal_noise_ratio(a, b, data_range=1.0)) return {"cos": round(cos, 5), "ssim": round(ssim, 5), "psnr": round(psnr, 3)} def main(): ap = argparse.ArgumentParser() ap.add_argument("--transformer", default=None) ap.add_argument("--vlm", default=None) ap.add_argument("--out", required=True) ap.add_argument("--skip-dit-isolated", action="store_true") args = ap.parse_args() os.makedirs(args.out, exist_ok=True) torch.cuda.reset_peak_memory_stats() pipe, load_info = load_glm_image_convrot( BASE_MODEL, transformer_path=args.transformer, vlm_path=args.vlm, device="cuda", dtype=torch.bfloat16, ) alloc_gb = torch.cuda.memory_allocated() / 1e9 print(f"[EVAL] Loaded in {load_info['load_s']:.2f}s | allocated={alloc_gb:.2f} GB | {load_info}", flush=True) orig_gen = pipe.generate_prior_tokens timings = {} forced_prior = [None] def hooked_gen(*g_args, **g_kwargs): if forced_prior[0] is not None: return forced_prior[0] torch.cuda.synchronize() ts = time.perf_counter() out = orig_gen(*g_args, **g_kwargs) torch.cuda.synchronize() timings["ar_s"] = time.perf_counter() - ts timings["prior_out"] = [t.detach().cpu() if torch.is_tensor(t) else t for t in out] if isinstance(out, (list, tuple)) else out.detach().cpu() return out pipe.generate_prior_tokens = hooked_gen stats = { "load_s": load_info["load_s"], "allocated_vram_gb": round(alloc_gb, 3), "transformer_bytes": os.path.getsize(args.transformer) if args.transformer else None, "vlm_bytes": os.path.getsize(args.vlm) if args.vlm else None, "load_info": load_info, "cases": {}, } for tag, prompt, h, w, seed in EVAL_CASES: ref_img = Image.open(os.path.join(BF16_DIR, f"{tag}.png")) ref_prior = torch.load(os.path.join(BF16_DIR, f"{tag}_prior.pt"), map_location="cpu", weights_only=False) # 1. End-to-end (Quant AR + Quant DiT) forced_prior[0] = None timings.clear() torch.cuda.reset_peak_memory_stats() gen = torch.Generator(device="cuda").manual_seed(seed) t0 = time.perf_counter() res = pipe(prompt=prompt, height=h, width=w, num_inference_steps=50, guidance_scale=1.5, generator=gen) torch.cuda.synchronize() total_s = time.perf_counter() - t0 peak_gb = torch.cuda.max_memory_allocated() / 1e9 img = res.images[0] img.save(os.path.join(args.out, f"{tag}.png")) m_e2e = img_metrics(ref_img, img) # Compare AR prior tokens vs BF16 prior tokens cand_prior = timings.get("prior_out") ar_exact = None if isinstance(ref_prior, (list, tuple)) and isinstance(cand_prior, (list, tuple)) and torch.is_tensor(ref_prior[0]) and torch.is_tensor(cand_prior[0]): rp, cp_t = ref_prior[0].view(-1), cand_prior[0].view(-1) n_min = min(rp.numel(), cp_t.numel()) ar_exact = round(float((rp[:n_min] == cp_t[:n_min]).float().mean()), 5) ar_s = timings.get("ar_s", 0.0) dit_s = max(0.0, total_s - ar_s) case_stat = { "total_s": round(total_s, 3), "ar_s": round(ar_s, 3), "dit_s": round(dit_s, 3), "peak_vram_gb": round(peak_gb, 3), "ar_token_match": ar_exact, "e2e": m_e2e, } # 2. DiT-isolated (exact BF16 prior tokens -> Quant DiT) if not args.skip_dit_isolated and isinstance(ref_prior, (list, tuple)): forced_prior[0] = tuple(t.to("cuda") if torch.is_tensor(t) else t for t in ref_prior) gen = torch.Generator(device="cuda").manual_seed(seed) res_dit = pipe(prompt=prompt, height=h, width=w, num_inference_steps=50, guidance_scale=1.5, generator=gen) torch.cuda.synchronize() dit_img = res_dit.images[0] dit_img.save(os.path.join(args.out, f"{tag}_dit_only.png")) case_stat["dit_only"] = img_metrics(ref_img, dit_img) forced_prior[0] = None stats["cases"][tag] = case_stat print( f" [{tag}] total={total_s:.2f}s (AR={ar_s:.2f}s, DiT={dit_s:.2f}s) vram={peak_gb:.2f}GB | " f"AR_match={ar_exact} | E2E: cos={m_e2e['cos']} ssim={m_e2e['ssim']} psnr={m_e2e['psnr']}dB | " f"DiT_only: {case_stat.get('dit_only')}", flush=True, ) with open(os.path.join(args.out, "stats.json"), "w") as f: json.dump(stats, f, indent=2) print("EVAL_GLM_DONE", flush=True) if __name__ == "__main__": main()