GLM-Image-ConvRot / scripts /eval_glm_image.py
SUSSYEATING's picture
Release GLM-Image ConvRot INT8, INT6 & INT4 checkpoints, metrics, and showcases
9ca4e71
Raw History Blame Contribute Delete
6.38 kB
"""Evaluate GLM-Image ConvRot INT8 / INT6 / INT4 checkpoints against BF16 baseline (end-to-end + DiT-isolated)."""
import argparse
import gc
import json
import os
import time
import numpy as np
import torch
from PIL import Image
from skimage.metrics import peak_signal_noise_ratio, structural_similarity
from glm_image_convrot import load_glm_image_convrot
BASE_MODEL = "/content/models/GLM-Image"
BF16_DIR = "/content/work/glm_bf16"
EVAL_CASES = [
(
"t2i_cabin_s42",
"A cozy wooden cabin beside a calm alpine lake at golden hour, warm glowing windows, pine trees reflected in crystal clear water, photorealistic, sharp detail",
1024,
1024,
42,
),
(
"t2i_poster_s1337",
"A vintage travel poster for 'ALPS EXPRESS' with bold cream typography at the top, a red steam train crossing a stone viaduct over snowy mountains, and small text 'SWITZERLAND 1964' at the bottom, crisp retro print illustration",
1024,
1024,
1337,
),
(
"t2i_watch_s777",
"Macro studio photograph of a luxury mechanical wristwatch on dark slate stone, brushed steel case, sapphire crystal catching soft rim light, crisp numerals and gears",
1024,
1024,
777,
),
]
def img_metrics(ref_img: Image.Image, cand_img: Image.Image) -> dict:
a = np.asarray(ref_img.convert("RGB"), dtype=np.float32) / 255.0
b = np.asarray(cand_img.convert("RGB"), dtype=np.float32) / 255.0
cos = float(np.dot(a.ravel(), b.ravel()) / (np.linalg.norm(a.ravel()) * np.linalg.norm(b.ravel()) + 1e-12))
ssim = float(structural_similarity(a, b, channel_axis=2, data_range=1.0))
psnr = float(peak_signal_noise_ratio(a, b, data_range=1.0))
return {"cos": round(cos, 5), "ssim": round(ssim, 5), "psnr": round(psnr, 3)}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--transformer", default=None)
ap.add_argument("--vlm", default=None)
ap.add_argument("--out", required=True)
ap.add_argument("--skip-dit-isolated", action="store_true")
args = ap.parse_args()
os.makedirs(args.out, exist_ok=True)
torch.cuda.reset_peak_memory_stats()
pipe, load_info = load_glm_image_convrot(
BASE_MODEL,
transformer_path=args.transformer,
vlm_path=args.vlm,
device="cuda",
dtype=torch.bfloat16,
)
alloc_gb = torch.cuda.memory_allocated() / 1e9
print(f"[EVAL] Loaded in {load_info['load_s']:.2f}s | allocated={alloc_gb:.2f} GB | {load_info}", flush=True)
orig_gen = pipe.generate_prior_tokens
timings = {}
forced_prior = [None]
def hooked_gen(*g_args, **g_kwargs):
if forced_prior[0] is not None:
return forced_prior[0]
torch.cuda.synchronize()
ts = time.perf_counter()
out = orig_gen(*g_args, **g_kwargs)
torch.cuda.synchronize()
timings["ar_s"] = time.perf_counter() - ts
timings["prior_out"] = [t.detach().cpu() if torch.is_tensor(t) else t for t in out] if isinstance(out, (list, tuple)) else out.detach().cpu()
return out
pipe.generate_prior_tokens = hooked_gen
stats = {
"load_s": load_info["load_s"],
"allocated_vram_gb": round(alloc_gb, 3),
"transformer_bytes": os.path.getsize(args.transformer) if args.transformer else None,
"vlm_bytes": os.path.getsize(args.vlm) if args.vlm else None,
"load_info": load_info,
"cases": {},
}
for tag, prompt, h, w, seed in EVAL_CASES:
ref_img = Image.open(os.path.join(BF16_DIR, f"{tag}.png"))
ref_prior = torch.load(os.path.join(BF16_DIR, f"{tag}_prior.pt"), map_location="cpu", weights_only=False)
# 1. End-to-end (Quant AR + Quant DiT)
forced_prior[0] = None
timings.clear()
torch.cuda.reset_peak_memory_stats()
gen = torch.Generator(device="cuda").manual_seed(seed)
t0 = time.perf_counter()
res = pipe(prompt=prompt, height=h, width=w, num_inference_steps=50, guidance_scale=1.5, generator=gen)
torch.cuda.synchronize()
total_s = time.perf_counter() - t0
peak_gb = torch.cuda.max_memory_allocated() / 1e9
img = res.images[0]
img.save(os.path.join(args.out, f"{tag}.png"))
m_e2e = img_metrics(ref_img, img)
# Compare AR prior tokens vs BF16 prior tokens
cand_prior = timings.get("prior_out")
ar_exact = None
if isinstance(ref_prior, (list, tuple)) and isinstance(cand_prior, (list, tuple)) and torch.is_tensor(ref_prior[0]) and torch.is_tensor(cand_prior[0]):
rp, cp_t = ref_prior[0].view(-1), cand_prior[0].view(-1)
n_min = min(rp.numel(), cp_t.numel())
ar_exact = round(float((rp[:n_min] == cp_t[:n_min]).float().mean()), 5)
ar_s = timings.get("ar_s", 0.0)
dit_s = max(0.0, total_s - ar_s)
case_stat = {
"total_s": round(total_s, 3),
"ar_s": round(ar_s, 3),
"dit_s": round(dit_s, 3),
"peak_vram_gb": round(peak_gb, 3),
"ar_token_match": ar_exact,
"e2e": m_e2e,
}
# 2. DiT-isolated (exact BF16 prior tokens -> Quant DiT)
if not args.skip_dit_isolated and isinstance(ref_prior, (list, tuple)):
forced_prior[0] = tuple(t.to("cuda") if torch.is_tensor(t) else t for t in ref_prior)
gen = torch.Generator(device="cuda").manual_seed(seed)
res_dit = pipe(prompt=prompt, height=h, width=w, num_inference_steps=50, guidance_scale=1.5, generator=gen)
torch.cuda.synchronize()
dit_img = res_dit.images[0]
dit_img.save(os.path.join(args.out, f"{tag}_dit_only.png"))
case_stat["dit_only"] = img_metrics(ref_img, dit_img)
forced_prior[0] = None
stats["cases"][tag] = case_stat
print(
f" [{tag}] total={total_s:.2f}s (AR={ar_s:.2f}s, DiT={dit_s:.2f}s) vram={peak_gb:.2f}GB | "
f"AR_match={ar_exact} | E2E: cos={m_e2e['cos']} ssim={m_e2e['ssim']} psnr={m_e2e['psnr']}dB | "
f"DiT_only: {case_stat.get('dit_only')}",
flush=True,
)
with open(os.path.join(args.out, "stats.json"), "w") as f:
json.dump(stats, f, indent=2)
print("EVAL_GLM_DONE", flush=True)
if __name__ == "__main__":
main()