File size: 6,381 Bytes
9ca4e71
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
"""Evaluate GLM-Image ConvRot INT8 / INT6 / INT4 checkpoints against BF16 baseline (end-to-end + DiT-isolated)."""
import argparse
import gc
import json
import os
import time

import numpy as np
import torch
from PIL import Image
from skimage.metrics import peak_signal_noise_ratio, structural_similarity

from glm_image_convrot import load_glm_image_convrot

BASE_MODEL = "/content/models/GLM-Image"
BF16_DIR = "/content/work/glm_bf16"

EVAL_CASES = [
    (
        "t2i_cabin_s42",
        "A cozy wooden cabin beside a calm alpine lake at golden hour, warm glowing windows, pine trees reflected in crystal clear water, photorealistic, sharp detail",
        1024,
        1024,
        42,
    ),
    (
        "t2i_poster_s1337",
        "A vintage travel poster for 'ALPS EXPRESS' with bold cream typography at the top, a red steam train crossing a stone viaduct over snowy mountains, and small text 'SWITZERLAND 1964' at the bottom, crisp retro print illustration",
        1024,
        1024,
        1337,
    ),
    (
        "t2i_watch_s777",
        "Macro studio photograph of a luxury mechanical wristwatch on dark slate stone, brushed steel case, sapphire crystal catching soft rim light, crisp numerals and gears",
        1024,
        1024,
        777,
    ),
]


def img_metrics(ref_img: Image.Image, cand_img: Image.Image) -> dict:
    a = np.asarray(ref_img.convert("RGB"), dtype=np.float32) / 255.0
    b = np.asarray(cand_img.convert("RGB"), dtype=np.float32) / 255.0
    cos = float(np.dot(a.ravel(), b.ravel()) / (np.linalg.norm(a.ravel()) * np.linalg.norm(b.ravel()) + 1e-12))
    ssim = float(structural_similarity(a, b, channel_axis=2, data_range=1.0))
    psnr = float(peak_signal_noise_ratio(a, b, data_range=1.0))
    return {"cos": round(cos, 5), "ssim": round(ssim, 5), "psnr": round(psnr, 3)}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--transformer", default=None)
    ap.add_argument("--vlm", default=None)
    ap.add_argument("--out", required=True)
    ap.add_argument("--skip-dit-isolated", action="store_true")
    args = ap.parse_args()

    os.makedirs(args.out, exist_ok=True)
    torch.cuda.reset_peak_memory_stats()
    pipe, load_info = load_glm_image_convrot(
        BASE_MODEL,
        transformer_path=args.transformer,
        vlm_path=args.vlm,
        device="cuda",
        dtype=torch.bfloat16,
    )
    alloc_gb = torch.cuda.memory_allocated() / 1e9
    print(f"[EVAL] Loaded in {load_info['load_s']:.2f}s | allocated={alloc_gb:.2f} GB | {load_info}", flush=True)

    orig_gen = pipe.generate_prior_tokens
    timings = {}
    forced_prior = [None]

    def hooked_gen(*g_args, **g_kwargs):
        if forced_prior[0] is not None:
            return forced_prior[0]
        torch.cuda.synchronize()
        ts = time.perf_counter()
        out = orig_gen(*g_args, **g_kwargs)
        torch.cuda.synchronize()
        timings["ar_s"] = time.perf_counter() - ts
        timings["prior_out"] = [t.detach().cpu() if torch.is_tensor(t) else t for t in out] if isinstance(out, (list, tuple)) else out.detach().cpu()
        return out

    pipe.generate_prior_tokens = hooked_gen

    stats = {
        "load_s": load_info["load_s"],
        "allocated_vram_gb": round(alloc_gb, 3),
        "transformer_bytes": os.path.getsize(args.transformer) if args.transformer else None,
        "vlm_bytes": os.path.getsize(args.vlm) if args.vlm else None,
        "load_info": load_info,
        "cases": {},
    }

    for tag, prompt, h, w, seed in EVAL_CASES:
        ref_img = Image.open(os.path.join(BF16_DIR, f"{tag}.png"))
        ref_prior = torch.load(os.path.join(BF16_DIR, f"{tag}_prior.pt"), map_location="cpu", weights_only=False)

        # 1. End-to-end (Quant AR + Quant DiT)
        forced_prior[0] = None
        timings.clear()
        torch.cuda.reset_peak_memory_stats()
        gen = torch.Generator(device="cuda").manual_seed(seed)
        t0 = time.perf_counter()
        res = pipe(prompt=prompt, height=h, width=w, num_inference_steps=50, guidance_scale=1.5, generator=gen)
        torch.cuda.synchronize()
        total_s = time.perf_counter() - t0
        peak_gb = torch.cuda.max_memory_allocated() / 1e9
        img = res.images[0]
        img.save(os.path.join(args.out, f"{tag}.png"))
        m_e2e = img_metrics(ref_img, img)

        # Compare AR prior tokens vs BF16 prior tokens
        cand_prior = timings.get("prior_out")
        ar_exact = None
        if isinstance(ref_prior, (list, tuple)) and isinstance(cand_prior, (list, tuple)) and torch.is_tensor(ref_prior[0]) and torch.is_tensor(cand_prior[0]):
            rp, cp_t = ref_prior[0].view(-1), cand_prior[0].view(-1)
            n_min = min(rp.numel(), cp_t.numel())
            ar_exact = round(float((rp[:n_min] == cp_t[:n_min]).float().mean()), 5)

        ar_s = timings.get("ar_s", 0.0)
        dit_s = max(0.0, total_s - ar_s)

        case_stat = {
            "total_s": round(total_s, 3),
            "ar_s": round(ar_s, 3),
            "dit_s": round(dit_s, 3),
            "peak_vram_gb": round(peak_gb, 3),
            "ar_token_match": ar_exact,
            "e2e": m_e2e,
        }

        # 2. DiT-isolated (exact BF16 prior tokens -> Quant DiT)
        if not args.skip_dit_isolated and isinstance(ref_prior, (list, tuple)):
            forced_prior[0] = tuple(t.to("cuda") if torch.is_tensor(t) else t for t in ref_prior)
            gen = torch.Generator(device="cuda").manual_seed(seed)
            res_dit = pipe(prompt=prompt, height=h, width=w, num_inference_steps=50, guidance_scale=1.5, generator=gen)
            torch.cuda.synchronize()
            dit_img = res_dit.images[0]
            dit_img.save(os.path.join(args.out, f"{tag}_dit_only.png"))
            case_stat["dit_only"] = img_metrics(ref_img, dit_img)
            forced_prior[0] = None

        stats["cases"][tag] = case_stat
        print(
            f"  [{tag}] total={total_s:.2f}s (AR={ar_s:.2f}s, DiT={dit_s:.2f}s) vram={peak_gb:.2f}GB | "
            f"AR_match={ar_exact} | E2E: cos={m_e2e['cos']} ssim={m_e2e['ssim']} psnr={m_e2e['psnr']}dB | "
            f"DiT_only: {case_stat.get('dit_only')}",
            flush=True,
        )

    with open(os.path.join(args.out, "stats.json"), "w") as f:
        json.dump(stats, f, indent=2)
    print("EVAL_GLM_DONE", flush=True)


if __name__ == "__main__":
    main()