File size: 8,378 Bytes
e38ae43
 
 
 
 
 
 
 
 
 
 
 
622badc
 
e38ae43
 
 
 
 
 
 
 
 
 
622badc
 
e38ae43
 
 
 
 
 
 
 
 
 
 
 
 
622badc
 
 
 
 
 
e38ae43
 
 
 
 
 
622badc
e38ae43
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d0e6d0d
e38ae43
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d0e6d0d
e38ae43
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d0e6d0d
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
"""Sample-Aligned Decoders (SAD) β€” side-by-side demo on PixArt-Ξ± 256px.

Generates a latent ONCE with the frozen PixArt-Ξ± base model, then decodes that
SAME latent with (a) the original SD-VAE decoder D0 and (b) the SAD decoder
fine-tuned for this generator. Content and layout stay identical β€” only color,
exposure and texture change, which is exactly the paper's claim.

The sampler settings are fixed to the authors' recipe (20 DPM-Solver++ steps,
guidance 4.5) because each SAD decoder is trained on latents from exactly that
configuration (see training/README.md in the SAD repo).
"""

import os

import time

import spaces  # must come before torch / diffusers / transformers
import gradio as gr
import numpy as np
import torch
from diffusers import AutoencoderKL, PixArtAlphaPipeline, Transformer2DModel

from sad import load_sad_decoder

HERE = os.path.dirname(os.path.abspath(__file__))

# ---------------------------------------------------------------------------
# Model loading (module scope, eager) β€” exactly the authors' recipe from
# training/gen_latents_pixart.py:
#   transformer from PixArt-alpha/PixArt-XL-2-256x256 (256px)
#   all other components from PixArt-alpha/PixArt-XL-2-512x512 (variant=fp16)
# The SAD decoder is architecturally identical, so it drops straight into the
# same AutoencoderKL.
# ---------------------------------------------------------------------------

TRANSFORMER_REPO = "PixArt-alpha/PixArt-XL-2-256x256"
PIPELINE_REPO = "PixArt-alpha/PixArt-XL-2-512x512"
SAD_CKPT = "pixart-alpha-256/sad-gan"

from transformers import T5Tokenizer

# Pre-converted fast tokenizer shipped with the Space (tokenizer.json) β€” avoids
# the sentencepiece/protobuf slow-tokenizer conversion entirely.
_tokenizer = T5Tokenizer.from_pretrained(os.path.join(HERE, "tokenizer"))

_transformer = Transformer2DModel.from_pretrained(
    TRANSFORMER_REPO, subfolder="transformer", torch_dtype=torch.float16
)
pipe = PixArtAlphaPipeline.from_pretrained(
    PIPELINE_REPO,
    transformer=_transformer,
    tokenizer=_tokenizer,
    variant="fp16",
    torch_dtype=torch.float16,
)
pipe.to("cuda")
pipe.set_progress_bar_config(disable=True)

# Two independent VAEs, decoded in fp32 like the authors' evaluation code:
#   vae_d0  β€” the original decoder D0
#   vae_sad β€” the same VAE with the SAD decoder swapped in (decoder.* only)
vae_d0 = AutoencoderKL.from_pretrained(PIPELINE_REPO, subfolder="vae").to("cuda").eval()
vae_sad = AutoencoderKL.from_pretrained(PIPELINE_REPO, subfolder="vae").to("cuda").eval()
load_sad_decoder(vae_sad, SAD_CKPT)

# Authors' fixed sampler recipe for this decoder.
STEPS = 20
GUIDANCE = 4.5


def _to_uint8(x: torch.Tensor) -> np.ndarray:
    """[-1,1] float [1,3,H,W] -> uint8 numpy [H,W,3] (authors' to_uint8)."""
    x = x.clamp(-1, 1)
    return ((x + 1) * 127.5).round().to(torch.uint8)[0].permute(1, 2, 0).cpu().numpy()


def heat(d: np.ndarray) -> np.ndarray:
    """|SAD - D0| heatmap with the paper's colormap (from SAD demo.py)."""
    anchors = np.array(
        [(0, 0, 4), (40, 11, 84), (101, 21, 110), (159, 42, 99), (212, 72, 66),
         (245, 125, 21), (250, 193, 39), (252, 255, 164)], np.float32
    )
    x = np.clip(d / np.percentile(d, 99.5), 0, 1)
    t = np.linspace(0, 1, len(anchors))
    return np.stack([np.interp(x, t, anchors[:, c]) for c in range(3)], -1).astype(np.uint8)


@spaces.GPU(duration=10)
def generate(prompt: str, seed: int = 0, show_diff: bool = True,
             progress=gr.Progress(track_tqdm=True)):
    """Generate one latent with PixArt-alpha and decode it with both decoders.

    The sampler is fixed to the authors' recipe (20 DPM-Solver++ steps,
    guidance 4.5) that the SAD decoder was trained for.

    Args:
        prompt: text prompt for PixArt-alpha.
        seed: random seed (the same latent is decoded by both decoders).
        show_diff: also render the |SAD - D0| heatmap.

    Returns:
        (original decoder image, SAD decoder image, optional heatmap, timing text)
    """
    t0 = time.perf_counter()
    gen = torch.Generator(device="cuda").manual_seed(int(seed))

    # 1) sample ONE latent with the frozen base generator (authors' exact call)
    latents = pipe(
        [prompt],
        height=256,
        width=256,
        num_inference_steps=STEPS,
        guidance_scale=GUIDANCE,
        generator=gen,
        output_type="latent",
        use_resolution_binning=False,
    ).images

    # 2) raw VAE space: undo the scaling factor
    raw = latents.float() / pipe.vae.config.scaling_factor

    # 3) decode the SAME latent with both decoders
    with torch.no_grad():
        x_d0 = vae_d0.decode(raw).sample.clamp(-1, 1)
        x_sad = vae_sad.decode(raw).sample.clamp(-1, 1)

    im_d0 = _to_uint8(x_d0)
    im_sad = _to_uint8(x_sad)
    diff_img = None
    if show_diff:
        d = np.abs(im_sad.astype(np.float32) - im_d0.astype(np.float32)).mean(-1)
        diff_img = heat(d)

    dt = time.perf_counter() - t0
    info = (
        f"1 latent ({STEPS} steps, cfg {GUIDANCE}) decoded twice in **{dt:.1f}s** "
        f"β€” same seed β‡’ same latent, so every difference you see is the decoder."
    )
    return im_d0, im_sad, diff_img, info


# ---------------------------------------------------------------------------
# UI
# ---------------------------------------------------------------------------

CSS = """
.gradio-container {max-width: 1100px !important; margin: 0 auto;}
.dark .gradio-container {background: var(--background-fill-primary);}
"""

with gr.Blocks() as demo:
    gr.Markdown(
        """
        # Sample-Aligned Decoders (SAD)

        **A decoder post-training framework for latent generative models** β€” fine-tune *only* the VAE
        decoder on the generator's own latents (~1 GPU-hour), keep the generator frozen. The decoder
        keeps its architecture, so it is a **drop-in swap with zero extra inference cost**.

        This demo runs **PixArt-Ξ± 256px**: one prompt β†’ one latent β†’ decoded twice, with the original
        decoder **Dβ‚€** and with the SAD decoder. The latent is identical, so any difference you see is
        purely the decoder correcting color cast, exposure and texture β€” content and layout stay put.
        Sampler fixed to the authors' recipe (20 DPM-Solver++ steps, cfg 4.5) that this SAD decoder
        was trained for.

        πŸ“„ [Paper](https://huggingface.co/papers/2611.22458) Β· πŸ’» [Code](https://github.com/torpedo2003/SAD) Β· πŸ€— [Decoders](https://huggingface.co/torpedo2003/SAD)
        """.strip()
    )
    with gr.Row():
        with gr.Column():
            prompt = gr.Textbox(
                label="Prompt",
                placeholder="A small cactus with a happy face in the Sahara desert.",
                lines=2,
            )
            seed = gr.Slider(0, 2**31 - 1, value=0, step=1, label="Seed")
            show_diff = gr.Checkbox(value=True, label="Show |SAD βˆ’ Dβ‚€| heatmap")
            btn = gr.Button("Generate & compare", variant="primary")
        with gr.Column():
            out_d0 = gr.Image(label="Original decoder Dβ‚€ (PixArt-Ξ±)", height=290)
            out_sad = gr.Image(label="SAD decoder (pixart-alpha-256/sad-gan)", height=290)
    diff_gallery = gr.Image(label="|SAD βˆ’ Dβ‚€| heatmap (paper colormap)", height=290)
    info = gr.Markdown()

    btn.click(
        fn=generate,
        inputs=[prompt, seed, show_diff],
        outputs=[out_d0, out_sad, diff_gallery, info],
        api_name="generate",
    )

    gr.Examples(
        examples=[
            ["A small cactus with a happy face in the Sahara desert.", 0],
            ["A 4k dslr image of a lemur wearing a red magician hat and a blue coat performing magic tricks with cards in a garden.", 3],
            ["Pirate ship trapped in a cosmic maelstrom nebula, rendered in cosmic beach whirlpool engine, volumetric lighting, spectacular, ambient lights, light pollution, cinematic atmosphere, art nouveau style, illustration art artwork by SenseiJaye, intricate detail.", 7],
            ["nature vs human nature, surreal, UHD, 8k, hyper details, rich colors, photograph.", 11],
        ],
        inputs=[prompt, seed],
        fn=generate,
        outputs=[out_d0, out_sad, diff_gallery, info],
        cache_examples=True,
        cache_mode="lazy",
    )

demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)