"""Sample-Aligned Decoders (SAD) — side-by-side demo on PixArt-α 256px. Generates a latent ONCE with the frozen PixArt-α base model, then decodes that SAME latent with (a) the original SD-VAE decoder D0 and (b) the SAD decoder fine-tuned for this generator. Content and layout stay identical — only color, exposure and texture change, which is exactly the paper's claim. The sampler settings are fixed to the authors' recipe (20 DPM-Solver++ steps, guidance 4.5) because each SAD decoder is trained on latents from exactly that configuration (see training/README.md in the SAD repo). """ import os import time import spaces # must come before torch / diffusers / transformers import gradio as gr import numpy as np import torch from diffusers import AutoencoderKL, PixArtAlphaPipeline, Transformer2DModel from sad import load_sad_decoder HERE = os.path.dirname(os.path.abspath(__file__)) # --------------------------------------------------------------------------- # Model loading (module scope, eager) — exactly the authors' recipe from # training/gen_latents_pixart.py: # transformer from PixArt-alpha/PixArt-XL-2-256x256 (256px) # all other components from PixArt-alpha/PixArt-XL-2-512x512 (variant=fp16) # The SAD decoder is architecturally identical, so it drops straight into the # same AutoencoderKL. # --------------------------------------------------------------------------- TRANSFORMER_REPO = "PixArt-alpha/PixArt-XL-2-256x256" PIPELINE_REPO = "PixArt-alpha/PixArt-XL-2-512x512" SAD_CKPT = "pixart-alpha-256/sad-gan" from transformers import T5Tokenizer # Pre-converted fast tokenizer shipped with the Space (tokenizer.json) — avoids # the sentencepiece/protobuf slow-tokenizer conversion entirely. _tokenizer = T5Tokenizer.from_pretrained(os.path.join(HERE, "tokenizer")) _transformer = Transformer2DModel.from_pretrained( TRANSFORMER_REPO, subfolder="transformer", torch_dtype=torch.float16 ) pipe = PixArtAlphaPipeline.from_pretrained( PIPELINE_REPO, transformer=_transformer, tokenizer=_tokenizer, variant="fp16", torch_dtype=torch.float16, ) pipe.to("cuda") pipe.set_progress_bar_config(disable=True) # Two independent VAEs, decoded in fp32 like the authors' evaluation code: # vae_d0 — the original decoder D0 # vae_sad — the same VAE with the SAD decoder swapped in (decoder.* only) vae_d0 = AutoencoderKL.from_pretrained(PIPELINE_REPO, subfolder="vae").to("cuda").eval() vae_sad = AutoencoderKL.from_pretrained(PIPELINE_REPO, subfolder="vae").to("cuda").eval() load_sad_decoder(vae_sad, SAD_CKPT) # Authors' fixed sampler recipe for this decoder. STEPS = 20 GUIDANCE = 4.5 def _to_uint8(x: torch.Tensor) -> np.ndarray: """[-1,1] float [1,3,H,W] -> uint8 numpy [H,W,3] (authors' to_uint8).""" x = x.clamp(-1, 1) return ((x + 1) * 127.5).round().to(torch.uint8)[0].permute(1, 2, 0).cpu().numpy() def heat(d: np.ndarray) -> np.ndarray: """|SAD - D0| heatmap with the paper's colormap (from SAD demo.py).""" anchors = np.array( [(0, 0, 4), (40, 11, 84), (101, 21, 110), (159, 42, 99), (212, 72, 66), (245, 125, 21), (250, 193, 39), (252, 255, 164)], np.float32 ) x = np.clip(d / np.percentile(d, 99.5), 0, 1) t = np.linspace(0, 1, len(anchors)) return np.stack([np.interp(x, t, anchors[:, c]) for c in range(3)], -1).astype(np.uint8) @spaces.GPU(duration=10) def generate(prompt: str, seed: int = 0, show_diff: bool = True, progress=gr.Progress(track_tqdm=True)): """Generate one latent with PixArt-alpha and decode it with both decoders. The sampler is fixed to the authors' recipe (20 DPM-Solver++ steps, guidance 4.5) that the SAD decoder was trained for. Args: prompt: text prompt for PixArt-alpha. seed: random seed (the same latent is decoded by both decoders). show_diff: also render the |SAD - D0| heatmap. Returns: (original decoder image, SAD decoder image, optional heatmap, timing text) """ t0 = time.perf_counter() gen = torch.Generator(device="cuda").manual_seed(int(seed)) # 1) sample ONE latent with the frozen base generator (authors' exact call) latents = pipe( [prompt], height=256, width=256, num_inference_steps=STEPS, guidance_scale=GUIDANCE, generator=gen, output_type="latent", use_resolution_binning=False, ).images # 2) raw VAE space: undo the scaling factor raw = latents.float() / pipe.vae.config.scaling_factor # 3) decode the SAME latent with both decoders with torch.no_grad(): x_d0 = vae_d0.decode(raw).sample.clamp(-1, 1) x_sad = vae_sad.decode(raw).sample.clamp(-1, 1) im_d0 = _to_uint8(x_d0) im_sad = _to_uint8(x_sad) diff_img = None if show_diff: d = np.abs(im_sad.astype(np.float32) - im_d0.astype(np.float32)).mean(-1) diff_img = heat(d) dt = time.perf_counter() - t0 info = ( f"1 latent ({STEPS} steps, cfg {GUIDANCE}) decoded twice in **{dt:.1f}s** " f"— same seed ⇒ same latent, so every difference you see is the decoder." ) return im_d0, im_sad, diff_img, info # --------------------------------------------------------------------------- # UI # --------------------------------------------------------------------------- CSS = """ .gradio-container {max-width: 1100px !important; margin: 0 auto;} .dark .gradio-container {background: var(--background-fill-primary);} """ with gr.Blocks() as demo: gr.Markdown( """ # Sample-Aligned Decoders (SAD) **A decoder post-training framework for latent generative models** — fine-tune *only* the VAE decoder on the generator's own latents (~1 GPU-hour), keep the generator frozen. The decoder keeps its architecture, so it is a **drop-in swap with zero extra inference cost**. This demo runs **PixArt-α 256px**: one prompt → one latent → decoded twice, with the original decoder **D₀** and with the SAD decoder. The latent is identical, so any difference you see is purely the decoder correcting color cast, exposure and texture — content and layout stay put. Sampler fixed to the authors' recipe (20 DPM-Solver++ steps, cfg 4.5) that this SAD decoder was trained for. 📄 [Paper](https://huggingface.co/papers/2611.22458) · 💻 [Code](https://github.com/torpedo2003/SAD) · 🤗 [Decoders](https://huggingface.co/torpedo2003/SAD) """.strip() ) with gr.Row(): with gr.Column(): prompt = gr.Textbox( label="Prompt", placeholder="A small cactus with a happy face in the Sahara desert.", lines=2, ) seed = gr.Slider(0, 2**31 - 1, value=0, step=1, label="Seed") show_diff = gr.Checkbox(value=True, label="Show |SAD − D₀| heatmap") btn = gr.Button("Generate & compare", variant="primary") with gr.Column(): out_d0 = gr.Image(label="Original decoder D₀ (PixArt-α)", height=290) out_sad = gr.Image(label="SAD decoder (pixart-alpha-256/sad-gan)", height=290) diff_gallery = gr.Image(label="|SAD − D₀| heatmap (paper colormap)", height=290) info = gr.Markdown() btn.click( fn=generate, inputs=[prompt, seed, show_diff], outputs=[out_d0, out_sad, diff_gallery, info], api_name="generate", ) gr.Examples( examples=[ ["A small cactus with a happy face in the Sahara desert.", 0], ["A 4k dslr image of a lemur wearing a red magician hat and a blue coat performing magic tricks with cards in a garden.", 3], ["Pirate ship trapped in a cosmic maelstrom nebula, rendered in cosmic beach whirlpool engine, volumetric lighting, spectacular, ambient lights, light pollution, cinematic atmosphere, art nouveau style, illustration art artwork by SenseiJaye, intricate detail.", 7], ["nature vs human nature, surreal, UHD, 8k, hyper details, rich colors, photograph.", 11], ], inputs=[prompt, seed], fn=generate, outputs=[out_d0, out_sad, diff_gallery, info], cache_examples=True, cache_mode="lazy", ) demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)