multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
d0e6d0d verified
Raw History Blame Contribute Delete
8.38 kB
"""Sample-Aligned Decoders (SAD) β€” side-by-side demo on PixArt-Ξ± 256px.
Generates a latent ONCE with the frozen PixArt-Ξ± base model, then decodes that
SAME latent with (a) the original SD-VAE decoder D0 and (b) the SAD decoder
fine-tuned for this generator. Content and layout stay identical β€” only color,
exposure and texture change, which is exactly the paper's claim.
The sampler settings are fixed to the authors' recipe (20 DPM-Solver++ steps,
guidance 4.5) because each SAD decoder is trained on latents from exactly that
configuration (see training/README.md in the SAD repo).
"""
import os
import time
import spaces # must come before torch / diffusers / transformers
import gradio as gr
import numpy as np
import torch
from diffusers import AutoencoderKL, PixArtAlphaPipeline, Transformer2DModel
from sad import load_sad_decoder
HERE = os.path.dirname(os.path.abspath(__file__))
# ---------------------------------------------------------------------------
# Model loading (module scope, eager) β€” exactly the authors' recipe from
# training/gen_latents_pixart.py:
# transformer from PixArt-alpha/PixArt-XL-2-256x256 (256px)
# all other components from PixArt-alpha/PixArt-XL-2-512x512 (variant=fp16)
# The SAD decoder is architecturally identical, so it drops straight into the
# same AutoencoderKL.
# ---------------------------------------------------------------------------
TRANSFORMER_REPO = "PixArt-alpha/PixArt-XL-2-256x256"
PIPELINE_REPO = "PixArt-alpha/PixArt-XL-2-512x512"
SAD_CKPT = "pixart-alpha-256/sad-gan"
from transformers import T5Tokenizer
# Pre-converted fast tokenizer shipped with the Space (tokenizer.json) β€” avoids
# the sentencepiece/protobuf slow-tokenizer conversion entirely.
_tokenizer = T5Tokenizer.from_pretrained(os.path.join(HERE, "tokenizer"))
_transformer = Transformer2DModel.from_pretrained(
TRANSFORMER_REPO, subfolder="transformer", torch_dtype=torch.float16
)
pipe = PixArtAlphaPipeline.from_pretrained(
PIPELINE_REPO,
transformer=_transformer,
tokenizer=_tokenizer,
variant="fp16",
torch_dtype=torch.float16,
)
pipe.to("cuda")
pipe.set_progress_bar_config(disable=True)
# Two independent VAEs, decoded in fp32 like the authors' evaluation code:
# vae_d0 β€” the original decoder D0
# vae_sad β€” the same VAE with the SAD decoder swapped in (decoder.* only)
vae_d0 = AutoencoderKL.from_pretrained(PIPELINE_REPO, subfolder="vae").to("cuda").eval()
vae_sad = AutoencoderKL.from_pretrained(PIPELINE_REPO, subfolder="vae").to("cuda").eval()
load_sad_decoder(vae_sad, SAD_CKPT)
# Authors' fixed sampler recipe for this decoder.
STEPS = 20
GUIDANCE = 4.5
def _to_uint8(x: torch.Tensor) -> np.ndarray:
"""[-1,1] float [1,3,H,W] -> uint8 numpy [H,W,3] (authors' to_uint8)."""
x = x.clamp(-1, 1)
return ((x + 1) * 127.5).round().to(torch.uint8)[0].permute(1, 2, 0).cpu().numpy()
def heat(d: np.ndarray) -> np.ndarray:
"""|SAD - D0| heatmap with the paper's colormap (from SAD demo.py)."""
anchors = np.array(
[(0, 0, 4), (40, 11, 84), (101, 21, 110), (159, 42, 99), (212, 72, 66),
(245, 125, 21), (250, 193, 39), (252, 255, 164)], np.float32
)
x = np.clip(d / np.percentile(d, 99.5), 0, 1)
t = np.linspace(0, 1, len(anchors))
return np.stack([np.interp(x, t, anchors[:, c]) for c in range(3)], -1).astype(np.uint8)
@spaces.GPU(duration=10)
def generate(prompt: str, seed: int = 0, show_diff: bool = True,
progress=gr.Progress(track_tqdm=True)):
"""Generate one latent with PixArt-alpha and decode it with both decoders.
The sampler is fixed to the authors' recipe (20 DPM-Solver++ steps,
guidance 4.5) that the SAD decoder was trained for.
Args:
prompt: text prompt for PixArt-alpha.
seed: random seed (the same latent is decoded by both decoders).
show_diff: also render the |SAD - D0| heatmap.
Returns:
(original decoder image, SAD decoder image, optional heatmap, timing text)
"""
t0 = time.perf_counter()
gen = torch.Generator(device="cuda").manual_seed(int(seed))
# 1) sample ONE latent with the frozen base generator (authors' exact call)
latents = pipe(
[prompt],
height=256,
width=256,
num_inference_steps=STEPS,
guidance_scale=GUIDANCE,
generator=gen,
output_type="latent",
use_resolution_binning=False,
).images
# 2) raw VAE space: undo the scaling factor
raw = latents.float() / pipe.vae.config.scaling_factor
# 3) decode the SAME latent with both decoders
with torch.no_grad():
x_d0 = vae_d0.decode(raw).sample.clamp(-1, 1)
x_sad = vae_sad.decode(raw).sample.clamp(-1, 1)
im_d0 = _to_uint8(x_d0)
im_sad = _to_uint8(x_sad)
diff_img = None
if show_diff:
d = np.abs(im_sad.astype(np.float32) - im_d0.astype(np.float32)).mean(-1)
diff_img = heat(d)
dt = time.perf_counter() - t0
info = (
f"1 latent ({STEPS} steps, cfg {GUIDANCE}) decoded twice in **{dt:.1f}s** "
f"β€” same seed β‡’ same latent, so every difference you see is the decoder."
)
return im_d0, im_sad, diff_img, info
# ---------------------------------------------------------------------------
# UI
# ---------------------------------------------------------------------------
CSS = """
.gradio-container {max-width: 1100px !important; margin: 0 auto;}
.dark .gradio-container {background: var(--background-fill-primary);}
"""
with gr.Blocks() as demo:
gr.Markdown(
"""
# Sample-Aligned Decoders (SAD)
**A decoder post-training framework for latent generative models** β€” fine-tune *only* the VAE
decoder on the generator's own latents (~1 GPU-hour), keep the generator frozen. The decoder
keeps its architecture, so it is a **drop-in swap with zero extra inference cost**.
This demo runs **PixArt-Ξ± 256px**: one prompt β†’ one latent β†’ decoded twice, with the original
decoder **Dβ‚€** and with the SAD decoder. The latent is identical, so any difference you see is
purely the decoder correcting color cast, exposure and texture β€” content and layout stay put.
Sampler fixed to the authors' recipe (20 DPM-Solver++ steps, cfg 4.5) that this SAD decoder
was trained for.
πŸ“„ [Paper](https://huggingface.co/papers/2611.22458) Β· πŸ’» [Code](https://github.com/torpedo2003/SAD) Β· πŸ€— [Decoders](https://huggingface.co/torpedo2003/SAD)
""".strip()
)
with gr.Row():
with gr.Column():
prompt = gr.Textbox(
label="Prompt",
placeholder="A small cactus with a happy face in the Sahara desert.",
lines=2,
)
seed = gr.Slider(0, 2**31 - 1, value=0, step=1, label="Seed")
show_diff = gr.Checkbox(value=True, label="Show |SAD βˆ’ Dβ‚€| heatmap")
btn = gr.Button("Generate & compare", variant="primary")
with gr.Column():
out_d0 = gr.Image(label="Original decoder Dβ‚€ (PixArt-Ξ±)", height=290)
out_sad = gr.Image(label="SAD decoder (pixart-alpha-256/sad-gan)", height=290)
diff_gallery = gr.Image(label="|SAD βˆ’ Dβ‚€| heatmap (paper colormap)", height=290)
info = gr.Markdown()
btn.click(
fn=generate,
inputs=[prompt, seed, show_diff],
outputs=[out_d0, out_sad, diff_gallery, info],
api_name="generate",
)
gr.Examples(
examples=[
["A small cactus with a happy face in the Sahara desert.", 0],
["A 4k dslr image of a lemur wearing a red magician hat and a blue coat performing magic tricks with cards in a garden.", 3],
["Pirate ship trapped in a cosmic maelstrom nebula, rendered in cosmic beach whirlpool engine, volumetric lighting, spectacular, ambient lights, light pollution, cinematic atmosphere, art nouveau style, illustration art artwork by SenseiJaye, intricate detail.", 7],
["nature vs human nature, surreal, UHD, 8k, hyper details, rich colors, photograph.", 11],
],
inputs=[prompt, seed],
fn=generate,
outputs=[out_d0, out_sad, diff_gallery, info],
cache_examples=True,
cache_mode="lazy",
)
demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)