Spaces:
Running on Zero
Running on Zero
File size: 8,378 Bytes
e38ae43 622badc e38ae43 622badc e38ae43 622badc e38ae43 622badc e38ae43 d0e6d0d e38ae43 d0e6d0d e38ae43 d0e6d0d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 | """Sample-Aligned Decoders (SAD) β side-by-side demo on PixArt-Ξ± 256px.
Generates a latent ONCE with the frozen PixArt-Ξ± base model, then decodes that
SAME latent with (a) the original SD-VAE decoder D0 and (b) the SAD decoder
fine-tuned for this generator. Content and layout stay identical β only color,
exposure and texture change, which is exactly the paper's claim.
The sampler settings are fixed to the authors' recipe (20 DPM-Solver++ steps,
guidance 4.5) because each SAD decoder is trained on latents from exactly that
configuration (see training/README.md in the SAD repo).
"""
import os
import time
import spaces # must come before torch / diffusers / transformers
import gradio as gr
import numpy as np
import torch
from diffusers import AutoencoderKL, PixArtAlphaPipeline, Transformer2DModel
from sad import load_sad_decoder
HERE = os.path.dirname(os.path.abspath(__file__))
# ---------------------------------------------------------------------------
# Model loading (module scope, eager) β exactly the authors' recipe from
# training/gen_latents_pixart.py:
# transformer from PixArt-alpha/PixArt-XL-2-256x256 (256px)
# all other components from PixArt-alpha/PixArt-XL-2-512x512 (variant=fp16)
# The SAD decoder is architecturally identical, so it drops straight into the
# same AutoencoderKL.
# ---------------------------------------------------------------------------
TRANSFORMER_REPO = "PixArt-alpha/PixArt-XL-2-256x256"
PIPELINE_REPO = "PixArt-alpha/PixArt-XL-2-512x512"
SAD_CKPT = "pixart-alpha-256/sad-gan"
from transformers import T5Tokenizer
# Pre-converted fast tokenizer shipped with the Space (tokenizer.json) β avoids
# the sentencepiece/protobuf slow-tokenizer conversion entirely.
_tokenizer = T5Tokenizer.from_pretrained(os.path.join(HERE, "tokenizer"))
_transformer = Transformer2DModel.from_pretrained(
TRANSFORMER_REPO, subfolder="transformer", torch_dtype=torch.float16
)
pipe = PixArtAlphaPipeline.from_pretrained(
PIPELINE_REPO,
transformer=_transformer,
tokenizer=_tokenizer,
variant="fp16",
torch_dtype=torch.float16,
)
pipe.to("cuda")
pipe.set_progress_bar_config(disable=True)
# Two independent VAEs, decoded in fp32 like the authors' evaluation code:
# vae_d0 β the original decoder D0
# vae_sad β the same VAE with the SAD decoder swapped in (decoder.* only)
vae_d0 = AutoencoderKL.from_pretrained(PIPELINE_REPO, subfolder="vae").to("cuda").eval()
vae_sad = AutoencoderKL.from_pretrained(PIPELINE_REPO, subfolder="vae").to("cuda").eval()
load_sad_decoder(vae_sad, SAD_CKPT)
# Authors' fixed sampler recipe for this decoder.
STEPS = 20
GUIDANCE = 4.5
def _to_uint8(x: torch.Tensor) -> np.ndarray:
"""[-1,1] float [1,3,H,W] -> uint8 numpy [H,W,3] (authors' to_uint8)."""
x = x.clamp(-1, 1)
return ((x + 1) * 127.5).round().to(torch.uint8)[0].permute(1, 2, 0).cpu().numpy()
def heat(d: np.ndarray) -> np.ndarray:
"""|SAD - D0| heatmap with the paper's colormap (from SAD demo.py)."""
anchors = np.array(
[(0, 0, 4), (40, 11, 84), (101, 21, 110), (159, 42, 99), (212, 72, 66),
(245, 125, 21), (250, 193, 39), (252, 255, 164)], np.float32
)
x = np.clip(d / np.percentile(d, 99.5), 0, 1)
t = np.linspace(0, 1, len(anchors))
return np.stack([np.interp(x, t, anchors[:, c]) for c in range(3)], -1).astype(np.uint8)
@spaces.GPU(duration=10)
def generate(prompt: str, seed: int = 0, show_diff: bool = True,
progress=gr.Progress(track_tqdm=True)):
"""Generate one latent with PixArt-alpha and decode it with both decoders.
The sampler is fixed to the authors' recipe (20 DPM-Solver++ steps,
guidance 4.5) that the SAD decoder was trained for.
Args:
prompt: text prompt for PixArt-alpha.
seed: random seed (the same latent is decoded by both decoders).
show_diff: also render the |SAD - D0| heatmap.
Returns:
(original decoder image, SAD decoder image, optional heatmap, timing text)
"""
t0 = time.perf_counter()
gen = torch.Generator(device="cuda").manual_seed(int(seed))
# 1) sample ONE latent with the frozen base generator (authors' exact call)
latents = pipe(
[prompt],
height=256,
width=256,
num_inference_steps=STEPS,
guidance_scale=GUIDANCE,
generator=gen,
output_type="latent",
use_resolution_binning=False,
).images
# 2) raw VAE space: undo the scaling factor
raw = latents.float() / pipe.vae.config.scaling_factor
# 3) decode the SAME latent with both decoders
with torch.no_grad():
x_d0 = vae_d0.decode(raw).sample.clamp(-1, 1)
x_sad = vae_sad.decode(raw).sample.clamp(-1, 1)
im_d0 = _to_uint8(x_d0)
im_sad = _to_uint8(x_sad)
diff_img = None
if show_diff:
d = np.abs(im_sad.astype(np.float32) - im_d0.astype(np.float32)).mean(-1)
diff_img = heat(d)
dt = time.perf_counter() - t0
info = (
f"1 latent ({STEPS} steps, cfg {GUIDANCE}) decoded twice in **{dt:.1f}s** "
f"β same seed β same latent, so every difference you see is the decoder."
)
return im_d0, im_sad, diff_img, info
# ---------------------------------------------------------------------------
# UI
# ---------------------------------------------------------------------------
CSS = """
.gradio-container {max-width: 1100px !important; margin: 0 auto;}
.dark .gradio-container {background: var(--background-fill-primary);}
"""
with gr.Blocks() as demo:
gr.Markdown(
"""
# Sample-Aligned Decoders (SAD)
**A decoder post-training framework for latent generative models** β fine-tune *only* the VAE
decoder on the generator's own latents (~1 GPU-hour), keep the generator frozen. The decoder
keeps its architecture, so it is a **drop-in swap with zero extra inference cost**.
This demo runs **PixArt-Ξ± 256px**: one prompt β one latent β decoded twice, with the original
decoder **Dβ** and with the SAD decoder. The latent is identical, so any difference you see is
purely the decoder correcting color cast, exposure and texture β content and layout stay put.
Sampler fixed to the authors' recipe (20 DPM-Solver++ steps, cfg 4.5) that this SAD decoder
was trained for.
π [Paper](https://huggingface.co/papers/2611.22458) Β· π» [Code](https://github.com/torpedo2003/SAD) Β· π€ [Decoders](https://huggingface.co/torpedo2003/SAD)
""".strip()
)
with gr.Row():
with gr.Column():
prompt = gr.Textbox(
label="Prompt",
placeholder="A small cactus with a happy face in the Sahara desert.",
lines=2,
)
seed = gr.Slider(0, 2**31 - 1, value=0, step=1, label="Seed")
show_diff = gr.Checkbox(value=True, label="Show |SAD β Dβ| heatmap")
btn = gr.Button("Generate & compare", variant="primary")
with gr.Column():
out_d0 = gr.Image(label="Original decoder Dβ (PixArt-Ξ±)", height=290)
out_sad = gr.Image(label="SAD decoder (pixart-alpha-256/sad-gan)", height=290)
diff_gallery = gr.Image(label="|SAD β Dβ| heatmap (paper colormap)", height=290)
info = gr.Markdown()
btn.click(
fn=generate,
inputs=[prompt, seed, show_diff],
outputs=[out_d0, out_sad, diff_gallery, info],
api_name="generate",
)
gr.Examples(
examples=[
["A small cactus with a happy face in the Sahara desert.", 0],
["A 4k dslr image of a lemur wearing a red magician hat and a blue coat performing magic tricks with cards in a garden.", 3],
["Pirate ship trapped in a cosmic maelstrom nebula, rendered in cosmic beach whirlpool engine, volumetric lighting, spectacular, ambient lights, light pollution, cinematic atmosphere, art nouveau style, illustration art artwork by SenseiJaye, intricate detail.", 7],
["nature vs human nature, surreal, UHD, 8k, hyper details, rich colors, photograph.", 11],
],
inputs=[prompt, seed],
fn=generate,
outputs=[out_d0, out_sad, diff_gallery, info],
cache_examples=True,
cache_mode="lazy",
)
demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)
|