Spaces:
Running on Zero
Running on Zero
Download app.py from hugging-apps/sample-aligned-decoders: direct link, hf CLI and curl.
- Browser
- Download file 8.38 kB
-
https://huggingface.co/spaces/hugging-apps/sample-aligned-decoders/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-apps/sample-aligned-decoders/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-apps/sample-aligned-decoders/resolve/main/app.py
8.38 kB
| """Sample-Aligned Decoders (SAD) β side-by-side demo on PixArt-Ξ± 256px. | |
| Generates a latent ONCE with the frozen PixArt-Ξ± base model, then decodes that | |
| SAME latent with (a) the original SD-VAE decoder D0 and (b) the SAD decoder | |
| fine-tuned for this generator. Content and layout stay identical β only color, | |
| exposure and texture change, which is exactly the paper's claim. | |
| The sampler settings are fixed to the authors' recipe (20 DPM-Solver++ steps, | |
| guidance 4.5) because each SAD decoder is trained on latents from exactly that | |
| configuration (see training/README.md in the SAD repo). | |
| """ | |
| import os | |
| import time | |
| import spaces # must come before torch / diffusers / transformers | |
| import gradio as gr | |
| import numpy as np | |
| import torch | |
| from diffusers import AutoencoderKL, PixArtAlphaPipeline, Transformer2DModel | |
| from sad import load_sad_decoder | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| # --------------------------------------------------------------------------- | |
| # Model loading (module scope, eager) β exactly the authors' recipe from | |
| # training/gen_latents_pixart.py: | |
| # transformer from PixArt-alpha/PixArt-XL-2-256x256 (256px) | |
| # all other components from PixArt-alpha/PixArt-XL-2-512x512 (variant=fp16) | |
| # The SAD decoder is architecturally identical, so it drops straight into the | |
| # same AutoencoderKL. | |
| # --------------------------------------------------------------------------- | |
| TRANSFORMER_REPO = "PixArt-alpha/PixArt-XL-2-256x256" | |
| PIPELINE_REPO = "PixArt-alpha/PixArt-XL-2-512x512" | |
| SAD_CKPT = "pixart-alpha-256/sad-gan" | |
| from transformers import T5Tokenizer | |
| # Pre-converted fast tokenizer shipped with the Space (tokenizer.json) β avoids | |
| # the sentencepiece/protobuf slow-tokenizer conversion entirely. | |
| _tokenizer = T5Tokenizer.from_pretrained(os.path.join(HERE, "tokenizer")) | |
| _transformer = Transformer2DModel.from_pretrained( | |
| TRANSFORMER_REPO, subfolder="transformer", torch_dtype=torch.float16 | |
| ) | |
| pipe = PixArtAlphaPipeline.from_pretrained( | |
| PIPELINE_REPO, | |
| transformer=_transformer, | |
| tokenizer=_tokenizer, | |
| variant="fp16", | |
| torch_dtype=torch.float16, | |
| ) | |
| pipe.to("cuda") | |
| pipe.set_progress_bar_config(disable=True) | |
| # Two independent VAEs, decoded in fp32 like the authors' evaluation code: | |
| # vae_d0 β the original decoder D0 | |
| # vae_sad β the same VAE with the SAD decoder swapped in (decoder.* only) | |
| vae_d0 = AutoencoderKL.from_pretrained(PIPELINE_REPO, subfolder="vae").to("cuda").eval() | |
| vae_sad = AutoencoderKL.from_pretrained(PIPELINE_REPO, subfolder="vae").to("cuda").eval() | |
| load_sad_decoder(vae_sad, SAD_CKPT) | |
| # Authors' fixed sampler recipe for this decoder. | |
| STEPS = 20 | |
| GUIDANCE = 4.5 | |
| def _to_uint8(x: torch.Tensor) -> np.ndarray: | |
| """[-1,1] float [1,3,H,W] -> uint8 numpy [H,W,3] (authors' to_uint8).""" | |
| x = x.clamp(-1, 1) | |
| return ((x + 1) * 127.5).round().to(torch.uint8)[0].permute(1, 2, 0).cpu().numpy() | |
| def heat(d: np.ndarray) -> np.ndarray: | |
| """|SAD - D0| heatmap with the paper's colormap (from SAD demo.py).""" | |
| anchors = np.array( | |
| [(0, 0, 4), (40, 11, 84), (101, 21, 110), (159, 42, 99), (212, 72, 66), | |
| (245, 125, 21), (250, 193, 39), (252, 255, 164)], np.float32 | |
| ) | |
| x = np.clip(d / np.percentile(d, 99.5), 0, 1) | |
| t = np.linspace(0, 1, len(anchors)) | |
| return np.stack([np.interp(x, t, anchors[:, c]) for c in range(3)], -1).astype(np.uint8) | |
| def generate(prompt: str, seed: int = 0, show_diff: bool = True, | |
| progress=gr.Progress(track_tqdm=True)): | |
| """Generate one latent with PixArt-alpha and decode it with both decoders. | |
| The sampler is fixed to the authors' recipe (20 DPM-Solver++ steps, | |
| guidance 4.5) that the SAD decoder was trained for. | |
| Args: | |
| prompt: text prompt for PixArt-alpha. | |
| seed: random seed (the same latent is decoded by both decoders). | |
| show_diff: also render the |SAD - D0| heatmap. | |
| Returns: | |
| (original decoder image, SAD decoder image, optional heatmap, timing text) | |
| """ | |
| t0 = time.perf_counter() | |
| gen = torch.Generator(device="cuda").manual_seed(int(seed)) | |
| # 1) sample ONE latent with the frozen base generator (authors' exact call) | |
| latents = pipe( | |
| [prompt], | |
| height=256, | |
| width=256, | |
| num_inference_steps=STEPS, | |
| guidance_scale=GUIDANCE, | |
| generator=gen, | |
| output_type="latent", | |
| use_resolution_binning=False, | |
| ).images | |
| # 2) raw VAE space: undo the scaling factor | |
| raw = latents.float() / pipe.vae.config.scaling_factor | |
| # 3) decode the SAME latent with both decoders | |
| with torch.no_grad(): | |
| x_d0 = vae_d0.decode(raw).sample.clamp(-1, 1) | |
| x_sad = vae_sad.decode(raw).sample.clamp(-1, 1) | |
| im_d0 = _to_uint8(x_d0) | |
| im_sad = _to_uint8(x_sad) | |
| diff_img = None | |
| if show_diff: | |
| d = np.abs(im_sad.astype(np.float32) - im_d0.astype(np.float32)).mean(-1) | |
| diff_img = heat(d) | |
| dt = time.perf_counter() - t0 | |
| info = ( | |
| f"1 latent ({STEPS} steps, cfg {GUIDANCE}) decoded twice in **{dt:.1f}s** " | |
| f"β same seed β same latent, so every difference you see is the decoder." | |
| ) | |
| return im_d0, im_sad, diff_img, info | |
| # --------------------------------------------------------------------------- | |
| # UI | |
| # --------------------------------------------------------------------------- | |
| CSS = """ | |
| .gradio-container {max-width: 1100px !important; margin: 0 auto;} | |
| .dark .gradio-container {background: var(--background-fill-primary);} | |
| """ | |
| with gr.Blocks() as demo: | |
| gr.Markdown( | |
| """ | |
| # Sample-Aligned Decoders (SAD) | |
| **A decoder post-training framework for latent generative models** β fine-tune *only* the VAE | |
| decoder on the generator's own latents (~1 GPU-hour), keep the generator frozen. The decoder | |
| keeps its architecture, so it is a **drop-in swap with zero extra inference cost**. | |
| This demo runs **PixArt-Ξ± 256px**: one prompt β one latent β decoded twice, with the original | |
| decoder **Dβ** and with the SAD decoder. The latent is identical, so any difference you see is | |
| purely the decoder correcting color cast, exposure and texture β content and layout stay put. | |
| Sampler fixed to the authors' recipe (20 DPM-Solver++ steps, cfg 4.5) that this SAD decoder | |
| was trained for. | |
| π [Paper](https://huggingface.co/papers/2611.22458) Β· π» [Code](https://github.com/torpedo2003/SAD) Β· π€ [Decoders](https://huggingface.co/torpedo2003/SAD) | |
| """.strip() | |
| ) | |
| with gr.Row(): | |
| with gr.Column(): | |
| prompt = gr.Textbox( | |
| label="Prompt", | |
| placeholder="A small cactus with a happy face in the Sahara desert.", | |
| lines=2, | |
| ) | |
| seed = gr.Slider(0, 2**31 - 1, value=0, step=1, label="Seed") | |
| show_diff = gr.Checkbox(value=True, label="Show |SAD β Dβ| heatmap") | |
| btn = gr.Button("Generate & compare", variant="primary") | |
| with gr.Column(): | |
| out_d0 = gr.Image(label="Original decoder Dβ (PixArt-Ξ±)", height=290) | |
| out_sad = gr.Image(label="SAD decoder (pixart-alpha-256/sad-gan)", height=290) | |
| diff_gallery = gr.Image(label="|SAD β Dβ| heatmap (paper colormap)", height=290) | |
| info = gr.Markdown() | |
| btn.click( | |
| fn=generate, | |
| inputs=[prompt, seed, show_diff], | |
| outputs=[out_d0, out_sad, diff_gallery, info], | |
| api_name="generate", | |
| ) | |
| gr.Examples( | |
| examples=[ | |
| ["A small cactus with a happy face in the Sahara desert.", 0], | |
| ["A 4k dslr image of a lemur wearing a red magician hat and a blue coat performing magic tricks with cards in a garden.", 3], | |
| ["Pirate ship trapped in a cosmic maelstrom nebula, rendered in cosmic beach whirlpool engine, volumetric lighting, spectacular, ambient lights, light pollution, cinematic atmosphere, art nouveau style, illustration art artwork by SenseiJaye, intricate detail.", 7], | |
| ["nature vs human nature, surreal, UHD, 8k, hyper details, rich colors, photograph.", 11], | |
| ], | |
| inputs=[prompt, seed], | |
| fn=generate, | |
| outputs=[out_d0, out_sad, diff_gallery, info], | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| ) | |
| demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True) | |