Spaces:
Running on Zero
Running on Zero
PradhaanSBhat commited on
Commit ·
5f8079c
1
Parent(s): 14b7057
Update HF Space Files
Browse files- .gitignore +103 -0
- README.md +72 -9
- assets/favicon.svg +1 -0
- box-editor.html +0 -0
- demo.py +394 -0
- pipelines/pipeline_flux_kontext.py +1277 -0
- requirements.txt +15 -0
.gitignore
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Byte-compiled / optimized / DLL files
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[cod]
|
| 4 |
+
*$py.class
|
| 5 |
+
|
| 6 |
+
# C extensions
|
| 7 |
+
*.so
|
| 8 |
+
|
| 9 |
+
# Distribution / packaging
|
| 10 |
+
.Python
|
| 11 |
+
build/
|
| 12 |
+
develop-eggs/
|
| 13 |
+
dist/
|
| 14 |
+
downloads/
|
| 15 |
+
eggs/
|
| 16 |
+
.eggs/
|
| 17 |
+
lib/
|
| 18 |
+
lib64/
|
| 19 |
+
parts/
|
| 20 |
+
sdist/
|
| 21 |
+
var/
|
| 22 |
+
*.egg-info/
|
| 23 |
+
.installed.cfg
|
| 24 |
+
*.egg
|
| 25 |
+
|
| 26 |
+
# Virtual environments
|
| 27 |
+
venv/
|
| 28 |
+
ENV/
|
| 29 |
+
env/
|
| 30 |
+
.venv/
|
| 31 |
+
.ipynb_checkpoints/
|
| 32 |
+
|
| 33 |
+
# PyInstaller
|
| 34 |
+
*.manifest
|
| 35 |
+
*.spec
|
| 36 |
+
|
| 37 |
+
# Installer logs
|
| 38 |
+
pip-log.txt
|
| 39 |
+
pip-delete-this-directory.txt
|
| 40 |
+
|
| 41 |
+
# Unit test / coverage reports
|
| 42 |
+
htmlcov/
|
| 43 |
+
.tox/
|
| 44 |
+
.nox/
|
| 45 |
+
.coverage
|
| 46 |
+
.coverage.*
|
| 47 |
+
.cache
|
| 48 |
+
nosetests.xml
|
| 49 |
+
coverage.xml
|
| 50 |
+
*.cover
|
| 51 |
+
.hypothesis/
|
| 52 |
+
|
| 53 |
+
# Jupyter Notebook
|
| 54 |
+
*.ipynb_checkpoints
|
| 55 |
+
|
| 56 |
+
# PyCharm
|
| 57 |
+
.idea/
|
| 58 |
+
*.iml
|
| 59 |
+
|
| 60 |
+
# VS Code
|
| 61 |
+
.vscode/
|
| 62 |
+
|
| 63 |
+
# Data, models, outputs
|
| 64 |
+
data/**
|
| 65 |
+
datasets/**
|
| 66 |
+
*.h5
|
| 67 |
+
*.hdf5
|
| 68 |
+
*.npz
|
| 69 |
+
*.npy
|
| 70 |
+
*.ckpt
|
| 71 |
+
*.pth
|
| 72 |
+
*.pt
|
| 73 |
+
*.onnx
|
| 74 |
+
*.pb
|
| 75 |
+
*.tflite
|
| 76 |
+
*.log
|
| 77 |
+
*.csv
|
| 78 |
+
*.tsv
|
| 79 |
+
*.pkl
|
| 80 |
+
*.pickle
|
| 81 |
+
*.joblib
|
| 82 |
+
*.mat
|
| 83 |
+
|
| 84 |
+
# System files
|
| 85 |
+
.DS_Store
|
| 86 |
+
Thumbs.db
|
| 87 |
+
|
| 88 |
+
# Temporary files
|
| 89 |
+
*.tmp
|
| 90 |
+
*.temp
|
| 91 |
+
*.swp
|
| 92 |
+
*.swo
|
| 93 |
+
|
| 94 |
+
# Slurm and job logs
|
| 95 |
+
slurm-*.out
|
| 96 |
+
|
| 97 |
+
# Gradio files
|
| 98 |
+
.gradio
|
| 99 |
+
|
| 100 |
+
# Other
|
| 101 |
+
*.bak
|
| 102 |
+
*.orig
|
| 103 |
+
*.old
|
README.md
CHANGED
|
@@ -1,15 +1,78 @@
|
|
| 1 |
---
|
| 2 |
-
title: Thinking
|
| 3 |
-
emoji:
|
| 4 |
colorFrom: red
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version:
|
| 8 |
-
|
| 9 |
-
app_file: app.py
|
| 10 |
pinned: false
|
| 11 |
-
|
| 12 |
-
short_description: 'NeurIPS''26-Thinking in Boxes: 3D Editing in Real Images Easy'
|
| 13 |
---
|
| 14 |
|
| 15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Thinking in Boxes
|
| 3 |
+
emoji: 📦
|
| 4 |
colorFrom: red
|
| 5 |
+
colorTo: gray
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 4.44.0
|
| 8 |
+
app_file: demo.py
|
|
|
|
| 9 |
pinned: false
|
| 10 |
+
short_description: 3D Editing in Real Images Made Easy (NeurIPS 2026)
|
|
|
|
| 11 |
---
|
| 12 |
|
| 13 |
+
<div align="center">
|
| 14 |
+
|
| 15 |
+
# Thinking in Boxes: 3D Editing in Real Images Made Easy
|
| 16 |
+
|
| 17 |
+
### NeurIPS 2026
|
| 18 |
+
|
| 19 |
+
[**Pradhaan S Bhat**](https://pradhaansbhat.github.io/)<sup>1</sup><sup>∗</sup> · [**Naveen Chandra R**](https://www.linkedin.com/in/naveen-chandra-r-7230aa192)<sup>1</sup><sup>∗</sup> · [**Rishubh Parihar**](https://rishubhpar.github.io/)<sup>1</sup> · [**Vaibhav Vavilala**](https://www.linkedin.com/in/vaibhav-vavilala)<sup>2</sup> · [**R. Venkatesh Babu**](https://cds.iisc.ac.in/faculty/venky/)<sup>1</sup> · [**D. A. Forsyth**](http://luthuli.cs.uiuc.edu/~daf/)<sup>3</sup> · [**Anand Bhattad**](https://anandbhattad.github.io/)<sup>4</sup>
|
| 20 |
+
|
| 21 |
+
<sup>1</sup> Indian Institute of Science · <sup>2</sup> Apple · <sup>3</sup> UIUC · <sup>4</sup> Johns Hopkins University
|
| 22 |
+
|
| 23 |
+
<sup>∗</sup> Equal Contribution
|
| 24 |
+
|
| 25 |
+
[](https://arxiv.org/abs/2606.20556)
|
| 26 |
+
[](https://thinking-in-boxes.github.io)
|
| 27 |
+
[](https://huggingface.co/pradhaansbhat/Thinking-In-Boxes)
|
| 28 |
+
[](https://huggingface.co/datasets/pradhaansbhat/Thinking-In-Boxes)
|
| 29 |
+
[](https://github.com/PradhaanSBhat/Thinking-In-Boxes)
|
| 30 |
+
|
| 31 |
+
</div>
|
| 32 |
+
|
| 33 |
+
---
|
| 34 |
+
|
| 35 |
+
## 📚 Abstract
|
| 36 |
+
|
| 37 |
+
Text and 2D-conditioning interfaces provide weak, ambiguous control over spatial
|
| 38 |
+
transformations in image editing — particularly under large object motions and
|
| 39 |
+
camera changes. Prior work has used 3D primitives such as boxes, but only as loose
|
| 40 |
+
conditioning signals indicating approximate object location rather than specifying
|
| 41 |
+
the transformation.
|
| 42 |
+
|
| 43 |
+
We instead use **3D boxes as structured specifications**: the user provides the input
|
| 44 |
+
and output boxes of the edit, casting editing as a well-posed geometry problem. This
|
| 45 |
+
"thinking in boxes" interface, where each box face is color-coded to convey 3D
|
| 46 |
+
orientation, gives precise control over translation, rotation, scaling, and viewpoint
|
| 47 |
+
changes in real images while preserving scene and object identity, and recovering
|
| 48 |
+
previously unseen object regions. To ground transformations in scene appearance, we
|
| 49 |
+
introduce a **depth-aligned planar floor** as a global reference frame, shaded with
|
| 50 |
+
depth-aware cues.
|
| 51 |
+
|
| 52 |
+
Conditioned on this structure, an image generator produces consistent results under
|
| 53 |
+
large transformations. Trained in two stages — on synthetic multi-object scenes and a
|
| 54 |
+
small set of real-world videos from Objectron — the system generalizes to complex,
|
| 55 |
+
in-the-wild real images. Our method operates directly on real photographs and
|
| 56 |
+
substantially **outperforms recent state-of-the-art methods** on large 3D edits.
|
| 57 |
+
|
| 58 |
+
---
|
| 59 |
+
|
| 60 |
+
# Demo Usage
|
| 61 |
+
|
| 62 |
+
Interactive demo for **[NeurIPS'26] Thinking in Boxes: 3D Editing in Real Images Made Easy**.
|
| 63 |
+
|
| 64 |
+
- Fit 3D boxes to objects, set a **source** pose, bookmark it on the left rail, edit to a **target** pose, then generate.
|
| 65 |
+
- Read the tooltips and hover over icons for details on how to use the 3D Editor.
|
| 66 |
+
- Runs on **ZeroGPU** (free, time-sliced A100). Generation consumes your daily GPU quota.
|
| 67 |
+
- 3D auto-detection runs in **mock** mode on the Space (the full WildDet3D detector needs a
|
| 68 |
+
dedicated GPU/deps). Place and refine 3D boxes by hand. (See the [GitHub Repository](https://github.com/PradhaanSBhat/Thinking-In-Boxes) to use WildDet3D)
|
| 69 |
+
|
| 70 |
+
## Setup notes (for whoever deploys this Space)
|
| 71 |
+
|
| 72 |
+
1. **Hardware:** Settings → Hardware → **ZeroGPU**.
|
| 73 |
+
2. **Secret:** add `HF_TOKEN` from an account that has accepted the
|
| 74 |
+
[FLUX.1-Kontext-dev](https://huggingface.co/black-forest-labs/FLUX.1-Kontext-dev) license —
|
| 75 |
+
the base model is gated and won't download otherwise.
|
| 76 |
+
3. **Files in the Space repo:** `assets/` `demo.py`, `app.py`, `box-editor.html`,
|
| 77 |
+
`pipelines/pipeline_flux_kontext.py` (the custom pipeline), and `requirements.txt`.
|
| 78 |
+
4. The default model + LoRA download on first boot (slow once, cached after).
|
assets/favicon.svg
ADDED
|
|
box-editor.html
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
demo.py
ADDED
|
@@ -0,0 +1,394 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Thinking in Boxes — Hugging Face Spaces (ZeroGPU) demo.
|
| 2 |
+
|
| 3 |
+
Self-contained Gradio-SDK app for a FREE ZeroGPU Space.
|
| 4 |
+
• Generation runs inside @spaces.GPU (the pipeline moves to CUDA per request),
|
| 5 |
+
• The 3D box editor is embedded via <iframe srcdoc=...>,
|
| 6 |
+
• Place and move boxes by hand.
|
| 7 |
+
|
| 8 |
+
Space config (README front-matter): sdk: gradio · app_file: demo.py
|
| 9 |
+
Secret: HF_TOKEN (account that accepted the FLUX.1-Kontext-dev license).
|
| 10 |
+
Local run: `python demo.py`.
|
| 11 |
+
"""
|
| 12 |
+
import base64
|
| 13 |
+
import inspect
|
| 14 |
+
import io
|
| 15 |
+
import json
|
| 16 |
+
import os
|
| 17 |
+
import random
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
from types import SimpleNamespace
|
| 20 |
+
|
| 21 |
+
os.environ.setdefault("HF_ENDPOINT", "https://huggingface.co")
|
| 22 |
+
os.environ["GRADIO_TEMP_DIR"] = os.environ.get("GRADIO_TEMP_DIR") or "/tmp/gradio_tib"
|
| 23 |
+
os.makedirs(os.environ["GRADIO_TEMP_DIR"], exist_ok=True)
|
| 24 |
+
|
| 25 |
+
import numpy as np
|
| 26 |
+
import torch
|
| 27 |
+
import gradio as gr
|
| 28 |
+
from diffusers import DiffusionPipeline
|
| 29 |
+
from PIL import Image
|
| 30 |
+
|
| 31 |
+
# ZeroGPU decorator; no-op fallback so this file also runs locally without `spaces`.
|
| 32 |
+
try:
|
| 33 |
+
import spaces
|
| 34 |
+
except Exception:
|
| 35 |
+
def _gpu(*a, **k):
|
| 36 |
+
if a and callable(a[0]):
|
| 37 |
+
return a[0]
|
| 38 |
+
def deco(f):
|
| 39 |
+
return f
|
| 40 |
+
return deco
|
| 41 |
+
spaces = SimpleNamespace(GPU=_gpu)
|
| 42 |
+
|
| 43 |
+
HERE = Path(__file__).resolve().parent
|
| 44 |
+
EDITOR_HTML = HERE / "box-editor.html"
|
| 45 |
+
|
| 46 |
+
# ── Defaults (mirror inference.sh) ──
|
| 47 |
+
DEFAULT_MODEL = "black-forest-labs/FLUX.1-Kontext-dev"
|
| 48 |
+
DEFAULT_PIPELINE = "pipelines/pipeline_flux_kontext.py"
|
| 49 |
+
DEFAULT_DTYPE = "bfloat16"
|
| 50 |
+
DEFAULT_LORA = "pradhaansbhat/Thinking-In-Boxes"
|
| 51 |
+
DEFAULT_WEIGHT = "pytorch_lora_weights.safetensors"
|
| 52 |
+
DEFAULT_ADAPTER = "100K+20K-LoRA"
|
| 53 |
+
DEFAULT_LOCAL = False
|
| 54 |
+
DEFAULT_STEPS = 28
|
| 55 |
+
DEFAULT_GUIDANCE = 3.5
|
| 56 |
+
DEFAULT_TRUE_CFG = 1.0
|
| 57 |
+
DEFAULT_SCALE = 1.0
|
| 58 |
+
DEFAULT_HEIGHT = 512
|
| 59 |
+
DEFAULT_WIDTH = 512
|
| 60 |
+
DEFAULT_MAX_SEQ = 512
|
| 61 |
+
|
| 62 |
+
DTYPE_MAP = {
|
| 63 |
+
"bfloat16": torch.bfloat16, "float16": torch.float16, "half": torch.float16,
|
| 64 |
+
"float32": torch.float32, "float": torch.float32, "float64": torch.float64, "double": torch.float64,
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
STATE = {"pipe": None, "max_sequence_length": DEFAULT_MAX_SEQ, "last_seed": 0}
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
# ─────────────────────────────────────────────────────── pipeline (CPU load) ──
|
| 71 |
+
def load_pipe_cpu():
|
| 72 |
+
"""Load the pipeline on CPU (ZeroGPU exposes CUDA only inside @spaces.GPU)."""
|
| 73 |
+
dtype = DTYPE_MAP[DEFAULT_DTYPE]
|
| 74 |
+
token = os.environ.get("HF_TOKEN") or None
|
| 75 |
+
pipe = DiffusionPipeline.from_pretrained(
|
| 76 |
+
DEFAULT_MODEL, custom_pipeline=DEFAULT_PIPELINE or None, trust_remote_code=True,
|
| 77 |
+
local_files_only=DEFAULT_LOCAL, token=token).to(dtype=dtype)
|
| 78 |
+
if DEFAULT_LORA:
|
| 79 |
+
pipe.transformer.load_lora_adapter(
|
| 80 |
+
DEFAULT_LORA, weight_name=DEFAULT_WEIGHT or None, adapter_name=DEFAULT_ADAPTER or None)
|
| 81 |
+
print(f"[tib-spaces] LoRA {DEFAULT_ADAPTER} loaded from {DEFAULT_LORA}", flush=True)
|
| 82 |
+
pipe.transformer.to(memory_format=torch.channels_last)
|
| 83 |
+
pipe.vae.to(memory_format=torch.channels_last)
|
| 84 |
+
return pipe
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def call_pipe(pipe, **kwargs):
|
| 88 |
+
sig = inspect.signature(pipe.__call__)
|
| 89 |
+
valid = {k: v for k, v in kwargs.items() if k in sig.parameters}
|
| 90 |
+
return pipe(**valid).images
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def dataurl_to_pil(u):
|
| 94 |
+
if not u:
|
| 95 |
+
return None
|
| 96 |
+
if "," in u:
|
| 97 |
+
u = u.split(",", 1)[1]
|
| 98 |
+
return Image.open(io.BytesIO(base64.b64decode(u))).convert("RGB")
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
# ──────────────────────────────────────────────── generation (ZeroGPU) ──
|
| 102 |
+
@spaces.GPU(duration=120)
|
| 103 |
+
def generate(image_u, src_u, tgt_u, steps, guidance, true_cfg, lora_scale,
|
| 104 |
+
randomize, height, width, num_images, progress=gr.Progress(track_tqdm=True)):
|
| 105 |
+
if STATE["pipe"] is None:
|
| 106 |
+
raise gr.Error("Model not loaded — check the Space logs (HF_TOKEN / license).")
|
| 107 |
+
pipe = STATE["pipe"].to("cuda")
|
| 108 |
+
src = dataurl_to_pil(src_u); tgt = dataurl_to_pil(tgt_u)
|
| 109 |
+
if src is None or tgt is None:
|
| 110 |
+
raise gr.Error("Grab the source + target boxes from the editor first.")
|
| 111 |
+
image = dataurl_to_pil(image_u)
|
| 112 |
+
seed = random.randint(0, 2**31 - 1) if randomize else STATE.get("last_seed", 0)
|
| 113 |
+
STATE["last_seed"] = seed
|
| 114 |
+
generator = torch.Generator(device="cuda").manual_seed(seed)
|
| 115 |
+
images = call_pipe(
|
| 116 |
+
pipe,
|
| 117 |
+
image=image, source_primitive=src, target_primitive=tgt,
|
| 118 |
+
prompt="", negative_prompt=None,
|
| 119 |
+
guidance_scale=float(guidance), true_cfg_scale=float(true_cfg),
|
| 120 |
+
num_inference_steps=int(steps), num_images_per_prompt=int(num_images),
|
| 121 |
+
height=int(height), width=int(width), max_area=int(height) * int(width),
|
| 122 |
+
max_sequence_length=int(STATE["max_sequence_length"]), generator=generator,
|
| 123 |
+
joint_attention_kwargs={"scale": float(lora_scale)},
|
| 124 |
+
)
|
| 125 |
+
return list(images), f"seed {seed}"
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def grab_show(status, src_u, tgt_u):
|
| 129 |
+
msg = (status or "").strip()
|
| 130 |
+
if msg.startswith("⚠") or not (src_u and tgt_u):
|
| 131 |
+
gr.Warning(msg.lstrip("⚠ ").strip() or "Set a source, edit the boxes, then grab.")
|
| 132 |
+
else:
|
| 133 |
+
gr.Info("✓ Grabbed image + source/target boxes from the editor.")
|
| 134 |
+
return dataurl_to_pil(src_u), dataurl_to_pil(tgt_u)
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
# ── mock detector (no WildDet3D on Spaces): plausible metric boxes, camera frame y-down ──
|
| 138 |
+
def run_mock(image_np, prompts, points=None, boxes=None):
|
| 139 |
+
H, W = image_np.shape[:2]
|
| 140 |
+
fx = 1.2 * W
|
| 141 |
+
K = [[fx, 0, W / 2], [0, fx, H / 2], [0, 0, 1]]
|
| 142 |
+
prompt_xy = []
|
| 143 |
+
if points:
|
| 144 |
+
prompt_xy = [(p[0], p[1]) for p in points if int(p[2]) == 1]
|
| 145 |
+
elif boxes:
|
| 146 |
+
prompt_xy = [((b[0] + b[2]) / 2, (b[1] + b[3]) / 2) for b in boxes]
|
| 147 |
+
if prompt_xy:
|
| 148 |
+
dets = []
|
| 149 |
+
for i, (u, v) in enumerate(prompt_xy):
|
| 150 |
+
Z = 6.0
|
| 151 |
+
X = (u - W / 2) * Z / fx
|
| 152 |
+
Y = (v - H / 2) * Z / fx
|
| 153 |
+
dets.append({"label": prompts[i] if i < len(prompts) else f"object {i+1}",
|
| 154 |
+
"class_id": i, "score": 0.88,
|
| 155 |
+
"box2d": [u - 40, v - 40, u + 40, v + 40],
|
| 156 |
+
"center": [X, Y, Z], "dims": [0.8, 1.0, 0.8], "quat": [1, 0, 0, 0]})
|
| 157 |
+
return {"image_hw": [H, W], "intrinsics": K, "detections": dets}
|
| 158 |
+
catalog = {
|
| 159 |
+
"car": {"center": [0.4, 0.9, 8.0], "dims": [1.8, 1.5, 4.3], "quat": [0.966, 0, 0.259, 0]},
|
| 160 |
+
"person": {"center": [-2.2, 0.5, 6.0], "dims": [0.6, 1.7, 0.6], "quat": [1, 0, 0, 0]},
|
| 161 |
+
"bicycle": {"center": [2.4, 0.8, 5.0], "dims": [0.6, 1.1, 1.7], "quat": [0.985, 0, 0.174, 0]},
|
| 162 |
+
"chair": {"center": [0.0, 0.6, 3.2], "dims": [0.6, 0.9, 0.6], "quat": [1, 0, 0, 0]},
|
| 163 |
+
"cup": {"center": [0.1, 0.4, 1.4], "dims": [0.09, 0.11, 0.09], "quat": [1, 0, 0, 0]},
|
| 164 |
+
}
|
| 165 |
+
dets = []
|
| 166 |
+
for i, p in enumerate(prompts):
|
| 167 |
+
pr = catalog.get(p.strip().lower(), {"center": [0, 0.6, 4 + i], "dims": [0.8, 0.8, 0.8], "quat": [1, 0, 0, 0]})
|
| 168 |
+
dets.append({"label": p.strip(), "class_id": i, "score": 0.9 - 0.05 * i,
|
| 169 |
+
"box2d": [W * 0.3, H * 0.4, W * 0.7, H * 0.8], **pr})
|
| 170 |
+
return {"image_hw": [H, W], "intrinsics": K, "detections": dets}
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def detect_objects(payload_json):
|
| 174 |
+
try:
|
| 175 |
+
req = json.loads(payload_json) if payload_json else {}
|
| 176 |
+
except Exception:
|
| 177 |
+
return json.dumps({"error": "bad request payload"})
|
| 178 |
+
img = dataurl_to_pil(req.get("image"))
|
| 179 |
+
if img is None:
|
| 180 |
+
return json.dumps({"error": "no image — load one in the editor first"})
|
| 181 |
+
arr = np.array(img).astype(np.float32)
|
| 182 |
+
prompts = [p.strip() for p in (req.get("prompts") or "").split(",") if p.strip()]
|
| 183 |
+
points = req.get("points") or None
|
| 184 |
+
boxes = req.get("boxes") or None
|
| 185 |
+
try:
|
| 186 |
+
return json.dumps(run_mock(arr, prompts, points, boxes))
|
| 187 |
+
except Exception as e:
|
| 188 |
+
return json.dumps({"error": str(e)})
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
# ───────────────────────────────────────────── editor bridge (browser JS) ──
|
| 192 |
+
GRAB_JS = """
|
| 193 |
+
() => {
|
| 194 |
+
const f = document.getElementById('tib-editor');
|
| 195 |
+
if (!f || !f.contentWindow) return ["", "", "", "⚠ Editor not loaded yet."];
|
| 196 |
+
let st;
|
| 197 |
+
try { st = f.contentWindow.getSourceTarget(); }
|
| 198 |
+
catch (e) { return ["", "", "", "⚠ Editor not ready: " + e.message]; }
|
| 199 |
+
if (!st || st.error) return ["", "", "", "⚠ " + ((st && st.error) || "no data")];
|
| 200 |
+
return [st.image || "", st.source, st.target, "✓ Grabbed image + source + target from the editor."];
|
| 201 |
+
}
|
| 202 |
+
"""
|
| 203 |
+
|
| 204 |
+
DETECT_BRIDGE_JS = """
|
| 205 |
+
() => {
|
| 206 |
+
if (window.__tibDetectBridge) return;
|
| 207 |
+
window.__tibDetectBridge = true;
|
| 208 |
+
const setVal = (el, v) => {
|
| 209 |
+
const proto = el.tagName === 'TEXTAREA' ? window.HTMLTextAreaElement.prototype
|
| 210 |
+
: window.HTMLInputElement.prototype;
|
| 211 |
+
const d = Object.getOwnPropertyDescriptor(proto, 'value');
|
| 212 |
+
d && d.set ? d.set.call(el, v) : (el.value = v);
|
| 213 |
+
el.dispatchEvent(new Event('input', { bubbles: true }));
|
| 214 |
+
};
|
| 215 |
+
window.addEventListener('message', (e) => {
|
| 216 |
+
const d = e.data; if (!d || d.type !== 'tib-detect') return;
|
| 217 |
+
const box = document.querySelector('#tib-detect-payload');
|
| 218 |
+
const ta = box && (box.querySelector('textarea') || box.querySelector('input'));
|
| 219 |
+
if (!ta) return;
|
| 220 |
+
setVal(ta, JSON.stringify(d));
|
| 221 |
+
setTimeout(() => { const b = document.querySelector('#tib-detect-trigger');
|
| 222 |
+
if (b) (b.querySelector('button') || b).click(); }, 50);
|
| 223 |
+
});
|
| 224 |
+
}
|
| 225 |
+
"""
|
| 226 |
+
|
| 227 |
+
DETECT_RESULT_JS = """
|
| 228 |
+
(resultJson) => {
|
| 229 |
+
const f = document.getElementById('tib-editor');
|
| 230 |
+
if (!f || !f.contentWindow) return;
|
| 231 |
+
let p; try { p = JSON.parse(resultJson || '{}'); } catch (e) { p = { error: 'bad detector result' }; }
|
| 232 |
+
const msg = (p && p.error) ? { type: 'tib-detect-result', error: p.error }
|
| 233 |
+
: { type: 'tib-detect-result', detections: p };
|
| 234 |
+
try { f.contentWindow.postMessage(msg, '*'); } catch (e) {}
|
| 235 |
+
}
|
| 236 |
+
"""
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
# ─���────────────────────────────────────────────────────────────── theme/CSS ──
|
| 240 |
+
TERRACOTTA = gr.themes.colors.Color(
|
| 241 |
+
name="terracotta", c50="#faf1ee", c100="#f3d9d0", c200="#e8b5a6", c300="#d98f78",
|
| 242 |
+
c400="#cf7458", c500="#c0604a", c600="#a5503d", c700="#7b4f3a", c800="#5f3b2c",
|
| 243 |
+
c900="#432a20", c950="#2a1a14")
|
| 244 |
+
|
| 245 |
+
THEME = gr.themes.Soft(
|
| 246 |
+
primary_hue=TERRACOTTA, secondary_hue=TERRACOTTA, neutral_hue=gr.themes.colors.stone,
|
| 247 |
+
font=[gr.themes.GoogleFont("Outfit"), "system-ui", "sans-serif"],
|
| 248 |
+
font_mono=[gr.themes.GoogleFont("JetBrains Mono"), "monospace"],
|
| 249 |
+
)
|
| 250 |
+
|
| 251 |
+
_VARS = """
|
| 252 |
+
--body-background-fill:#F6F3EE; --background-fill-primary:#FFFFFF; --background-fill-secondary:#EDEAE3;
|
| 253 |
+
--block-background-fill:#FFFFFF; --block-border-color:#D8D4CB; --border-color-primary:#D8D4CB;
|
| 254 |
+
--border-color-accent-subdued:#e6c3ba; --panel-background-fill:#FFFFFF;
|
| 255 |
+
--body-text-color:#1A1714; --body-text-color-subdued:#6B6560;
|
| 256 |
+
--block-label-text-color:#C0604A; --block-label-background-fill:#F1EBE3; --block-title-text-color:#C0604A;
|
| 257 |
+
--button-primary-background-fill:#C0604A; --button-primary-background-fill-hover:#7B4F3A;
|
| 258 |
+
--button-primary-text-color:#FFFFFF; --button-primary-border-color:#C0604A;
|
| 259 |
+
--button-secondary-background-fill:#FFFFFF; --button-secondary-background-fill-hover:#EDEAE3;
|
| 260 |
+
--button-secondary-border-color:#D8D4CB; --button-secondary-text-color:#1A1714;
|
| 261 |
+
--input-background-fill:#FFFFFF; --input-border-color:#D8D4CB; --neutral-800:#3a352f;
|
| 262 |
+
--color-accent:#C0604A; --color-accent-soft:#f3d9d0; --link-text-color:#C0604A;
|
| 263 |
+
--checkbox-background-color:#FFFFFF; --checkbox-border-color:#C0604A;
|
| 264 |
+
--checkbox-background-color-selected:#C0604A; --checkbox-border-color-selected:#C0604A;
|
| 265 |
+
--checkbox-background-color-focus:#C0604A; --checkbox-border-color-focus:#C0604A;
|
| 266 |
+
"""
|
| 267 |
+
CSS = """
|
| 268 |
+
@import url('https://fonts.googleapis.com/css2?family=Cormorant+Garamond:ital,wght@0,500;0,600;1,500;1,600&display=swap');
|
| 269 |
+
:root, .gradio-container, gradio-app, .dark { __VARS__ }
|
| 270 |
+
body, .gradio-container, gradio-app, .dark { background:#F6F3EE !important; color:#1A1714 !important; }
|
| 271 |
+
.gradio-container{max-width:100%!important;width:100%!important;margin:0!important;padding:0 28px 28px!important}
|
| 272 |
+
.gradio-container .main, .main.fillable, .fillable.app, main.contain{max-width:100%!important}
|
| 273 |
+
.dark input, .dark textarea{background:#FFFFFF!important;color:#1A1714!important}
|
| 274 |
+
.gradio-container .block > label > span:first-child,
|
| 275 |
+
.gradio-container .block .head > label > span:first-child,
|
| 276 |
+
.gradio-container .block > .container > span:first-child,
|
| 277 |
+
.gradio-container .block .wrap > .head span{color:#C0604A!important;background:#F1EBE3!important}
|
| 278 |
+
.gradio-container label > span{color:#C0604A}
|
| 279 |
+
button.primary, .primary{color:#FFF6F1!important}
|
| 280 |
+
.gr-button-primary, button[variant="primary"]{color:#FFF6F1!important}
|
| 281 |
+
#tib-header{position:relative;text-align:center;margin:2px 0 14px;padding:22px 0 18px;overflow:hidden}
|
| 282 |
+
#tib-header .v{position:relative;z-index:2;display:inline-block;font-family:'Outfit',system-ui,sans-serif;font-weight:600;font-size:.78rem;letter-spacing:.1em;text-transform:uppercase;color:#C0604A;background:#F1EBE3;border:1px solid #e6c3ba;border-radius:999px;padding:4px 15px;margin-bottom:12px}
|
| 283 |
+
#tib-header .t{position:relative;z-index:2;font-family:'Cormorant Garamond',Georgia,serif;font-weight:600;font-size:3.1rem;line-height:1;letter-spacing:-.01em;color:#1A1714}
|
| 284 |
+
#tib-header .t em{font-style:italic;color:#C0604A}
|
| 285 |
+
#tib-header .s{position:relative;z-index:2;font-family:'Outfit',sans-serif;font-weight:400;color:#6B6560;font-size:1.06rem;margin-top:6px;letter-spacing:.01em}
|
| 286 |
+
.tib-cube{position:absolute;z-index:1;opacity:.5;animation:tibfloat 9s ease-in-out infinite}
|
| 287 |
+
.tib-cube.c1{left:6%;top:24%;width:54px;animation-delay:0s}
|
| 288 |
+
.tib-cube.c2{left:16%;top:58%;width:38px;opacity:.35;animation-delay:1.4s}
|
| 289 |
+
.tib-cube.c3{right:8%;top:20%;width:60px;animation-delay:.6s}
|
| 290 |
+
.tib-cube.c4{right:19%;top:60%;width:40px;opacity:.35;animation-delay:2.1s}
|
| 291 |
+
.tib-cube.c5{left:38%;top:8%;width:30px;opacity:.28;animation-delay:1.1s}
|
| 292 |
+
.tib-cube.c6{right:36%;top:70%;width:32px;opacity:.3;animation-delay:.3s}
|
| 293 |
+
@keyframes tibfloat{0%,100%{transform:translateY(0) rotate(-6deg)}50%{transform:translateY(-12px) rotate(6deg)}}
|
| 294 |
+
h2,h3{font-family:'Cormorant Garamond',Georgia,serif!important;color:#1A1714}
|
| 295 |
+
.tib-edcol{display:flex!important;flex-direction:column;min-height:0}
|
| 296 |
+
.tib-edcol > *, .tib-edcol .block, .tib-edcol .html-container, .tib-edcol .prose{
|
| 297 |
+
display:flex!important;flex-direction:column;flex:1 1 auto!important;min-height:0;height:auto!important}
|
| 298 |
+
#tib-editor{flex:1 1 auto;height:100%!important;min-height:620px;display:block}
|
| 299 |
+
.tib-ctrl{align-self:stretch;flex-wrap:nowrap!important;overflow-y:auto;padding-right:8px;gap:12px}
|
| 300 |
+
.tib-ctrl .prose:empty, .tib-ctrl .md:empty, .tib-ctrl .block.padded:empty{display:none!important;min-height:0!important;padding:0!important;margin:0!important}
|
| 301 |
+
.tib-sq .image-container{aspect-ratio:1/1;min-height:190px}
|
| 302 |
+
.tib-sq .image-container img{object-fit:contain}
|
| 303 |
+
.tib-out .wrap, .tib-out .empty, .tib-out .grid-wrap, .tib-out .fixed-height, .tib-out .preview{
|
| 304 |
+
aspect-ratio:1/1!important;height:auto!important;min-height:0!important;max-height:none!important;
|
| 305 |
+
display:flex!important;align-items:center!important;justify-content:center!important;overflow:hidden}
|
| 306 |
+
.tib-out .grid-container{width:100%!important;height:100%!important;display:grid!important;grid-template-columns:1fr!important;place-items:center!important}
|
| 307 |
+
.tib-out .thumbnail-item, .tib-out .image-container{width:100%!important;height:100%!important;display:flex!important;align-items:center!important;justify-content:center!important}
|
| 308 |
+
.tib-out img{max-width:100%!important;max-height:100%!important;width:auto!important;height:auto!important;object-fit:contain!important;margin:auto!important}
|
| 309 |
+
/* accordion titles + section headings readable on cream (were invisible) */
|
| 310 |
+
.gradio-container .label-wrap, .gradio-container .label-wrap > span, .gradio-container .label-wrap span{color:#1A1714!important}
|
| 311 |
+
.gradio-container h4, .gradio-container .prose h4{color:#1A1714!important}
|
| 312 |
+
.tib-step button, button.tib-step{position:relative;padding-left:46px!important;font-weight:600!important}
|
| 313 |
+
.tib-step button::before, button.tib-step::before{
|
| 314 |
+
position:absolute;left:12px;top:50%;transform:translateY(-50%);
|
| 315 |
+
width:24px;height:24px;border-radius:50%;display:flex;align-items:center;justify-content:center;
|
| 316 |
+
font-family:'Outfit',system-ui,sans-serif;font-weight:700;font-size:.88rem;line-height:1;border:1.5px solid #C0604A}
|
| 317 |
+
.tib-step1 button::before, button.tib-step1::before{content:"1";background:#C0604A;color:#FFF6F1}
|
| 318 |
+
.tib-step2 button::before, button.tib-step2::before{content:"2";background:#FFF6F1;color:#C0604A;border-color:#FFF6F1}
|
| 319 |
+
.gradio-container input[type=checkbox]{accent-color:#C0604A;width:18px;height:18px;border:1.5px solid #C0604A!important;background:#FFFFFF!important;appearance:auto!important;-webkit-appearance:auto!important;opacity:1!important}
|
| 320 |
+
.gradio-container input[type=checkbox]:checked{background:#C0604A!important;border-color:#C0604A!important}
|
| 321 |
+
.tib-hidden{display:none!important}
|
| 322 |
+
footer{display:none!important}
|
| 323 |
+
""".replace("__VARS__", _VARS)
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
def _editor_srcdoc():
|
| 327 |
+
"""Inline box-editor.html into an iframe srcdoc (Gradio SDK serves no custom route)."""
|
| 328 |
+
html = EDITOR_HTML.read_text(encoding="utf-8")
|
| 329 |
+
esc = html.replace("&", "&").replace('"', """)
|
| 330 |
+
return ('<iframe id="tib-editor" srcdoc="' + esc + '" '
|
| 331 |
+
'style="width:100%;height:100%;min-height:620px;border:1px solid #D8D4CB;'
|
| 332 |
+
'border-radius:12px;background:#F6F3EE"></iframe>')
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
def build_demo():
|
| 336 |
+
iframe = _editor_srcdoc()
|
| 337 |
+
with gr.Blocks(title="Thinking in Boxes", theme=THEME, css=CSS) as demo:
|
| 338 |
+
cube = ('<svg class="tib-cube {c}" viewBox="0 0 100 100" fill="none" stroke="#C0604A" '
|
| 339 |
+
'stroke-width="4" stroke-linejoin="round">'
|
| 340 |
+
'<path d="M20 35 L50 20 L80 35 L50 50 Z"/>'
|
| 341 |
+
'<path d="M20 35 L20 70 L50 85 L50 50 Z"/>'
|
| 342 |
+
'<path d="M80 35 L80 70 L50 85 L50 50 Z"/></svg>')
|
| 343 |
+
cubes = "".join(cube.format(c=f"c{i}") for i in range(1, 7))
|
| 344 |
+
gr.HTML(f'<div id="tib-header">{cubes}'
|
| 345 |
+
'<div class="v">NeurIPS 2026</div>'
|
| 346 |
+
'<div class="t">Thinking in <em>Boxes</em></div>'
|
| 347 |
+
'<div class="s">3D Editing in Real Images Made Easy</div></div>')
|
| 348 |
+
|
| 349 |
+
with gr.Row(equal_height=True):
|
| 350 |
+
with gr.Column(scale=3, elem_classes="tib-edcol"):
|
| 351 |
+
gr.HTML(iframe)
|
| 352 |
+
with gr.Column(scale=2, elem_classes="tib-ctrl"):
|
| 353 |
+
grab = gr.Button("Grab image + boxes from editor", elem_classes="tib-step tib-step1")
|
| 354 |
+
with gr.Row(equal_height=True):
|
| 355 |
+
src_prev = gr.Image(label="Source boxes", elem_classes="tib-sq", show_label=True)
|
| 356 |
+
tgt_prev = gr.Image(label="Target boxes", elem_classes="tib-sq", show_label=True)
|
| 357 |
+
image_u = gr.Textbox(visible=False); src_u = gr.Textbox(visible=False)
|
| 358 |
+
tgt_u = gr.Textbox(visible=False); status_u = gr.Textbox(visible=False)
|
| 359 |
+
randomize = gr.Checkbox(value=True, label="🎲 Randomize seed")
|
| 360 |
+
gen = gr.Button("Generate", variant="primary", elem_classes="tib-step tib-step2")
|
| 361 |
+
out = gr.Gallery(label="Generated edit", columns=1, object_fit="contain",
|
| 362 |
+
show_label=True, elem_classes="tib-out")
|
| 363 |
+
seed_used = gr.Markdown()
|
| 364 |
+
|
| 365 |
+
detect_payload = gr.Textbox(elem_id="tib-detect-payload", elem_classes="tib-hidden")
|
| 366 |
+
detect_result = gr.Textbox(elem_id="tib-detect-result", elem_classes="tib-hidden")
|
| 367 |
+
detect_trigger = gr.Button("detect", elem_id="tib-detect-trigger", elem_classes="tib-hidden")
|
| 368 |
+
|
| 369 |
+
with gr.Accordion("Generation settings", open=False):
|
| 370 |
+
with gr.Row():
|
| 371 |
+
steps = gr.Slider(1, 100, value=DEFAULT_STEPS, step=1, label="Inference steps")
|
| 372 |
+
num_images = gr.Slider(1, 4, value=1, step=1, label="Images per run")
|
| 373 |
+
with gr.Row():
|
| 374 |
+
height = gr.Slider(256, 1536, value=DEFAULT_HEIGHT, step=16, label="Height")
|
| 375 |
+
width = gr.Slider(256, 1536, value=DEFAULT_WIDTH, step=16, label="Width")
|
| 376 |
+
guidance = gr.State(DEFAULT_GUIDANCE); true_cfg = gr.State(DEFAULT_TRUE_CFG)
|
| 377 |
+
lora_scale = gr.State(DEFAULT_SCALE)
|
| 378 |
+
|
| 379 |
+
grab.click(None, None, [image_u, src_u, tgt_u, status_u], js=GRAB_JS).then(
|
| 380 |
+
grab_show, [status_u, src_u, tgt_u], [src_prev, tgt_prev])
|
| 381 |
+
detect_trigger.click(detect_objects, [detect_payload], [detect_result]).then(
|
| 382 |
+
None, [detect_result], None, js=DETECT_RESULT_JS)
|
| 383 |
+
demo.load(None, None, None, js=DETECT_BRIDGE_JS)
|
| 384 |
+
gen.click(generate,
|
| 385 |
+
[image_u, src_u, tgt_u, steps, guidance, true_cfg, lora_scale,
|
| 386 |
+
randomize, height, width, num_images],
|
| 387 |
+
[out, seed_used])
|
| 388 |
+
return demo
|
| 389 |
+
|
| 390 |
+
|
| 391 |
+
demo = build_demo()
|
| 392 |
+
|
| 393 |
+
if __name__ == "__main__":
|
| 394 |
+
demo.launch(favicon_path=str(HERE / "assets" / "favicon.svg"))
|
pipelines/pipeline_flux_kontext.py
ADDED
|
@@ -0,0 +1,1277 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 Black Forest Labs and The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
import inspect
|
| 16 |
+
from typing import Any, Callable
|
| 17 |
+
|
| 18 |
+
import numpy as np
|
| 19 |
+
import torch
|
| 20 |
+
from transformers import (
|
| 21 |
+
CLIPImageProcessor,
|
| 22 |
+
CLIPTextModel,
|
| 23 |
+
CLIPTokenizer,
|
| 24 |
+
CLIPVisionModelWithProjection,
|
| 25 |
+
T5EncoderModel,
|
| 26 |
+
T5TokenizerFast,
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
from diffusers.image_processor import PipelineImageInput, VaeImageProcessor
|
| 30 |
+
from diffusers.loaders import FluxIPAdapterMixin, FluxLoraLoaderMixin, FromSingleFileMixin, TextualInversionLoaderMixin
|
| 31 |
+
from diffusers.models import AutoencoderKL, FluxTransformer2DModel
|
| 32 |
+
from diffusers.schedulers import FlowMatchEulerDiscreteScheduler
|
| 33 |
+
from diffusers.utils import (
|
| 34 |
+
USE_PEFT_BACKEND,
|
| 35 |
+
deprecate,
|
| 36 |
+
is_torch_xla_available,
|
| 37 |
+
logging,
|
| 38 |
+
replace_example_docstring,
|
| 39 |
+
scale_lora_layers,
|
| 40 |
+
unscale_lora_layers,
|
| 41 |
+
)
|
| 42 |
+
from diffusers.utils.torch_utils import randn_tensor
|
| 43 |
+
from diffusers.pipelines.pipeline_utils import DiffusionPipeline
|
| 44 |
+
from diffusers.pipelines.flux.pipeline_output import FluxPipelineOutput
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
if is_torch_xla_available():
|
| 48 |
+
import torch_xla.core.xla_model as xm
|
| 49 |
+
|
| 50 |
+
XLA_AVAILABLE = True
|
| 51 |
+
else:
|
| 52 |
+
XLA_AVAILABLE = False
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
|
| 56 |
+
|
| 57 |
+
EXAMPLE_DOC_STRING = """
|
| 58 |
+
Examples:
|
| 59 |
+
```py
|
| 60 |
+
>>> import torch
|
| 61 |
+
>>> from diffusers import FluxKontextPipeline
|
| 62 |
+
>>> from diffusers.utils import load_image
|
| 63 |
+
|
| 64 |
+
>>> pipe = FluxKontextPipeline.from_pretrained(
|
| 65 |
+
... "black-forest-labs/FLUX.1-Kontext-dev", torch_dtype=torch.bfloat16
|
| 66 |
+
... )
|
| 67 |
+
>>> pipe.to("cuda")
|
| 68 |
+
|
| 69 |
+
>>> image = load_image(
|
| 70 |
+
... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/yarn-art-pikachu.png"
|
| 71 |
+
... ).convert("RGB")
|
| 72 |
+
>>> prompt = "Make Pikachu hold a sign that says 'Black Forest Labs is awesome', yarn art style, detailed, vibrant colors"
|
| 73 |
+
>>> image = pipe(
|
| 74 |
+
... image=image,
|
| 75 |
+
... prompt=prompt,
|
| 76 |
+
... guidance_scale=2.5,
|
| 77 |
+
... generator=torch.Generator().manual_seed(42),
|
| 78 |
+
... ).images[0]
|
| 79 |
+
>>> image.save("output.png")
|
| 80 |
+
```
|
| 81 |
+
"""
|
| 82 |
+
|
| 83 |
+
PREFERRED_KONTEXT_RESOLUTIONS = [
|
| 84 |
+
(672, 1568),
|
| 85 |
+
(688, 1504),
|
| 86 |
+
(720, 1456),
|
| 87 |
+
(752, 1392),
|
| 88 |
+
(800, 1328),
|
| 89 |
+
(832, 1248),
|
| 90 |
+
(880, 1184),
|
| 91 |
+
(944, 1104),
|
| 92 |
+
(1024, 1024),
|
| 93 |
+
(1104, 944),
|
| 94 |
+
(1184, 880),
|
| 95 |
+
(1248, 832),
|
| 96 |
+
(1328, 800),
|
| 97 |
+
(1392, 752),
|
| 98 |
+
(1456, 720),
|
| 99 |
+
(1504, 688),
|
| 100 |
+
(1568, 672),
|
| 101 |
+
]
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def calculate_shift(
|
| 105 |
+
image_seq_len,
|
| 106 |
+
base_seq_len: int = 256,
|
| 107 |
+
max_seq_len: int = 4096,
|
| 108 |
+
base_shift: float = 0.5,
|
| 109 |
+
max_shift: float = 1.15,
|
| 110 |
+
):
|
| 111 |
+
m = (max_shift - base_shift) / (max_seq_len - base_seq_len)
|
| 112 |
+
b = base_shift - m * base_seq_len
|
| 113 |
+
mu = image_seq_len * m + b
|
| 114 |
+
return mu
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
|
| 118 |
+
def retrieve_timesteps(
|
| 119 |
+
scheduler,
|
| 120 |
+
num_inference_steps: int | None = None,
|
| 121 |
+
device: str | torch.device | None = None,
|
| 122 |
+
timesteps: list[int] | None = None,
|
| 123 |
+
sigmas: list[float] | None = None,
|
| 124 |
+
**kwargs,
|
| 125 |
+
):
|
| 126 |
+
r"""
|
| 127 |
+
Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
|
| 128 |
+
custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
|
| 129 |
+
|
| 130 |
+
Args:
|
| 131 |
+
scheduler (`SchedulerMixin`):
|
| 132 |
+
The scheduler to get timesteps from.
|
| 133 |
+
num_inference_steps (`int`):
|
| 134 |
+
The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
|
| 135 |
+
must be `None`.
|
| 136 |
+
device (`str` or `torch.device`, *optional*):
|
| 137 |
+
The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
|
| 138 |
+
timesteps (`list[int]`, *optional*):
|
| 139 |
+
Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
|
| 140 |
+
`num_inference_steps` and `sigmas` must be `None`.
|
| 141 |
+
sigmas (`list[float]`, *optional*):
|
| 142 |
+
Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
|
| 143 |
+
`num_inference_steps` and `timesteps` must be `None`.
|
| 144 |
+
|
| 145 |
+
Returns:
|
| 146 |
+
`tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
|
| 147 |
+
second element is the number of inference steps.
|
| 148 |
+
"""
|
| 149 |
+
if timesteps is not None and sigmas is not None:
|
| 150 |
+
raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
|
| 151 |
+
if timesteps is not None:
|
| 152 |
+
accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
|
| 153 |
+
if not accepts_timesteps:
|
| 154 |
+
raise ValueError(
|
| 155 |
+
f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
|
| 156 |
+
f" timestep schedules. Please check whether you are using the correct scheduler."
|
| 157 |
+
)
|
| 158 |
+
scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
|
| 159 |
+
timesteps = scheduler.timesteps
|
| 160 |
+
num_inference_steps = len(timesteps)
|
| 161 |
+
elif sigmas is not None:
|
| 162 |
+
accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
|
| 163 |
+
if not accept_sigmas:
|
| 164 |
+
raise ValueError(
|
| 165 |
+
f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
|
| 166 |
+
f" sigmas schedules. Please check whether you are using the correct scheduler."
|
| 167 |
+
)
|
| 168 |
+
scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
|
| 169 |
+
timesteps = scheduler.timesteps
|
| 170 |
+
num_inference_steps = len(timesteps)
|
| 171 |
+
else:
|
| 172 |
+
scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
|
| 173 |
+
timesteps = scheduler.timesteps
|
| 174 |
+
return timesteps, num_inference_steps
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
|
| 178 |
+
def retrieve_latents(
|
| 179 |
+
encoder_output: torch.Tensor, generator: torch.Generator | None = None, sample_mode: str = "sample"
|
| 180 |
+
):
|
| 181 |
+
if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
|
| 182 |
+
return encoder_output.latent_dist.sample(generator)
|
| 183 |
+
elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
|
| 184 |
+
return encoder_output.latent_dist.mode()
|
| 185 |
+
elif hasattr(encoder_output, "latents"):
|
| 186 |
+
return encoder_output.latents
|
| 187 |
+
else:
|
| 188 |
+
raise AttributeError("Could not access latents of provided encoder_output")
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
class FluxKontextPipeline(
|
| 192 |
+
DiffusionPipeline,
|
| 193 |
+
FluxLoraLoaderMixin,
|
| 194 |
+
FromSingleFileMixin,
|
| 195 |
+
TextualInversionLoaderMixin,
|
| 196 |
+
FluxIPAdapterMixin,
|
| 197 |
+
):
|
| 198 |
+
r"""
|
| 199 |
+
The Flux Kontext pipeline for image-to-image and text-to-image generation.
|
| 200 |
+
|
| 201 |
+
Reference: https://bfl.ai/announcements/flux-1-kontext-dev
|
| 202 |
+
|
| 203 |
+
Args:
|
| 204 |
+
transformer ([`FluxTransformer2DModel`]):
|
| 205 |
+
Conditional Transformer (MMDiT) architecture to denoise the encoded image latents.
|
| 206 |
+
scheduler ([`FlowMatchEulerDiscreteScheduler`]):
|
| 207 |
+
A scheduler to be used in combination with `transformer` to denoise the encoded image latents.
|
| 208 |
+
vae ([`AutoencoderKL`]):
|
| 209 |
+
Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
|
| 210 |
+
text_encoder ([`CLIPTextModel`]):
|
| 211 |
+
[CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
|
| 212 |
+
the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
|
| 213 |
+
text_encoder_2 ([`T5EncoderModel`]):
|
| 214 |
+
[T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5EncoderModel), specifically
|
| 215 |
+
the [google/t5-v1_1-xxl](https://huggingface.co/google/t5-v1_1-xxl) variant.
|
| 216 |
+
tokenizer (`CLIPTokenizer`):
|
| 217 |
+
Tokenizer of class
|
| 218 |
+
[CLIPTokenizer](https://huggingface.co/docs/transformers/en/model_doc/clip#transformers.CLIPTokenizer).
|
| 219 |
+
tokenizer_2 (`T5TokenizerFast`):
|
| 220 |
+
Second Tokenizer of class
|
| 221 |
+
[T5TokenizerFast](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5TokenizerFast).
|
| 222 |
+
"""
|
| 223 |
+
|
| 224 |
+
model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->transformer->vae"
|
| 225 |
+
_optional_components = ["image_encoder", "feature_extractor"]
|
| 226 |
+
_callback_tensor_inputs = ["latents", "prompt_embeds"]
|
| 227 |
+
|
| 228 |
+
def __init__(
|
| 229 |
+
self,
|
| 230 |
+
scheduler: FlowMatchEulerDiscreteScheduler,
|
| 231 |
+
vae: AutoencoderKL,
|
| 232 |
+
text_encoder: CLIPTextModel,
|
| 233 |
+
tokenizer: CLIPTokenizer,
|
| 234 |
+
text_encoder_2: T5EncoderModel,
|
| 235 |
+
tokenizer_2: T5TokenizerFast,
|
| 236 |
+
transformer: FluxTransformer2DModel,
|
| 237 |
+
image_encoder: CLIPVisionModelWithProjection = None,
|
| 238 |
+
feature_extractor: CLIPImageProcessor = None,
|
| 239 |
+
):
|
| 240 |
+
super().__init__()
|
| 241 |
+
|
| 242 |
+
self.register_modules(
|
| 243 |
+
vae=vae,
|
| 244 |
+
text_encoder=text_encoder,
|
| 245 |
+
text_encoder_2=text_encoder_2,
|
| 246 |
+
tokenizer=tokenizer,
|
| 247 |
+
tokenizer_2=tokenizer_2,
|
| 248 |
+
transformer=transformer,
|
| 249 |
+
scheduler=scheduler,
|
| 250 |
+
image_encoder=image_encoder,
|
| 251 |
+
feature_extractor=feature_extractor,
|
| 252 |
+
)
|
| 253 |
+
self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
|
| 254 |
+
# Flux latents are turned into 2x2 patches and packed. This means the latent width and height has to be divisible
|
| 255 |
+
# by the patch size. So the vae scale factor is multiplied by the patch size to account for this
|
| 256 |
+
self.latent_channels = self.vae.config.latent_channels if getattr(self, "vae", None) else 16
|
| 257 |
+
self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2)
|
| 258 |
+
self.tokenizer_max_length = (
|
| 259 |
+
self.tokenizer.model_max_length if hasattr(self, "tokenizer") and self.tokenizer is not None else 77
|
| 260 |
+
)
|
| 261 |
+
self.default_sample_size = 128
|
| 262 |
+
|
| 263 |
+
# Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_t5_prompt_embeds
|
| 264 |
+
def _get_t5_prompt_embeds(
|
| 265 |
+
self,
|
| 266 |
+
prompt: str | list[str] = None,
|
| 267 |
+
num_images_per_prompt: int = 1,
|
| 268 |
+
max_sequence_length: int = 512,
|
| 269 |
+
device: torch.device | None = None,
|
| 270 |
+
dtype: torch.dtype | None = None,
|
| 271 |
+
):
|
| 272 |
+
device = device or self._execution_device
|
| 273 |
+
dtype = dtype or self.text_encoder.dtype
|
| 274 |
+
|
| 275 |
+
prompt = [prompt] if isinstance(prompt, str) else prompt
|
| 276 |
+
batch_size = len(prompt)
|
| 277 |
+
|
| 278 |
+
if isinstance(self, TextualInversionLoaderMixin):
|
| 279 |
+
prompt = self.maybe_convert_prompt(prompt, self.tokenizer_2)
|
| 280 |
+
|
| 281 |
+
text_inputs = self.tokenizer_2(
|
| 282 |
+
prompt,
|
| 283 |
+
padding="max_length",
|
| 284 |
+
max_length=max_sequence_length,
|
| 285 |
+
truncation=True,
|
| 286 |
+
return_length=False,
|
| 287 |
+
return_overflowing_tokens=False,
|
| 288 |
+
return_tensors="pt",
|
| 289 |
+
)
|
| 290 |
+
text_input_ids = text_inputs.input_ids
|
| 291 |
+
untruncated_ids = self.tokenizer_2(prompt, padding="longest", return_tensors="pt").input_ids
|
| 292 |
+
|
| 293 |
+
if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids):
|
| 294 |
+
removed_text = self.tokenizer_2.batch_decode(untruncated_ids[:, self.tokenizer_max_length - 1 : -1])
|
| 295 |
+
logger.warning(
|
| 296 |
+
"The following part of your input was truncated because `max_sequence_length` is set to "
|
| 297 |
+
f" {max_sequence_length} tokens: {removed_text}"
|
| 298 |
+
)
|
| 299 |
+
|
| 300 |
+
prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0]
|
| 301 |
+
|
| 302 |
+
dtype = self.text_encoder_2.dtype
|
| 303 |
+
prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
|
| 304 |
+
|
| 305 |
+
_, seq_len, _ = prompt_embeds.shape
|
| 306 |
+
|
| 307 |
+
# duplicate text embeddings and attention mask for each generation per prompt, using mps friendly method
|
| 308 |
+
prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
|
| 309 |
+
prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
|
| 310 |
+
|
| 311 |
+
return prompt_embeds
|
| 312 |
+
|
| 313 |
+
# Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_clip_prompt_embeds
|
| 314 |
+
def _get_clip_prompt_embeds(
|
| 315 |
+
self,
|
| 316 |
+
prompt: str | list[str],
|
| 317 |
+
num_images_per_prompt: int = 1,
|
| 318 |
+
device: torch.device | None = None,
|
| 319 |
+
):
|
| 320 |
+
device = device or self._execution_device
|
| 321 |
+
|
| 322 |
+
prompt = [prompt] if isinstance(prompt, str) else prompt
|
| 323 |
+
batch_size = len(prompt)
|
| 324 |
+
|
| 325 |
+
if isinstance(self, TextualInversionLoaderMixin):
|
| 326 |
+
prompt = self.maybe_convert_prompt(prompt, self.tokenizer)
|
| 327 |
+
|
| 328 |
+
text_inputs = self.tokenizer(
|
| 329 |
+
prompt,
|
| 330 |
+
padding="max_length",
|
| 331 |
+
max_length=self.tokenizer_max_length,
|
| 332 |
+
truncation=True,
|
| 333 |
+
return_overflowing_tokens=False,
|
| 334 |
+
return_length=False,
|
| 335 |
+
return_tensors="pt",
|
| 336 |
+
)
|
| 337 |
+
|
| 338 |
+
text_input_ids = text_inputs.input_ids
|
| 339 |
+
untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
|
| 340 |
+
if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids):
|
| 341 |
+
removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer_max_length - 1 : -1])
|
| 342 |
+
logger.warning(
|
| 343 |
+
"The following part of your input was truncated because CLIP can only handle sequences up to"
|
| 344 |
+
f" {self.tokenizer_max_length} tokens: {removed_text}"
|
| 345 |
+
)
|
| 346 |
+
prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False)
|
| 347 |
+
|
| 348 |
+
# Use pooled output of CLIPTextModel
|
| 349 |
+
prompt_embeds = prompt_embeds.pooler_output
|
| 350 |
+
prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
|
| 351 |
+
|
| 352 |
+
# duplicate text embeddings for each generation per prompt, using mps friendly method
|
| 353 |
+
prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt)
|
| 354 |
+
prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, -1)
|
| 355 |
+
|
| 356 |
+
return prompt_embeds
|
| 357 |
+
|
| 358 |
+
# Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.encode_prompt
|
| 359 |
+
def encode_prompt(
|
| 360 |
+
self,
|
| 361 |
+
prompt: str | list[str],
|
| 362 |
+
prompt_2: str | list[str] | None = None,
|
| 363 |
+
device: torch.device | None = None,
|
| 364 |
+
num_images_per_prompt: int = 1,
|
| 365 |
+
prompt_embeds: torch.FloatTensor | None = None,
|
| 366 |
+
pooled_prompt_embeds: torch.FloatTensor | None = None,
|
| 367 |
+
max_sequence_length: int = 512,
|
| 368 |
+
lora_scale: float | None = None,
|
| 369 |
+
):
|
| 370 |
+
r"""
|
| 371 |
+
|
| 372 |
+
Args:
|
| 373 |
+
prompt (`str` or `list[str]`, *optional*):
|
| 374 |
+
prompt to be encoded
|
| 375 |
+
prompt_2 (`str` or `list[str]`, *optional*):
|
| 376 |
+
The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
|
| 377 |
+
used in all text-encoders
|
| 378 |
+
device: (`torch.device`):
|
| 379 |
+
torch device
|
| 380 |
+
num_images_per_prompt (`int`):
|
| 381 |
+
number of images that should be generated per prompt
|
| 382 |
+
prompt_embeds (`torch.FloatTensor`, *optional*):
|
| 383 |
+
Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
|
| 384 |
+
provided, text embeddings will be generated from `prompt` input argument.
|
| 385 |
+
pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
|
| 386 |
+
Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
|
| 387 |
+
If not provided, pooled text embeddings will be generated from `prompt` input argument.
|
| 388 |
+
lora_scale (`float`, *optional*):
|
| 389 |
+
A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
|
| 390 |
+
"""
|
| 391 |
+
device = device or self._execution_device
|
| 392 |
+
|
| 393 |
+
# set lora scale so that monkey patched LoRA
|
| 394 |
+
# function of text encoder can correctly access it
|
| 395 |
+
if lora_scale is not None and isinstance(self, FluxLoraLoaderMixin):
|
| 396 |
+
self._lora_scale = lora_scale
|
| 397 |
+
|
| 398 |
+
# dynamically adjust the LoRA scale
|
| 399 |
+
if self.text_encoder is not None and USE_PEFT_BACKEND:
|
| 400 |
+
scale_lora_layers(self.text_encoder, lora_scale)
|
| 401 |
+
if self.text_encoder_2 is not None and USE_PEFT_BACKEND:
|
| 402 |
+
scale_lora_layers(self.text_encoder_2, lora_scale)
|
| 403 |
+
|
| 404 |
+
prompt = [prompt] if isinstance(prompt, str) else prompt
|
| 405 |
+
|
| 406 |
+
if prompt_embeds is None:
|
| 407 |
+
prompt_2 = prompt_2 or prompt
|
| 408 |
+
prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2
|
| 409 |
+
|
| 410 |
+
# We only use the pooled prompt output from the CLIPTextModel
|
| 411 |
+
pooled_prompt_embeds = self._get_clip_prompt_embeds(
|
| 412 |
+
prompt=prompt,
|
| 413 |
+
device=device,
|
| 414 |
+
num_images_per_prompt=num_images_per_prompt,
|
| 415 |
+
)
|
| 416 |
+
prompt_embeds = self._get_t5_prompt_embeds(
|
| 417 |
+
prompt=prompt_2,
|
| 418 |
+
num_images_per_prompt=num_images_per_prompt,
|
| 419 |
+
max_sequence_length=max_sequence_length,
|
| 420 |
+
device=device,
|
| 421 |
+
)
|
| 422 |
+
|
| 423 |
+
if self.text_encoder is not None:
|
| 424 |
+
if isinstance(self, FluxLoraLoaderMixin) and USE_PEFT_BACKEND:
|
| 425 |
+
# Retrieve the original scale by scaling back the LoRA layers
|
| 426 |
+
unscale_lora_layers(self.text_encoder, lora_scale)
|
| 427 |
+
|
| 428 |
+
if self.text_encoder_2 is not None:
|
| 429 |
+
if isinstance(self, FluxLoraLoaderMixin) and USE_PEFT_BACKEND:
|
| 430 |
+
# Retrieve the original scale by scaling back the LoRA layers
|
| 431 |
+
unscale_lora_layers(self.text_encoder_2, lora_scale)
|
| 432 |
+
|
| 433 |
+
dtype = self.text_encoder.dtype if self.text_encoder is not None else self.transformer.dtype
|
| 434 |
+
text_ids = torch.zeros(prompt_embeds.shape[1], 3).to(device=device, dtype=dtype)
|
| 435 |
+
|
| 436 |
+
return prompt_embeds, pooled_prompt_embeds, text_ids
|
| 437 |
+
|
| 438 |
+
# Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.encode_image
|
| 439 |
+
def encode_image(self, image, device, num_images_per_prompt):
|
| 440 |
+
dtype = next(self.image_encoder.parameters()).dtype
|
| 441 |
+
|
| 442 |
+
if not isinstance(image, torch.Tensor):
|
| 443 |
+
image = self.feature_extractor(image, return_tensors="pt").pixel_values
|
| 444 |
+
|
| 445 |
+
image = image.to(device=device, dtype=dtype)
|
| 446 |
+
image_embeds = self.image_encoder(image).image_embeds
|
| 447 |
+
image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0)
|
| 448 |
+
return image_embeds
|
| 449 |
+
|
| 450 |
+
# Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.prepare_ip_adapter_image_embeds
|
| 451 |
+
def prepare_ip_adapter_image_embeds(
|
| 452 |
+
self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt
|
| 453 |
+
):
|
| 454 |
+
image_embeds = []
|
| 455 |
+
if ip_adapter_image_embeds is None:
|
| 456 |
+
if not isinstance(ip_adapter_image, list):
|
| 457 |
+
ip_adapter_image = [ip_adapter_image]
|
| 458 |
+
|
| 459 |
+
if len(ip_adapter_image) != self.transformer.encoder_hid_proj.num_ip_adapters:
|
| 460 |
+
raise ValueError(
|
| 461 |
+
f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {self.transformer.encoder_hid_proj.num_ip_adapters} IP Adapters."
|
| 462 |
+
)
|
| 463 |
+
|
| 464 |
+
for single_ip_adapter_image in ip_adapter_image:
|
| 465 |
+
single_image_embeds = self.encode_image(single_ip_adapter_image, device, 1)
|
| 466 |
+
image_embeds.append(single_image_embeds[None, :])
|
| 467 |
+
else:
|
| 468 |
+
if not isinstance(ip_adapter_image_embeds, list):
|
| 469 |
+
ip_adapter_image_embeds = [ip_adapter_image_embeds]
|
| 470 |
+
|
| 471 |
+
if len(ip_adapter_image_embeds) != self.transformer.encoder_hid_proj.num_ip_adapters:
|
| 472 |
+
raise ValueError(
|
| 473 |
+
f"`ip_adapter_image_embeds` must have same length as the number of IP Adapters. Got {len(ip_adapter_image_embeds)} image embeds and {self.transformer.encoder_hid_proj.num_ip_adapters} IP Adapters."
|
| 474 |
+
)
|
| 475 |
+
|
| 476 |
+
for single_image_embeds in ip_adapter_image_embeds:
|
| 477 |
+
image_embeds.append(single_image_embeds)
|
| 478 |
+
|
| 479 |
+
ip_adapter_image_embeds = []
|
| 480 |
+
for single_image_embeds in image_embeds:
|
| 481 |
+
single_image_embeds = torch.cat([single_image_embeds] * num_images_per_prompt, dim=0)
|
| 482 |
+
single_image_embeds = single_image_embeds.to(device=device)
|
| 483 |
+
ip_adapter_image_embeds.append(single_image_embeds)
|
| 484 |
+
|
| 485 |
+
return ip_adapter_image_embeds
|
| 486 |
+
|
| 487 |
+
# Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.check_inputs
|
| 488 |
+
def check_inputs(
|
| 489 |
+
self,
|
| 490 |
+
prompt,
|
| 491 |
+
prompt_2,
|
| 492 |
+
height,
|
| 493 |
+
width,
|
| 494 |
+
negative_prompt=None,
|
| 495 |
+
negative_prompt_2=None,
|
| 496 |
+
prompt_embeds=None,
|
| 497 |
+
negative_prompt_embeds=None,
|
| 498 |
+
pooled_prompt_embeds=None,
|
| 499 |
+
negative_pooled_prompt_embeds=None,
|
| 500 |
+
callback_on_step_end_tensor_inputs=None,
|
| 501 |
+
max_sequence_length=None,
|
| 502 |
+
):
|
| 503 |
+
if height % (self.vae_scale_factor * 2) != 0 or width % (self.vae_scale_factor * 2) != 0:
|
| 504 |
+
logger.warning(
|
| 505 |
+
f"`height` and `width` have to be divisible by {self.vae_scale_factor * 2} but are {height} and {width}. Dimensions will be resized accordingly"
|
| 506 |
+
)
|
| 507 |
+
|
| 508 |
+
if callback_on_step_end_tensor_inputs is not None and not all(
|
| 509 |
+
k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
|
| 510 |
+
):
|
| 511 |
+
raise ValueError(
|
| 512 |
+
f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
|
| 513 |
+
)
|
| 514 |
+
|
| 515 |
+
if prompt is not None and prompt_embeds is not None:
|
| 516 |
+
raise ValueError(
|
| 517 |
+
f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
|
| 518 |
+
" only forward one of the two."
|
| 519 |
+
)
|
| 520 |
+
elif prompt_2 is not None and prompt_embeds is not None:
|
| 521 |
+
raise ValueError(
|
| 522 |
+
f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
|
| 523 |
+
" only forward one of the two."
|
| 524 |
+
)
|
| 525 |
+
elif prompt is None and prompt_embeds is None:
|
| 526 |
+
raise ValueError(
|
| 527 |
+
"Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
|
| 528 |
+
)
|
| 529 |
+
elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
|
| 530 |
+
raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
|
| 531 |
+
elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)):
|
| 532 |
+
raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}")
|
| 533 |
+
|
| 534 |
+
if negative_prompt is not None and negative_prompt_embeds is not None:
|
| 535 |
+
raise ValueError(
|
| 536 |
+
f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
|
| 537 |
+
f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
|
| 538 |
+
)
|
| 539 |
+
elif negative_prompt_2 is not None and negative_prompt_embeds is not None:
|
| 540 |
+
raise ValueError(
|
| 541 |
+
f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:"
|
| 542 |
+
f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
|
| 543 |
+
)
|
| 544 |
+
|
| 545 |
+
if prompt_embeds is not None and pooled_prompt_embeds is None:
|
| 546 |
+
raise ValueError(
|
| 547 |
+
"If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`."
|
| 548 |
+
)
|
| 549 |
+
if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None:
|
| 550 |
+
raise ValueError(
|
| 551 |
+
"If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`."
|
| 552 |
+
)
|
| 553 |
+
|
| 554 |
+
if max_sequence_length is not None and max_sequence_length > 512:
|
| 555 |
+
raise ValueError(f"`max_sequence_length` cannot be greater than 512 but is {max_sequence_length}")
|
| 556 |
+
|
| 557 |
+
@staticmethod
|
| 558 |
+
# Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._prepare_latent_image_ids
|
| 559 |
+
def _prepare_latent_image_ids(batch_size, height, width, device, dtype):
|
| 560 |
+
latent_image_ids = torch.zeros(height, width, 3)
|
| 561 |
+
latent_image_ids[..., 1] = latent_image_ids[..., 1] + torch.arange(height)[:, None]
|
| 562 |
+
latent_image_ids[..., 2] = latent_image_ids[..., 2] + torch.arange(width)[None, :]
|
| 563 |
+
|
| 564 |
+
latent_image_id_height, latent_image_id_width, latent_image_id_channels = latent_image_ids.shape
|
| 565 |
+
|
| 566 |
+
latent_image_ids = latent_image_ids.reshape(
|
| 567 |
+
latent_image_id_height * latent_image_id_width, latent_image_id_channels
|
| 568 |
+
)
|
| 569 |
+
|
| 570 |
+
return latent_image_ids.to(device=device, dtype=dtype)
|
| 571 |
+
|
| 572 |
+
@staticmethod
|
| 573 |
+
# Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._pack_latents
|
| 574 |
+
def _pack_latents(latents, batch_size, num_channels_latents, height, width):
|
| 575 |
+
latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2)
|
| 576 |
+
latents = latents.permute(0, 2, 4, 1, 3, 5)
|
| 577 |
+
latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4)
|
| 578 |
+
|
| 579 |
+
return latents
|
| 580 |
+
|
| 581 |
+
@staticmethod
|
| 582 |
+
# Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._unpack_latents
|
| 583 |
+
def _unpack_latents(latents, height, width, vae_scale_factor):
|
| 584 |
+
batch_size, num_patches, channels = latents.shape
|
| 585 |
+
|
| 586 |
+
# VAE applies 8x compression on images but we must also account for packing which requires
|
| 587 |
+
# latent height and width to be divisible by 2.
|
| 588 |
+
height = 2 * (int(height) // (vae_scale_factor * 2))
|
| 589 |
+
width = 2 * (int(width) // (vae_scale_factor * 2))
|
| 590 |
+
|
| 591 |
+
latents = latents.view(batch_size, height // 2, width // 2, channels // 4, 2, 2)
|
| 592 |
+
latents = latents.permute(0, 3, 1, 4, 2, 5)
|
| 593 |
+
|
| 594 |
+
latents = latents.reshape(batch_size, channels // (2 * 2), height, width)
|
| 595 |
+
|
| 596 |
+
return latents
|
| 597 |
+
|
| 598 |
+
def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator):
|
| 599 |
+
if isinstance(generator, list):
|
| 600 |
+
image_latents = [
|
| 601 |
+
retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i], sample_mode="argmax")
|
| 602 |
+
for i in range(image.shape[0])
|
| 603 |
+
]
|
| 604 |
+
image_latents = torch.cat(image_latents, dim=0)
|
| 605 |
+
else:
|
| 606 |
+
image_latents = retrieve_latents(self.vae.encode(image), generator=generator, sample_mode="argmax")
|
| 607 |
+
|
| 608 |
+
image_latents = (image_latents - self.vae.config.shift_factor) * self.vae.config.scaling_factor
|
| 609 |
+
|
| 610 |
+
return image_latents
|
| 611 |
+
|
| 612 |
+
# Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.enable_vae_slicing
|
| 613 |
+
def enable_vae_slicing(self):
|
| 614 |
+
r"""
|
| 615 |
+
Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to
|
| 616 |
+
compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.
|
| 617 |
+
"""
|
| 618 |
+
depr_message = f"Calling `enable_vae_slicing()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.enable_slicing()`."
|
| 619 |
+
deprecate(
|
| 620 |
+
"enable_vae_slicing",
|
| 621 |
+
"0.40.0",
|
| 622 |
+
depr_message,
|
| 623 |
+
)
|
| 624 |
+
self.vae.enable_slicing()
|
| 625 |
+
|
| 626 |
+
# Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.disable_vae_slicing
|
| 627 |
+
def disable_vae_slicing(self):
|
| 628 |
+
r"""
|
| 629 |
+
Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to
|
| 630 |
+
computing decoding in one step.
|
| 631 |
+
"""
|
| 632 |
+
depr_message = f"Calling `disable_vae_slicing()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.disable_slicing()`."
|
| 633 |
+
deprecate(
|
| 634 |
+
"disable_vae_slicing",
|
| 635 |
+
"0.40.0",
|
| 636 |
+
depr_message,
|
| 637 |
+
)
|
| 638 |
+
self.vae.disable_slicing()
|
| 639 |
+
|
| 640 |
+
# Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.enable_vae_tiling
|
| 641 |
+
def enable_vae_tiling(self):
|
| 642 |
+
r"""
|
| 643 |
+
Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to
|
| 644 |
+
compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow
|
| 645 |
+
processing larger images.
|
| 646 |
+
"""
|
| 647 |
+
depr_message = f"Calling `enable_vae_tiling()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.enable_tiling()`."
|
| 648 |
+
deprecate(
|
| 649 |
+
"enable_vae_tiling",
|
| 650 |
+
"0.40.0",
|
| 651 |
+
depr_message,
|
| 652 |
+
)
|
| 653 |
+
self.vae.enable_tiling()
|
| 654 |
+
|
| 655 |
+
# Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.disable_vae_tiling
|
| 656 |
+
def disable_vae_tiling(self):
|
| 657 |
+
r"""
|
| 658 |
+
Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to
|
| 659 |
+
computing decoding in one step.
|
| 660 |
+
"""
|
| 661 |
+
depr_message = f"Calling `disable_vae_tiling()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.disable_tiling()`."
|
| 662 |
+
deprecate(
|
| 663 |
+
"disable_vae_tiling",
|
| 664 |
+
"0.40.0",
|
| 665 |
+
depr_message,
|
| 666 |
+
)
|
| 667 |
+
self.vae.disable_tiling()
|
| 668 |
+
|
| 669 |
+
def prepare_latents(
|
| 670 |
+
self,
|
| 671 |
+
image: torch.Tensor | None,
|
| 672 |
+
##########
|
| 673 |
+
source_primitive: torch.Tensor | None,
|
| 674 |
+
target_primitive: torch.Tensor | None,
|
| 675 |
+
##########
|
| 676 |
+
batch_size: int,
|
| 677 |
+
num_channels_latents: int,
|
| 678 |
+
height: int,
|
| 679 |
+
width: int,
|
| 680 |
+
dtype: torch.dtype,
|
| 681 |
+
device: torch.device,
|
| 682 |
+
generator: torch.Generator | list[torch.Generator] | None = None,
|
| 683 |
+
latents: torch.Tensor | None = None,
|
| 684 |
+
):
|
| 685 |
+
if isinstance(generator, list) and len(generator) != batch_size:
|
| 686 |
+
raise ValueError(
|
| 687 |
+
f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
|
| 688 |
+
f" size of {batch_size}. Make sure the batch size matches the length of the generators."
|
| 689 |
+
)
|
| 690 |
+
|
| 691 |
+
# VAE applies 8x compression on images but we must also account for packing which requires
|
| 692 |
+
# latent height and width to be divisible by 2.
|
| 693 |
+
height = 2 * (int(height) // (self.vae_scale_factor * 2))
|
| 694 |
+
width = 2 * (int(width) // (self.vae_scale_factor * 2))
|
| 695 |
+
shape = (batch_size, num_channels_latents, height, width)
|
| 696 |
+
|
| 697 |
+
image_latents = image_ids = None
|
| 698 |
+
if image is not None:
|
| 699 |
+
image = image.to(device=device, dtype=dtype)
|
| 700 |
+
if image.shape[1] != self.latent_channels:
|
| 701 |
+
image_latents = self._encode_vae_image(image=image, generator=generator)
|
| 702 |
+
else:
|
| 703 |
+
image_latents = image
|
| 704 |
+
if batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] == 0:
|
| 705 |
+
# expand init_latents for batch_size
|
| 706 |
+
additional_image_per_prompt = batch_size // image_latents.shape[0]
|
| 707 |
+
image_latents = torch.cat([image_latents] * additional_image_per_prompt, dim=0)
|
| 708 |
+
elif batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] != 0:
|
| 709 |
+
raise ValueError(
|
| 710 |
+
f"Cannot duplicate `image` of batch size {image_latents.shape[0]} to {batch_size} text prompts."
|
| 711 |
+
)
|
| 712 |
+
else:
|
| 713 |
+
image_latents = torch.cat([image_latents], dim=0)
|
| 714 |
+
|
| 715 |
+
image_latent_height, image_latent_width = image_latents.shape[2:]
|
| 716 |
+
image_latents = self._pack_latents(
|
| 717 |
+
image_latents, batch_size, num_channels_latents, image_latent_height, image_latent_width
|
| 718 |
+
)
|
| 719 |
+
image_ids = self._prepare_latent_image_ids(
|
| 720 |
+
batch_size, image_latent_height // 2, image_latent_width // 2, device, dtype
|
| 721 |
+
)
|
| 722 |
+
# image ids are the same as latent ids with the first dimension set to 1 instead of 0
|
| 723 |
+
image_ids[..., 0] = 1
|
| 724 |
+
|
| 725 |
+
##########
|
| 726 |
+
source_primitive_latents = source_primitive_ids = None
|
| 727 |
+
if source_primitive is not None:
|
| 728 |
+
source_primitive = source_primitive.to(device=device, dtype=dtype)
|
| 729 |
+
if source_primitive.shape[1] != self.latent_channels:
|
| 730 |
+
source_primitive_latents = self._encode_vae_image(image=source_primitive, generator=generator)
|
| 731 |
+
else:
|
| 732 |
+
source_primitive_latents = source_primitive
|
| 733 |
+
if batch_size > source_primitive_latents.shape[0] and batch_size % source_primitive_latents.shape[0] == 0:
|
| 734 |
+
# expand init_latents for batch_size
|
| 735 |
+
additional_source_primitive_per_prompt = batch_size // source_primitive_latents.shape[0]
|
| 736 |
+
source_primitive_latents = torch.cat([source_primitive_latents] * additional_source_primitive_per_prompt, dim=0)
|
| 737 |
+
elif batch_size > source_primitive_latents.shape[0] and batch_size % source_primitive_latents.shape[0] != 0:
|
| 738 |
+
raise ValueError(
|
| 739 |
+
f"Cannot duplicate `source_primitive` of batch size {source_primitive_latents.shape[0]} to {batch_size} text prompts."
|
| 740 |
+
)
|
| 741 |
+
else:
|
| 742 |
+
source_primitive_latents = torch.cat([source_primitive_latents], dim=0)
|
| 743 |
+
|
| 744 |
+
source_primitive_latent_height, source_primitive_latent_width = source_primitive_latents.shape[2:]
|
| 745 |
+
source_primitive_latents = self._pack_latents(
|
| 746 |
+
source_primitive_latents, batch_size, num_channels_latents, source_primitive_latent_height, source_primitive_latent_width
|
| 747 |
+
)
|
| 748 |
+
source_primitive_ids = self._prepare_latent_image_ids(
|
| 749 |
+
batch_size, source_primitive_latent_height // 2, source_primitive_latent_width // 2, device, dtype
|
| 750 |
+
)
|
| 751 |
+
# image ids are the same as latent ids with the first dimension set to 2 instead of 0
|
| 752 |
+
source_primitive_ids[..., 0] = 2
|
| 753 |
+
|
| 754 |
+
target_primitive_latents = target_primitive_ids = None
|
| 755 |
+
if target_primitive is not None:
|
| 756 |
+
target_primitive = target_primitive.to(device=device, dtype=dtype)
|
| 757 |
+
if target_primitive.shape[1] != self.latent_channels:
|
| 758 |
+
target_primitive_latents = self._encode_vae_image(image=target_primitive, generator=generator)
|
| 759 |
+
else:
|
| 760 |
+
target_primitive_latents = target_primitive
|
| 761 |
+
if batch_size > target_primitive_latents.shape[0] and batch_size % target_primitive_latents.shape[0] == 0:
|
| 762 |
+
# expand init_latents for batch_size
|
| 763 |
+
additional_target_primitive_per_prompt = batch_size // target_primitive_latents.shape[0]
|
| 764 |
+
target_primitive_latents = torch.cat([target_primitive_latents] * additional_target_primitive_per_prompt, dim=0)
|
| 765 |
+
elif batch_size > target_primitive_latents.shape[0] and batch_size % target_primitive_latents.shape[0] != 0:
|
| 766 |
+
raise ValueError(
|
| 767 |
+
f"Cannot duplicate `target_primitive` of batch size {target_primitive_latents.shape[0]} to {batch_size} text prompts."
|
| 768 |
+
)
|
| 769 |
+
else:
|
| 770 |
+
target_primitive_latents = torch.cat([target_primitive_latents], dim=0)
|
| 771 |
+
|
| 772 |
+
target_primitive_latent_height, target_primitive_latent_width = target_primitive_latents.shape[2:]
|
| 773 |
+
target_primitive_latents = self._pack_latents(
|
| 774 |
+
target_primitive_latents, batch_size, num_channels_latents, target_primitive_latent_height, target_primitive_latent_width
|
| 775 |
+
)
|
| 776 |
+
target_primitive_ids = self._prepare_latent_image_ids(
|
| 777 |
+
batch_size, target_primitive_latent_height // 2, target_primitive_latent_width // 2, device, dtype
|
| 778 |
+
)
|
| 779 |
+
# image ids are the same as latent ids with the first dimension set to 3 instead of 0
|
| 780 |
+
target_primitive_ids[..., 0] = 3
|
| 781 |
+
|
| 782 |
+
latent_ids = self._prepare_latent_image_ids(batch_size, height // 2, width // 2, device, dtype)
|
| 783 |
+
|
| 784 |
+
if latents is None:
|
| 785 |
+
latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
|
| 786 |
+
latents = self._pack_latents(latents, batch_size, num_channels_latents, height, width)
|
| 787 |
+
else:
|
| 788 |
+
latents = latents.to(device=device, dtype=dtype)
|
| 789 |
+
|
| 790 |
+
return latents, image_latents, source_primitive_latents, target_primitive_latents, latent_ids, image_ids, source_primitive_ids, target_primitive_ids
|
| 791 |
+
##########
|
| 792 |
+
|
| 793 |
+
@property
|
| 794 |
+
def guidance_scale(self):
|
| 795 |
+
return self._guidance_scale
|
| 796 |
+
|
| 797 |
+
@property
|
| 798 |
+
def joint_attention_kwargs(self):
|
| 799 |
+
return self._joint_attention_kwargs
|
| 800 |
+
|
| 801 |
+
@property
|
| 802 |
+
def num_timesteps(self):
|
| 803 |
+
return self._num_timesteps
|
| 804 |
+
|
| 805 |
+
@property
|
| 806 |
+
def current_timestep(self):
|
| 807 |
+
return self._current_timestep
|
| 808 |
+
|
| 809 |
+
@property
|
| 810 |
+
def interrupt(self):
|
| 811 |
+
return self._interrupt
|
| 812 |
+
|
| 813 |
+
@torch.no_grad()
|
| 814 |
+
@replace_example_docstring(EXAMPLE_DOC_STRING)
|
| 815 |
+
def __call__(
|
| 816 |
+
self,
|
| 817 |
+
image: PipelineImageInput | None = None,
|
| 818 |
+
##########
|
| 819 |
+
source_primitive: PipelineImageInput | None = None,
|
| 820 |
+
target_primitive: PipelineImageInput | None = None,
|
| 821 |
+
##########
|
| 822 |
+
prompt: str | list[str] = None,
|
| 823 |
+
prompt_2: str | list[str] | None = None,
|
| 824 |
+
negative_prompt: str | list[str] = None,
|
| 825 |
+
negative_prompt_2: str | list[str] | None = None,
|
| 826 |
+
true_cfg_scale: float = 1.0,
|
| 827 |
+
height: int | None = None,
|
| 828 |
+
width: int | None = None,
|
| 829 |
+
num_inference_steps: int = 28,
|
| 830 |
+
sigmas: list[float] | None = None,
|
| 831 |
+
guidance_scale: float = 3.5,
|
| 832 |
+
num_images_per_prompt: int | None = 1,
|
| 833 |
+
generator: torch.Generator | list[torch.Generator] | None = None,
|
| 834 |
+
latents: torch.FloatTensor | None = None,
|
| 835 |
+
prompt_embeds: torch.FloatTensor | None = None,
|
| 836 |
+
pooled_prompt_embeds: torch.FloatTensor | None = None,
|
| 837 |
+
ip_adapter_image: PipelineImageInput | None = None,
|
| 838 |
+
ip_adapter_image_embeds: list[torch.Tensor] | None = None,
|
| 839 |
+
negative_ip_adapter_image: PipelineImageInput | None = None,
|
| 840 |
+
negative_ip_adapter_image_embeds: list[torch.Tensor] | None = None,
|
| 841 |
+
negative_prompt_embeds: torch.FloatTensor | None = None,
|
| 842 |
+
negative_pooled_prompt_embeds: torch.FloatTensor | None = None,
|
| 843 |
+
output_type: str | None = "pil",
|
| 844 |
+
return_dict: bool = True,
|
| 845 |
+
joint_attention_kwargs: dict[str, Any] | None = None,
|
| 846 |
+
callback_on_step_end: Callable[[int, int], None] | None = None,
|
| 847 |
+
callback_on_step_end_tensor_inputs: list[str] = ["latents"],
|
| 848 |
+
max_sequence_length: int = 512,
|
| 849 |
+
max_area: int = 1024**2,
|
| 850 |
+
_auto_resize: bool = False,
|
| 851 |
+
):
|
| 852 |
+
r"""
|
| 853 |
+
Function invoked when calling the pipeline for generation.
|
| 854 |
+
|
| 855 |
+
Args:
|
| 856 |
+
image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`):
|
| 857 |
+
`Image`, numpy array or tensor representing an image batch to be used as the starting point. For both
|
| 858 |
+
numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list
|
| 859 |
+
or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a
|
| 860 |
+
list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image
|
| 861 |
+
latents as `image`, but if passing latents directly it is not encoded again.
|
| 862 |
+
##########
|
| 863 |
+
source_primitive (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`):
|
| 864 |
+
`Image`, numpy array or tensor representing the primitive for the image to be used as the starting point. For both
|
| 865 |
+
numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list
|
| 866 |
+
or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a
|
| 867 |
+
list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image
|
| 868 |
+
latents as `image`, but if passing latents directly it is not encoded again.
|
| 869 |
+
target_primitive (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`):
|
| 870 |
+
`Image`, numpy array or tensor representing the primitive for the edited image to be obtained. For both
|
| 871 |
+
numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list
|
| 872 |
+
or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a
|
| 873 |
+
list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image
|
| 874 |
+
latents as `image`, but if passing latents directly it is not encoded again.
|
| 875 |
+
##########
|
| 876 |
+
prompt (`str` or `list[str]`, *optional*):
|
| 877 |
+
The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
|
| 878 |
+
instead.
|
| 879 |
+
prompt_2 (`str` or `list[str]`, *optional*):
|
| 880 |
+
The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
|
| 881 |
+
will be used instead.
|
| 882 |
+
negative_prompt (`str` or `list[str]`, *optional*):
|
| 883 |
+
The prompt or prompts not to guide the image generation. If not defined, one has to pass
|
| 884 |
+
`negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is
|
| 885 |
+
not greater than `1`).
|
| 886 |
+
negative_prompt_2 (`str` or `list[str]`, *optional*):
|
| 887 |
+
The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
|
| 888 |
+
`text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders.
|
| 889 |
+
true_cfg_scale (`float`, *optional*, defaults to 1.0):
|
| 890 |
+
When > 1.0 and a provided `negative_prompt`, enables true classifier-free guidance.
|
| 891 |
+
height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
|
| 892 |
+
The height in pixels of the generated image. This is set to 1024 by default for the best results.
|
| 893 |
+
width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
|
| 894 |
+
The width in pixels of the generated image. This is set to 1024 by default for the best results.
|
| 895 |
+
num_inference_steps (`int`, *optional*, defaults to 50):
|
| 896 |
+
The number of denoising steps. More denoising steps usually lead to a higher quality image at the
|
| 897 |
+
expense of slower inference.
|
| 898 |
+
sigmas (`list[float]`, *optional*):
|
| 899 |
+
Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
|
| 900 |
+
their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
|
| 901 |
+
will be used.
|
| 902 |
+
guidance_scale (`float`, *optional*, defaults to 3.5):
|
| 903 |
+
Embedded guidance scale is enabled by setting `guidance_scale` > 1. Higher `guidance_scale` encourages
|
| 904 |
+
a model to generate images more aligned with prompt at the expense of lower image quality.
|
| 905 |
+
|
| 906 |
+
Guidance-distilled models approximates true classifier-free guidance for `guidance_scale` > 1. Refer to
|
| 907 |
+
the [paper](https://huggingface.co/papers/2210.03142) to learn more.
|
| 908 |
+
num_images_per_prompt (`int`, *optional*, defaults to 1):
|
| 909 |
+
The number of images to generate per prompt.
|
| 910 |
+
generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
|
| 911 |
+
One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
|
| 912 |
+
to make generation deterministic.
|
| 913 |
+
latents (`torch.FloatTensor`, *optional*):
|
| 914 |
+
Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
|
| 915 |
+
generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
|
| 916 |
+
tensor will be generated by sampling using the supplied random `generator`.
|
| 917 |
+
prompt_embeds (`torch.FloatTensor`, *optional*):
|
| 918 |
+
Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
|
| 919 |
+
provided, text embeddings will be generated from `prompt` input argument.
|
| 920 |
+
pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
|
| 921 |
+
Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
|
| 922 |
+
If not provided, pooled text embeddings will be generated from `prompt` input argument.
|
| 923 |
+
ip_adapter_image: (`PipelineImageInput`, *optional*):
|
| 924 |
+
Optional image input to work with IP Adapters.
|
| 925 |
+
ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*):
|
| 926 |
+
Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of
|
| 927 |
+
IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not
|
| 928 |
+
provided, embeddings are computed from the `ip_adapter_image` input argument.
|
| 929 |
+
negative_ip_adapter_image:
|
| 930 |
+
(`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.
|
| 931 |
+
negative_ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*):
|
| 932 |
+
Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of
|
| 933 |
+
IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not
|
| 934 |
+
provided, embeddings are computed from the `ip_adapter_image` input argument.
|
| 935 |
+
negative_prompt_embeds (`torch.FloatTensor`, *optional*):
|
| 936 |
+
Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
|
| 937 |
+
weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
|
| 938 |
+
argument.
|
| 939 |
+
negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
|
| 940 |
+
Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
|
| 941 |
+
weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
|
| 942 |
+
input argument.
|
| 943 |
+
output_type (`str`, *optional*, defaults to `"pil"`):
|
| 944 |
+
The output format of the generate image. Choose between
|
| 945 |
+
[PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
|
| 946 |
+
return_dict (`bool`, *optional*, defaults to `True`):
|
| 947 |
+
Whether or not to return a [`~pipelines.flux.FluxPipelineOutput`] instead of a plain tuple.
|
| 948 |
+
joint_attention_kwargs (`dict`, *optional*):
|
| 949 |
+
A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
|
| 950 |
+
`self.processor` in
|
| 951 |
+
[diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
|
| 952 |
+
callback_on_step_end (`Callable`, *optional*):
|
| 953 |
+
A function that calls at the end of each denoising steps during the inference. The function is called
|
| 954 |
+
with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
|
| 955 |
+
callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
|
| 956 |
+
`callback_on_step_end_tensor_inputs`.
|
| 957 |
+
callback_on_step_end_tensor_inputs (`list`, *optional*):
|
| 958 |
+
The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
|
| 959 |
+
will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
|
| 960 |
+
`._callback_tensor_inputs` attribute of your pipeline class.
|
| 961 |
+
max_sequence_length (`int` defaults to 512):
|
| 962 |
+
Maximum sequence length to use with the `prompt`.
|
| 963 |
+
max_area (`int`, defaults to `1024 ** 2`):
|
| 964 |
+
The maximum area of the generated image in pixels. The height and width will be adjusted to fit this
|
| 965 |
+
area while maintaining the aspect ratio.
|
| 966 |
+
|
| 967 |
+
Examples:
|
| 968 |
+
|
| 969 |
+
Returns:
|
| 970 |
+
[`~pipelines.flux.FluxPipelineOutput`] or `tuple`: [`~pipelines.flux.FluxPipelineOutput`] if `return_dict`
|
| 971 |
+
is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the generated
|
| 972 |
+
images.
|
| 973 |
+
"""
|
| 974 |
+
|
| 975 |
+
height = height or self.default_sample_size * self.vae_scale_factor
|
| 976 |
+
width = width or self.default_sample_size * self.vae_scale_factor
|
| 977 |
+
|
| 978 |
+
original_height, original_width = height, width
|
| 979 |
+
aspect_ratio = width / height
|
| 980 |
+
width = round((max_area * aspect_ratio) ** 0.5)
|
| 981 |
+
height = round((max_area / aspect_ratio) ** 0.5)
|
| 982 |
+
|
| 983 |
+
multiple_of = self.vae_scale_factor * 2
|
| 984 |
+
width = width // multiple_of * multiple_of
|
| 985 |
+
height = height // multiple_of * multiple_of
|
| 986 |
+
|
| 987 |
+
if height != original_height or width != original_width:
|
| 988 |
+
logger.warning(
|
| 989 |
+
f"Generation `height` and `width` have been adjusted to {height} and {width} to fit the model requirements."
|
| 990 |
+
)
|
| 991 |
+
|
| 992 |
+
# 1. Check inputs. Raise error if not correct
|
| 993 |
+
self.check_inputs(
|
| 994 |
+
prompt,
|
| 995 |
+
prompt_2,
|
| 996 |
+
height,
|
| 997 |
+
width,
|
| 998 |
+
negative_prompt=negative_prompt,
|
| 999 |
+
negative_prompt_2=negative_prompt_2,
|
| 1000 |
+
prompt_embeds=prompt_embeds,
|
| 1001 |
+
negative_prompt_embeds=negative_prompt_embeds,
|
| 1002 |
+
pooled_prompt_embeds=pooled_prompt_embeds,
|
| 1003 |
+
negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
|
| 1004 |
+
callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs,
|
| 1005 |
+
max_sequence_length=max_sequence_length,
|
| 1006 |
+
)
|
| 1007 |
+
|
| 1008 |
+
self._guidance_scale = guidance_scale
|
| 1009 |
+
self._joint_attention_kwargs = joint_attention_kwargs
|
| 1010 |
+
self._current_timestep = None
|
| 1011 |
+
self._interrupt = False
|
| 1012 |
+
|
| 1013 |
+
# 2. Define call parameters
|
| 1014 |
+
if prompt is not None and isinstance(prompt, str):
|
| 1015 |
+
batch_size = 1
|
| 1016 |
+
elif prompt is not None and isinstance(prompt, list):
|
| 1017 |
+
batch_size = len(prompt)
|
| 1018 |
+
else:
|
| 1019 |
+
batch_size = prompt_embeds.shape[0]
|
| 1020 |
+
|
| 1021 |
+
device = self._execution_device
|
| 1022 |
+
|
| 1023 |
+
lora_scale = (
|
| 1024 |
+
self.joint_attention_kwargs.get("scale", None) if self.joint_attention_kwargs is not None else None
|
| 1025 |
+
)
|
| 1026 |
+
has_neg_prompt = negative_prompt is not None or (
|
| 1027 |
+
negative_prompt_embeds is not None and negative_pooled_prompt_embeds is not None
|
| 1028 |
+
)
|
| 1029 |
+
do_true_cfg = true_cfg_scale > 1 and has_neg_prompt
|
| 1030 |
+
(
|
| 1031 |
+
prompt_embeds,
|
| 1032 |
+
pooled_prompt_embeds,
|
| 1033 |
+
text_ids,
|
| 1034 |
+
) = self.encode_prompt(
|
| 1035 |
+
prompt=prompt,
|
| 1036 |
+
prompt_2=prompt_2,
|
| 1037 |
+
prompt_embeds=prompt_embeds,
|
| 1038 |
+
pooled_prompt_embeds=pooled_prompt_embeds,
|
| 1039 |
+
device=device,
|
| 1040 |
+
num_images_per_prompt=num_images_per_prompt,
|
| 1041 |
+
max_sequence_length=max_sequence_length,
|
| 1042 |
+
lora_scale=lora_scale,
|
| 1043 |
+
)
|
| 1044 |
+
if do_true_cfg:
|
| 1045 |
+
(
|
| 1046 |
+
negative_prompt_embeds,
|
| 1047 |
+
negative_pooled_prompt_embeds,
|
| 1048 |
+
negative_text_ids,
|
| 1049 |
+
) = self.encode_prompt(
|
| 1050 |
+
prompt=negative_prompt,
|
| 1051 |
+
prompt_2=negative_prompt_2,
|
| 1052 |
+
prompt_embeds=negative_prompt_embeds,
|
| 1053 |
+
pooled_prompt_embeds=negative_pooled_prompt_embeds,
|
| 1054 |
+
device=device,
|
| 1055 |
+
num_images_per_prompt=num_images_per_prompt,
|
| 1056 |
+
max_sequence_length=max_sequence_length,
|
| 1057 |
+
lora_scale=lora_scale,
|
| 1058 |
+
)
|
| 1059 |
+
|
| 1060 |
+
# 3a. Preprocess image
|
| 1061 |
+
if image is not None and not (isinstance(image, torch.Tensor) and image.size(1) == self.latent_channels):
|
| 1062 |
+
img = image[0] if isinstance(image, list) else image
|
| 1063 |
+
image_height, image_width = self.image_processor.get_default_height_width(img)
|
| 1064 |
+
aspect_ratio = image_width / image_height
|
| 1065 |
+
if _auto_resize:
|
| 1066 |
+
# Kontext is trained on specific resolutions, using one of them is recommended
|
| 1067 |
+
_, image_width, image_height = min(
|
| 1068 |
+
(abs(aspect_ratio - w / h), w, h) for w, h in PREFERRED_KONTEXT_RESOLUTIONS
|
| 1069 |
+
)
|
| 1070 |
+
image_width = image_width // multiple_of * multiple_of
|
| 1071 |
+
image_height = image_height // multiple_of * multiple_of
|
| 1072 |
+
image = self.image_processor.resize(image, image_height, image_width)
|
| 1073 |
+
image = self.image_processor.preprocess(image, image_height, image_width)
|
| 1074 |
+
|
| 1075 |
+
##########
|
| 1076 |
+
# 3b. Preprocess Source Primitive
|
| 1077 |
+
if source_primitive is not None and not (isinstance(source_primitive, torch.Tensor) and source_primitive.size(1) == self.latent_channels):
|
| 1078 |
+
img = source_primitive[0] if isinstance(source_primitive, list) else source_primitive
|
| 1079 |
+
source_primitive_height, source_primitive_width = self.image_processor.get_default_height_width(img)
|
| 1080 |
+
aspect_ratio = source_primitive_width / source_primitive_height
|
| 1081 |
+
if _auto_resize:
|
| 1082 |
+
# Kontext is trained on specific resolutions, using one of them is recommended
|
| 1083 |
+
_, source_primitive_width, source_primitive_height = min(
|
| 1084 |
+
(abs(aspect_ratio - w / h), w, h) for w, h in PREFERRED_KONTEXT_RESOLUTIONS
|
| 1085 |
+
)
|
| 1086 |
+
source_primitive_width = source_primitive_width // multiple_of * multiple_of
|
| 1087 |
+
source_primitive_height = source_primitive_height // multiple_of * multiple_of
|
| 1088 |
+
source_primitive = self.image_processor.resize(source_primitive, source_primitive_height, source_primitive_width)
|
| 1089 |
+
source_primitive = self.image_processor.preprocess(source_primitive, source_primitive_height, source_primitive_width)
|
| 1090 |
+
|
| 1091 |
+
# 3c. Preprocess Target Primitive
|
| 1092 |
+
if target_primitive is not None and not (isinstance(target_primitive, torch.Tensor) and target_primitive.size(1) == self.latent_channels):
|
| 1093 |
+
img = target_primitive[0] if isinstance(target_primitive, list) else target_primitive
|
| 1094 |
+
target_primitive_height, target_primitive_width = self.image_processor.get_default_height_width(img)
|
| 1095 |
+
aspect_ratio = target_primitive_width / target_primitive_height
|
| 1096 |
+
if _auto_resize:
|
| 1097 |
+
# Kontext is trained on specific resolutions, using one of them is recommended
|
| 1098 |
+
_, target_primitive_width, target_primitive_height = min(
|
| 1099 |
+
(abs(aspect_ratio - w / h), w, h) for w, h in PREFERRED_KONTEXT_RESOLUTIONS
|
| 1100 |
+
)
|
| 1101 |
+
target_primitive_width = target_primitive_width // multiple_of * multiple_of
|
| 1102 |
+
target_primitive_height = target_primitive_height // multiple_of * multiple_of
|
| 1103 |
+
target_primitive = self.image_processor.resize(target_primitive, target_primitive_height, target_primitive_width)
|
| 1104 |
+
target_primitive = self.image_processor.preprocess(target_primitive, target_primitive_height, target_primitive_width)
|
| 1105 |
+
##########
|
| 1106 |
+
|
| 1107 |
+
# 4. Prepare latent variables
|
| 1108 |
+
num_channels_latents = self.transformer.config.in_channels // 4
|
| 1109 |
+
##########
|
| 1110 |
+
latents, image_latents, source_primitive_latents, target_primitive_latents, latent_ids, image_ids, source_primitive_ids, target_primitive_ids = self.prepare_latents(
|
| 1111 |
+
image,
|
| 1112 |
+
source_primitive,
|
| 1113 |
+
target_primitive,
|
| 1114 |
+
batch_size * num_images_per_prompt,
|
| 1115 |
+
num_channels_latents,
|
| 1116 |
+
height,
|
| 1117 |
+
width,
|
| 1118 |
+
prompt_embeds.dtype,
|
| 1119 |
+
device,
|
| 1120 |
+
generator,
|
| 1121 |
+
latents,
|
| 1122 |
+
)
|
| 1123 |
+
if image_ids is not None:
|
| 1124 |
+
latent_ids = torch.cat([latent_ids, image_ids, source_primitive_ids, target_primitive_ids], dim=0) # dim 0 is sequence dimension
|
| 1125 |
+
##########
|
| 1126 |
+
|
| 1127 |
+
# 5. Prepare timesteps
|
| 1128 |
+
sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas
|
| 1129 |
+
image_seq_len = latents.shape[1]
|
| 1130 |
+
mu = calculate_shift(
|
| 1131 |
+
image_seq_len,
|
| 1132 |
+
self.scheduler.config.get("base_image_seq_len", 256),
|
| 1133 |
+
self.scheduler.config.get("max_image_seq_len", 4096),
|
| 1134 |
+
self.scheduler.config.get("base_shift", 0.5),
|
| 1135 |
+
self.scheduler.config.get("max_shift", 1.15),
|
| 1136 |
+
)
|
| 1137 |
+
timesteps, num_inference_steps = retrieve_timesteps(
|
| 1138 |
+
self.scheduler,
|
| 1139 |
+
num_inference_steps,
|
| 1140 |
+
device,
|
| 1141 |
+
sigmas=sigmas,
|
| 1142 |
+
mu=mu,
|
| 1143 |
+
)
|
| 1144 |
+
num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
|
| 1145 |
+
self._num_timesteps = len(timesteps)
|
| 1146 |
+
|
| 1147 |
+
# handle guidance
|
| 1148 |
+
if self.transformer.config.guidance_embeds:
|
| 1149 |
+
guidance = torch.full([1], guidance_scale, device=device, dtype=torch.float32)
|
| 1150 |
+
guidance = guidance.expand(latents.shape[0])
|
| 1151 |
+
else:
|
| 1152 |
+
guidance = None
|
| 1153 |
+
|
| 1154 |
+
if (ip_adapter_image is not None or ip_adapter_image_embeds is not None) and (
|
| 1155 |
+
negative_ip_adapter_image is None and negative_ip_adapter_image_embeds is None
|
| 1156 |
+
):
|
| 1157 |
+
negative_ip_adapter_image = np.zeros((width, height, 3), dtype=np.uint8)
|
| 1158 |
+
negative_ip_adapter_image = [negative_ip_adapter_image] * self.transformer.encoder_hid_proj.num_ip_adapters
|
| 1159 |
+
|
| 1160 |
+
elif (ip_adapter_image is None and ip_adapter_image_embeds is None) and (
|
| 1161 |
+
negative_ip_adapter_image is not None or negative_ip_adapter_image_embeds is not None
|
| 1162 |
+
):
|
| 1163 |
+
ip_adapter_image = np.zeros((width, height, 3), dtype=np.uint8)
|
| 1164 |
+
ip_adapter_image = [ip_adapter_image] * self.transformer.encoder_hid_proj.num_ip_adapters
|
| 1165 |
+
|
| 1166 |
+
if self.joint_attention_kwargs is None:
|
| 1167 |
+
self._joint_attention_kwargs = {}
|
| 1168 |
+
|
| 1169 |
+
image_embeds = None
|
| 1170 |
+
negative_image_embeds = None
|
| 1171 |
+
if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
|
| 1172 |
+
image_embeds = self.prepare_ip_adapter_image_embeds(
|
| 1173 |
+
ip_adapter_image,
|
| 1174 |
+
ip_adapter_image_embeds,
|
| 1175 |
+
device,
|
| 1176 |
+
batch_size * num_images_per_prompt,
|
| 1177 |
+
)
|
| 1178 |
+
if negative_ip_adapter_image is not None or negative_ip_adapter_image_embeds is not None:
|
| 1179 |
+
negative_image_embeds = self.prepare_ip_adapter_image_embeds(
|
| 1180 |
+
negative_ip_adapter_image,
|
| 1181 |
+
negative_ip_adapter_image_embeds,
|
| 1182 |
+
device,
|
| 1183 |
+
batch_size * num_images_per_prompt,
|
| 1184 |
+
)
|
| 1185 |
+
|
| 1186 |
+
# 6. Denoising loop
|
| 1187 |
+
# We set the index here to remove DtoH sync, helpful especially during compilation.
|
| 1188 |
+
# Check out more details here: https://github.com/huggingface/diffusers/pull/11696
|
| 1189 |
+
self.scheduler.set_begin_index(0)
|
| 1190 |
+
with self.progress_bar(total=num_inference_steps) as progress_bar:
|
| 1191 |
+
for i, t in enumerate(timesteps):
|
| 1192 |
+
if self.interrupt:
|
| 1193 |
+
continue
|
| 1194 |
+
|
| 1195 |
+
self._current_timestep = t
|
| 1196 |
+
if image_embeds is not None:
|
| 1197 |
+
self._joint_attention_kwargs["ip_adapter_image_embeds"] = image_embeds
|
| 1198 |
+
|
| 1199 |
+
latent_model_input = latents
|
| 1200 |
+
##########
|
| 1201 |
+
if image_latents is not None:
|
| 1202 |
+
latent_model_input = torch.cat([latents, image_latents, source_primitive_latents, target_primitive_latents], dim=1)
|
| 1203 |
+
##########
|
| 1204 |
+
timestep = t.expand(latents.shape[0]).to(latents.dtype)
|
| 1205 |
+
|
| 1206 |
+
noise_pred = self.transformer(
|
| 1207 |
+
hidden_states=latent_model_input,
|
| 1208 |
+
timestep=timestep / 1000,
|
| 1209 |
+
guidance=guidance,
|
| 1210 |
+
pooled_projections=pooled_prompt_embeds,
|
| 1211 |
+
encoder_hidden_states=prompt_embeds,
|
| 1212 |
+
txt_ids=text_ids,
|
| 1213 |
+
img_ids=latent_ids,
|
| 1214 |
+
joint_attention_kwargs=self.joint_attention_kwargs,
|
| 1215 |
+
return_dict=False,
|
| 1216 |
+
)[0]
|
| 1217 |
+
noise_pred = noise_pred[:, : latents.size(1)]
|
| 1218 |
+
|
| 1219 |
+
if do_true_cfg:
|
| 1220 |
+
if negative_image_embeds is not None:
|
| 1221 |
+
self._joint_attention_kwargs["ip_adapter_image_embeds"] = negative_image_embeds
|
| 1222 |
+
neg_noise_pred = self.transformer(
|
| 1223 |
+
hidden_states=latent_model_input,
|
| 1224 |
+
timestep=timestep / 1000,
|
| 1225 |
+
guidance=guidance,
|
| 1226 |
+
pooled_projections=negative_pooled_prompt_embeds,
|
| 1227 |
+
encoder_hidden_states=negative_prompt_embeds,
|
| 1228 |
+
txt_ids=negative_text_ids,
|
| 1229 |
+
img_ids=latent_ids,
|
| 1230 |
+
joint_attention_kwargs=self.joint_attention_kwargs,
|
| 1231 |
+
return_dict=False,
|
| 1232 |
+
)[0]
|
| 1233 |
+
neg_noise_pred = neg_noise_pred[:, : latents.size(1)]
|
| 1234 |
+
noise_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
|
| 1235 |
+
|
| 1236 |
+
# compute the previous noisy sample x_t -> x_t-1
|
| 1237 |
+
latents_dtype = latents.dtype
|
| 1238 |
+
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
| 1239 |
+
|
| 1240 |
+
if latents.dtype != latents_dtype:
|
| 1241 |
+
if torch.backends.mps.is_available():
|
| 1242 |
+
# some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
|
| 1243 |
+
latents = latents.to(latents_dtype)
|
| 1244 |
+
|
| 1245 |
+
if callback_on_step_end is not None:
|
| 1246 |
+
callback_kwargs = {}
|
| 1247 |
+
for k in callback_on_step_end_tensor_inputs:
|
| 1248 |
+
callback_kwargs[k] = locals()[k]
|
| 1249 |
+
callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
|
| 1250 |
+
|
| 1251 |
+
latents = callback_outputs.pop("latents", latents)
|
| 1252 |
+
prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
|
| 1253 |
+
|
| 1254 |
+
# call the callback, if provided
|
| 1255 |
+
if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
|
| 1256 |
+
progress_bar.update()
|
| 1257 |
+
|
| 1258 |
+
if XLA_AVAILABLE:
|
| 1259 |
+
xm.mark_step()
|
| 1260 |
+
|
| 1261 |
+
self._current_timestep = None
|
| 1262 |
+
|
| 1263 |
+
if output_type == "latent":
|
| 1264 |
+
image = latents
|
| 1265 |
+
else:
|
| 1266 |
+
latents = self._unpack_latents(latents, height, width, self.vae_scale_factor)
|
| 1267 |
+
latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor
|
| 1268 |
+
image = self.vae.decode(latents, return_dict=False)[0]
|
| 1269 |
+
image = self.image_processor.postprocess(image, output_type=output_type)
|
| 1270 |
+
|
| 1271 |
+
# Offload all models
|
| 1272 |
+
self.maybe_free_model_hooks()
|
| 1273 |
+
|
| 1274 |
+
if not return_dict:
|
| 1275 |
+
return (image,)
|
| 1276 |
+
|
| 1277 |
+
return FluxPipelineOutput(images=image)
|
requirements.txt
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Hugging Face Space (ZeroGPU) requirements for spaces_app.py
|
| 2 |
+
spaces
|
| 3 |
+
gradio>=4.44
|
| 4 |
+
diffusers
|
| 5 |
+
transformers
|
| 6 |
+
accelerate
|
| 7 |
+
peft
|
| 8 |
+
safetensors
|
| 9 |
+
huggingface_hub
|
| 10 |
+
sentencepiece
|
| 11 |
+
protobuf
|
| 12 |
+
Pillow
|
| 13 |
+
numpy
|
| 14 |
+
torch
|
| 15 |
+
torchvision
|