blanchon commited on
Commit
c5d1d5a
·
verified ·
1 Parent(s): 1e76965

Create pipeline.py

Browse files
Files changed (1) hide show
  1. pipeline.py +1092 -0
pipeline.py ADDED
@@ -0,0 +1,1092 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 Black Forest Labs and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from typing import Any, Callable, Dict, List, Optional, Union
17
+
18
+ import numpy as np
19
+ import torch
20
+ from transformers import (
21
+ CLIPImageProcessor,
22
+ CLIPTextModel,
23
+ CLIPTokenizer,
24
+ CLIPVisionModelWithProjection,
25
+ T5EncoderModel,
26
+ T5TokenizerFast,
27
+ )
28
+
29
+ from diffusers.image_processor import PipelineImageInput, VaeImageProcessor
30
+ from diffusers.loaders import FluxIPAdapterMixin, FluxLoraLoaderMixin, FromSingleFileMixin, TextualInversionLoaderMixin
31
+ from diffusers.models import AutoencoderKL, AutoencoderDC, FluxTransformer2DModel
32
+ from diffusers.schedulers import FlowMatchEulerDiscreteScheduler
33
+ from diffusers.utils import (
34
+ USE_PEFT_BACKEND,
35
+ deprecate,
36
+ is_torch_xla_available,
37
+ logging,
38
+ replace_example_docstring,
39
+ scale_lora_layers,
40
+ unscale_lora_layers,
41
+ )
42
+ from diffusers.utils.torch_utils import randn_tensor
43
+ from diffusers.pipelines.pipeline_utils import DiffusionPipeline
44
+ from diffusers.pipelines.flux.pipeline_output import FluxPipelineOutput
45
+ from fluxdcgen_transformer import DCFluxTransformer2DModel
46
+
47
+ if is_torch_xla_available():
48
+ import torch_xla.core.xla_model as xm
49
+
50
+ XLA_AVAILABLE = True
51
+ else:
52
+ XLA_AVAILABLE = False
53
+
54
+
55
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
56
+
57
+ EXAMPLE_DOC_STRING = """
58
+ Examples:
59
+ ```py
60
+ >>> import torch
61
+ >>> from diffusers import FluxPipeline
62
+
63
+ >>> pipe = FluxPipeline.from_pretrained("black-forest-labs/FLUX.1-schnell", torch_dtype=torch.bfloat16)
64
+ >>> pipe.to("cuda")
65
+ >>> prompt = "A cat holding a sign that says hello world"
66
+ >>> # Depending on the variant being used, the pipeline call will slightly vary.
67
+ >>> # Refer to the pipeline documentation for more details.
68
+ >>> image = pipe(prompt, num_inference_steps=4, guidance_scale=0.0).images[0]
69
+ >>> image.save("flux.png")
70
+ ```
71
+ """
72
+
73
+
74
+ def calculate_shift(
75
+ image_seq_len,
76
+ base_seq_len: int = 256,
77
+ max_seq_len: int = 4096,
78
+ base_shift: float = 0.5,
79
+ max_shift: float = 1.15,
80
+ ):
81
+ m = (max_shift - base_shift) / (max_seq_len - base_seq_len)
82
+ b = base_shift - m * base_seq_len
83
+ mu = image_seq_len * m + b
84
+ return mu
85
+
86
+
87
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
88
+ def retrieve_timesteps(
89
+ scheduler,
90
+ num_inference_steps: Optional[int] = None,
91
+ device: Optional[Union[str, torch.device]] = None,
92
+ timesteps: Optional[List[int]] = None,
93
+ sigmas: Optional[List[float]] = None,
94
+ **kwargs,
95
+ ):
96
+ r"""
97
+ Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
98
+ custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
99
+
100
+ Args:
101
+ scheduler (`SchedulerMixin`):
102
+ The scheduler to get timesteps from.
103
+ num_inference_steps (`int`):
104
+ The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
105
+ must be `None`.
106
+ device (`str` or `torch.device`, *optional*):
107
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
108
+ timesteps (`List[int]`, *optional*):
109
+ Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
110
+ `num_inference_steps` and `sigmas` must be `None`.
111
+ sigmas (`List[float]`, *optional*):
112
+ Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
113
+ `num_inference_steps` and `timesteps` must be `None`.
114
+
115
+ Returns:
116
+ `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
117
+ second element is the number of inference steps.
118
+ """
119
+ if timesteps is not None and sigmas is not None:
120
+ raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
121
+ if timesteps is not None:
122
+ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
123
+ if not accepts_timesteps:
124
+ raise ValueError(
125
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
126
+ f" timestep schedules. Please check whether you are using the correct scheduler."
127
+ )
128
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
129
+ timesteps = scheduler.timesteps
130
+ num_inference_steps = len(timesteps)
131
+ elif sigmas is not None:
132
+ accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
133
+ if not accept_sigmas:
134
+ raise ValueError(
135
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
136
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
137
+ )
138
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
139
+ timesteps = scheduler.timesteps
140
+ num_inference_steps = len(timesteps)
141
+ else:
142
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
143
+ timesteps = scheduler.timesteps
144
+ return timesteps, num_inference_steps
145
+
146
+
147
+ class FluxPipeline(
148
+ DiffusionPipeline,
149
+ FluxLoraLoaderMixin,
150
+ FromSingleFileMixin,
151
+ TextualInversionLoaderMixin,
152
+ FluxIPAdapterMixin,
153
+ ):
154
+ r"""
155
+ The Flux pipeline for text-to-image generation.
156
+
157
+ Reference: https://blackforestlabs.ai/announcing-black-forest-labs/
158
+
159
+ Args:
160
+ transformer ([`FluxTransformer2DModel`]):
161
+ Conditional Transformer (MMDiT) architecture to denoise the encoded image latents.
162
+ scheduler ([`FlowMatchEulerDiscreteScheduler`]):
163
+ A scheduler to be used in combination with `transformer` to denoise the encoded image latents.
164
+ vae ([`AutoencoderKL`] or [`AutoencoderDC`]):
165
+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
166
+ Can be either the standard AutoencoderKL or the deeply compressed AutoencoderDC.
167
+ text_encoder ([`CLIPTextModel`]):
168
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
169
+ the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
170
+ text_encoder_2 ([`T5EncoderModel`]):
171
+ [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5EncoderModel), specifically
172
+ the [google/t5-v1_1-xxl](https://huggingface.co/google/t5-v1_1-xxl) variant.
173
+ tokenizer (`CLIPTokenizer`):
174
+ Tokenizer of class
175
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/en/model_doc/clip#transformers.CLIPTokenizer).
176
+ tokenizer_2 (`T5TokenizerFast`):
177
+ Second Tokenizer of class
178
+ [T5TokenizerFast](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5TokenizerFast).
179
+ """
180
+
181
+ model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->transformer->vae"
182
+ _optional_components = ["image_encoder", "feature_extractor"]
183
+ _callback_tensor_inputs = ["latents", "prompt_embeds"]
184
+
185
+ def __init__(
186
+ self,
187
+ scheduler: FlowMatchEulerDiscreteScheduler,
188
+ vae: Union[AutoencoderKL, AutoencoderDC],
189
+ text_encoder: CLIPTextModel,
190
+ tokenizer: CLIPTokenizer,
191
+ text_encoder_2: T5EncoderModel,
192
+ tokenizer_2: T5TokenizerFast,
193
+ transformer: DCFluxTransformer2DModel,
194
+ image_encoder: CLIPVisionModelWithProjection = None,
195
+ feature_extractor: CLIPImageProcessor = None,
196
+ ):
197
+ super().__init__()
198
+
199
+ self.register_modules(
200
+ vae=vae,
201
+ text_encoder=text_encoder,
202
+ text_encoder_2=text_encoder_2,
203
+ tokenizer=tokenizer,
204
+ tokenizer_2=tokenizer_2,
205
+ transformer=transformer,
206
+ scheduler=scheduler,
207
+ image_encoder=image_encoder,
208
+ feature_extractor=feature_extractor,
209
+ )
210
+ # Handle both AutoencoderKL (has block_out_channels) and AutoencoderDC (has spatial_compression_ratio)
211
+ if getattr(self, "vae", None):
212
+ if hasattr(self.vae, "spatial_compression_ratio"):
213
+ self.vae_scale_factor = self.vae.spatial_compression_ratio
214
+ else:
215
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
216
+ else:
217
+ self.vae_scale_factor = 8
218
+
219
+ # AutoencoderDC doesn't use packing, so we handle image processor accordingly
220
+ is_dc_ae = hasattr(self.vae, "spatial_compression_ratio") if getattr(self, "vae", None) else False
221
+ if is_dc_ae:
222
+ # DC-AE doesn't use packing
223
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
224
+ else:
225
+ # Flux latents are turned into 2x2 patches and packed. This means the latent width and height has to be divisible
226
+ # by the patch size. So the vae scale factor is multiplied by the patch size to account for this
227
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2)
228
+ self.tokenizer_max_length = (
229
+ self.tokenizer.model_max_length if hasattr(self, "tokenizer") and self.tokenizer is not None else 77
230
+ )
231
+ self.default_sample_size = 128
232
+
233
+ def _get_t5_prompt_embeds(
234
+ self,
235
+ prompt: Union[str, List[str]] = None,
236
+ num_images_per_prompt: int = 1,
237
+ max_sequence_length: int = 512,
238
+ device: Optional[torch.device] = None,
239
+ dtype: Optional[torch.dtype] = None,
240
+ ):
241
+ device = device or self._execution_device
242
+ dtype = dtype or self.text_encoder.dtype
243
+
244
+ prompt = [prompt] if isinstance(prompt, str) else prompt
245
+ batch_size = len(prompt)
246
+
247
+ if isinstance(self, TextualInversionLoaderMixin):
248
+ prompt = self.maybe_convert_prompt(prompt, self.tokenizer_2)
249
+
250
+ text_inputs = self.tokenizer_2(
251
+ prompt,
252
+ padding="max_length",
253
+ max_length=max_sequence_length,
254
+ truncation=True,
255
+ return_length=False,
256
+ return_overflowing_tokens=False,
257
+ return_tensors="pt",
258
+ )
259
+ text_input_ids = text_inputs.input_ids
260
+ untruncated_ids = self.tokenizer_2(prompt, padding="longest", return_tensors="pt").input_ids
261
+
262
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids):
263
+ removed_text = self.tokenizer_2.batch_decode(untruncated_ids[:, self.tokenizer_max_length - 1 : -1])
264
+ logger.warning(
265
+ "The following part of your input was truncated because `max_sequence_length` is set to "
266
+ f" {max_sequence_length} tokens: {removed_text}"
267
+ )
268
+
269
+ prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0]
270
+
271
+ dtype = self.text_encoder_2.dtype
272
+ prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
273
+
274
+ _, seq_len, _ = prompt_embeds.shape
275
+
276
+ # duplicate text embeddings and attention mask for each generation per prompt, using mps friendly method
277
+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
278
+ prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
279
+
280
+ return prompt_embeds
281
+
282
+ def _get_clip_prompt_embeds(
283
+ self,
284
+ prompt: Union[str, List[str]],
285
+ num_images_per_prompt: int = 1,
286
+ device: Optional[torch.device] = None,
287
+ ):
288
+ device = device or self._execution_device
289
+
290
+ prompt = [prompt] if isinstance(prompt, str) else prompt
291
+ batch_size = len(prompt)
292
+
293
+ if isinstance(self, TextualInversionLoaderMixin):
294
+ prompt = self.maybe_convert_prompt(prompt, self.tokenizer)
295
+
296
+ text_inputs = self.tokenizer(
297
+ prompt,
298
+ padding="max_length",
299
+ max_length=self.tokenizer_max_length,
300
+ truncation=True,
301
+ return_overflowing_tokens=False,
302
+ return_length=False,
303
+ return_tensors="pt",
304
+ )
305
+
306
+ text_input_ids = text_inputs.input_ids
307
+ untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
308
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids):
309
+ removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer_max_length - 1 : -1])
310
+ logger.warning(
311
+ "The following part of your input was truncated because CLIP can only handle sequences up to"
312
+ f" {self.tokenizer_max_length} tokens: {removed_text}"
313
+ )
314
+ prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False)
315
+
316
+ # Use pooled output of CLIPTextModel
317
+ prompt_embeds = prompt_embeds.pooler_output
318
+ prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
319
+
320
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
321
+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt)
322
+ prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, -1)
323
+
324
+ return prompt_embeds
325
+
326
+ def encode_prompt(
327
+ self,
328
+ prompt: Union[str, List[str]],
329
+ prompt_2: Optional[Union[str, List[str]]] = None,
330
+ device: Optional[torch.device] = None,
331
+ num_images_per_prompt: int = 1,
332
+ prompt_embeds: Optional[torch.FloatTensor] = None,
333
+ pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
334
+ max_sequence_length: int = 512,
335
+ lora_scale: Optional[float] = None,
336
+ ):
337
+ r"""
338
+
339
+ Args:
340
+ prompt (`str` or `List[str]`, *optional*):
341
+ prompt to be encoded
342
+ prompt_2 (`str` or `List[str]`, *optional*):
343
+ The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
344
+ used in all text-encoders
345
+ device: (`torch.device`):
346
+ torch device
347
+ num_images_per_prompt (`int`):
348
+ number of images that should be generated per prompt
349
+ prompt_embeds (`torch.FloatTensor`, *optional*):
350
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
351
+ provided, text embeddings will be generated from `prompt` input argument.
352
+ pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
353
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
354
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
355
+ lora_scale (`float`, *optional*):
356
+ A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
357
+ """
358
+ device = device or self._execution_device
359
+
360
+ # set lora scale so that monkey patched LoRA
361
+ # function of text encoder can correctly access it
362
+ if lora_scale is not None and isinstance(self, FluxLoraLoaderMixin):
363
+ self._lora_scale = lora_scale
364
+
365
+ # dynamically adjust the LoRA scale
366
+ if self.text_encoder is not None and USE_PEFT_BACKEND:
367
+ scale_lora_layers(self.text_encoder, lora_scale)
368
+ if self.text_encoder_2 is not None and USE_PEFT_BACKEND:
369
+ scale_lora_layers(self.text_encoder_2, lora_scale)
370
+
371
+ prompt = [prompt] if isinstance(prompt, str) else prompt
372
+
373
+ if prompt_embeds is None:
374
+ prompt_2 = prompt_2 or prompt
375
+ prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2
376
+
377
+ # We only use the pooled prompt output from the CLIPTextModel
378
+ pooled_prompt_embeds = self._get_clip_prompt_embeds(
379
+ prompt=prompt,
380
+ device=device,
381
+ num_images_per_prompt=num_images_per_prompt,
382
+ )
383
+ prompt_embeds = self._get_t5_prompt_embeds(
384
+ prompt=prompt_2,
385
+ num_images_per_prompt=num_images_per_prompt,
386
+ max_sequence_length=max_sequence_length,
387
+ device=device,
388
+ )
389
+
390
+ if self.text_encoder is not None:
391
+ if isinstance(self, FluxLoraLoaderMixin) and USE_PEFT_BACKEND:
392
+ # Retrieve the original scale by scaling back the LoRA layers
393
+ unscale_lora_layers(self.text_encoder, lora_scale)
394
+
395
+ if self.text_encoder_2 is not None:
396
+ if isinstance(self, FluxLoraLoaderMixin) and USE_PEFT_BACKEND:
397
+ # Retrieve the original scale by scaling back the LoRA layers
398
+ unscale_lora_layers(self.text_encoder_2, lora_scale)
399
+
400
+ dtype = self.text_encoder.dtype if self.text_encoder is not None else self.transformer.dtype
401
+ text_ids = torch.zeros(prompt_embeds.shape[1], 3).to(device=device, dtype=dtype)
402
+
403
+ return prompt_embeds, pooled_prompt_embeds, text_ids
404
+
405
+ def encode_image(self, image, device, num_images_per_prompt):
406
+ dtype = next(self.image_encoder.parameters()).dtype
407
+
408
+ if not isinstance(image, torch.Tensor):
409
+ image = self.feature_extractor(image, return_tensors="pt").pixel_values
410
+
411
+ image = image.to(device=device, dtype=dtype)
412
+ image_embeds = self.image_encoder(image).image_embeds
413
+ image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0)
414
+ return image_embeds
415
+
416
+ def prepare_ip_adapter_image_embeds(
417
+ self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt
418
+ ):
419
+ image_embeds = []
420
+ if ip_adapter_image_embeds is None:
421
+ if not isinstance(ip_adapter_image, list):
422
+ ip_adapter_image = [ip_adapter_image]
423
+
424
+ if len(ip_adapter_image) != self.transformer.encoder_hid_proj.num_ip_adapters:
425
+ raise ValueError(
426
+ f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {self.transformer.encoder_hid_proj.num_ip_adapters} IP Adapters."
427
+ )
428
+
429
+ for single_ip_adapter_image in ip_adapter_image:
430
+ single_image_embeds = self.encode_image(single_ip_adapter_image, device, 1)
431
+ image_embeds.append(single_image_embeds[None, :])
432
+ else:
433
+ if not isinstance(ip_adapter_image_embeds, list):
434
+ ip_adapter_image_embeds = [ip_adapter_image_embeds]
435
+
436
+ if len(ip_adapter_image_embeds) != self.transformer.encoder_hid_proj.num_ip_adapters:
437
+ raise ValueError(
438
+ f"`ip_adapter_image_embeds` must have same length as the number of IP Adapters. Got {len(ip_adapter_image_embeds)} image embeds and {self.transformer.encoder_hid_proj.num_ip_adapters} IP Adapters."
439
+ )
440
+
441
+ for single_image_embeds in ip_adapter_image_embeds:
442
+ image_embeds.append(single_image_embeds)
443
+
444
+ ip_adapter_image_embeds = []
445
+ for single_image_embeds in image_embeds:
446
+ single_image_embeds = torch.cat([single_image_embeds] * num_images_per_prompt, dim=0)
447
+ single_image_embeds = single_image_embeds.to(device=device)
448
+ ip_adapter_image_embeds.append(single_image_embeds)
449
+
450
+ return ip_adapter_image_embeds
451
+
452
+ def check_inputs(
453
+ self,
454
+ prompt,
455
+ prompt_2,
456
+ height,
457
+ width,
458
+ negative_prompt=None,
459
+ negative_prompt_2=None,
460
+ prompt_embeds=None,
461
+ negative_prompt_embeds=None,
462
+ pooled_prompt_embeds=None,
463
+ negative_pooled_prompt_embeds=None,
464
+ callback_on_step_end_tensor_inputs=None,
465
+ max_sequence_length=None,
466
+ ):
467
+ # AutoencoderDC doesn't use packing, so divisibility requirement is different
468
+ is_dc_ae = hasattr(self.vae, "spatial_compression_ratio")
469
+ divisor = self.vae_scale_factor if is_dc_ae else (self.vae_scale_factor * 2)
470
+
471
+ if height % divisor != 0 or width % divisor != 0:
472
+ logger.warning(
473
+ f"`height` and `width` have to be divisible by {divisor} but are {height} and {width}. Dimensions will be resized accordingly"
474
+ )
475
+
476
+ if callback_on_step_end_tensor_inputs is not None and not all(
477
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
478
+ ):
479
+ raise ValueError(
480
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
481
+ )
482
+
483
+ if prompt is not None and prompt_embeds is not None:
484
+ raise ValueError(
485
+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
486
+ " only forward one of the two."
487
+ )
488
+ elif prompt_2 is not None and prompt_embeds is not None:
489
+ raise ValueError(
490
+ f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
491
+ " only forward one of the two."
492
+ )
493
+ elif prompt is None and prompt_embeds is None:
494
+ raise ValueError(
495
+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
496
+ )
497
+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
498
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
499
+ elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)):
500
+ raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}")
501
+
502
+ if negative_prompt is not None and negative_prompt_embeds is not None:
503
+ raise ValueError(
504
+ f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
505
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
506
+ )
507
+ elif negative_prompt_2 is not None and negative_prompt_embeds is not None:
508
+ raise ValueError(
509
+ f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:"
510
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
511
+ )
512
+
513
+ if prompt_embeds is not None and pooled_prompt_embeds is None:
514
+ raise ValueError(
515
+ "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`."
516
+ )
517
+ if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None:
518
+ raise ValueError(
519
+ "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`."
520
+ )
521
+
522
+ if max_sequence_length is not None and max_sequence_length > 512:
523
+ raise ValueError(f"`max_sequence_length` cannot be greater than 512 but is {max_sequence_length}")
524
+
525
+ @staticmethod
526
+ def _prepare_latent_image_ids(batch_size, height, width, device, dtype):
527
+ latent_image_ids = torch.zeros(height, width, 3)
528
+ latent_image_ids[..., 1] = latent_image_ids[..., 1] + torch.arange(height)[:, None]
529
+ latent_image_ids[..., 2] = latent_image_ids[..., 2] + torch.arange(width)[None, :]
530
+
531
+ latent_image_id_height, latent_image_id_width, latent_image_id_channels = latent_image_ids.shape
532
+
533
+ latent_image_ids = latent_image_ids.reshape(
534
+ latent_image_id_height * latent_image_id_width, latent_image_id_channels
535
+ )
536
+
537
+ return latent_image_ids.to(device=device, dtype=dtype)
538
+
539
+ @staticmethod
540
+ def _pack_latents(latents, batch_size, num_channels_latents, height, width):
541
+ latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2)
542
+ latents = latents.permute(0, 2, 4, 1, 3, 5)
543
+ latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4)
544
+
545
+ return latents
546
+
547
+ @staticmethod
548
+ def _unpack_latents(latents, height, width, vae_scale_factor):
549
+ batch_size, num_patches, channels = latents.shape
550
+
551
+ # VAE applies 8x compression on images but we must also account for packing which requires
552
+ # latent height and width to be divisible by 2.
553
+ height = 2 * (int(height) // (vae_scale_factor * 2))
554
+ width = 2 * (int(width) // (vae_scale_factor * 2))
555
+
556
+ latents = latents.view(batch_size, height // 2, width // 2, channels // 4, 2, 2)
557
+ latents = latents.permute(0, 3, 1, 4, 2, 5)
558
+
559
+ latents = latents.reshape(batch_size, channels // (2 * 2), height, width)
560
+
561
+ return latents
562
+
563
+ def enable_vae_slicing(self):
564
+ r"""
565
+ Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to
566
+ compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.
567
+ """
568
+ depr_message = f"Calling `enable_vae_slicing()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.enable_slicing()`."
569
+ deprecate(
570
+ "enable_vae_slicing",
571
+ "0.40.0",
572
+ depr_message,
573
+ )
574
+ self.vae.enable_slicing()
575
+
576
+ def disable_vae_slicing(self):
577
+ r"""
578
+ Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to
579
+ computing decoding in one step.
580
+ """
581
+ depr_message = f"Calling `disable_vae_slicing()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.disable_slicing()`."
582
+ deprecate(
583
+ "disable_vae_slicing",
584
+ "0.40.0",
585
+ depr_message,
586
+ )
587
+ self.vae.disable_slicing()
588
+
589
+ def enable_vae_tiling(self):
590
+ r"""
591
+ Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to
592
+ compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow
593
+ processing larger images.
594
+ """
595
+ depr_message = f"Calling `enable_vae_tiling()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.enable_tiling()`."
596
+ deprecate(
597
+ "enable_vae_tiling",
598
+ "0.40.0",
599
+ depr_message,
600
+ )
601
+ self.vae.enable_tiling()
602
+
603
+ def disable_vae_tiling(self):
604
+ r"""
605
+ Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to
606
+ computing decoding in one step.
607
+ """
608
+ depr_message = f"Calling `disable_vae_tiling()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.disable_tiling()`."
609
+ deprecate(
610
+ "disable_vae_tiling",
611
+ "0.40.0",
612
+ depr_message,
613
+ )
614
+ self.vae.disable_tiling()
615
+
616
+ def prepare_latents(
617
+ self,
618
+ batch_size,
619
+ num_channels_latents,
620
+ height,
621
+ width,
622
+ dtype,
623
+ device,
624
+ generator,
625
+ latents=None,
626
+ ):
627
+ # AutoencoderDC doesn't use packing, so we skip packing for it
628
+ is_dc_ae = hasattr(self.vae, "spatial_compression_ratio")
629
+
630
+ if is_dc_ae:
631
+ # For DC-AE, no packing needed
632
+ height = int(height) // self.vae_scale_factor
633
+ width = int(width) // self.vae_scale_factor
634
+ else:
635
+ # VAE applies 8x compression on images but we must also account for packing which requires
636
+ # latent height and width to be divisible by 2.
637
+ height = 2 * (int(height) // (self.vae_scale_factor * 2))
638
+ width = 2 * (int(width) // (self.vae_scale_factor * 2))
639
+
640
+ shape = (batch_size, num_channels_latents, height, width)
641
+
642
+ if latents is not None:
643
+ latents = latents.to(device=device, dtype=dtype)
644
+ if is_dc_ae:
645
+ # For DC-AE, reshape to (batch, height*width, channels)
646
+ latents = latents.view(batch_size, num_channels_latents, height * width)
647
+ latents = latents.permute(0, 2, 1)
648
+ latent_image_ids = self._prepare_latent_image_ids(batch_size, height, width, device, dtype)
649
+ else:
650
+ latent_image_ids = self._prepare_latent_image_ids(batch_size, height // 2, width // 2, device, dtype)
651
+ return latents, latent_image_ids
652
+
653
+ if isinstance(generator, list) and len(generator) != batch_size:
654
+ raise ValueError(
655
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
656
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
657
+ )
658
+
659
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
660
+
661
+ if not is_dc_ae:
662
+ # Only pack for standard AutoencoderKL
663
+ latents = self._pack_latents(latents, batch_size, num_channels_latents, height, width)
664
+ latent_image_ids = self._prepare_latent_image_ids(batch_size, height // 2, width // 2, device, dtype)
665
+ else:
666
+ # For DC-AE, no packing but we still need to reshape to (batch, height*width, channels)
667
+ latents = latents.view(batch_size, num_channels_latents, height * width)
668
+ latents = latents.permute(0, 2, 1) # (batch, height*width, channels)
669
+ latent_image_ids = self._prepare_latent_image_ids(batch_size, height, width, device, dtype)
670
+
671
+ return latents, latent_image_ids
672
+
673
+ @property
674
+ def guidance_scale(self):
675
+ return self._guidance_scale
676
+
677
+ @property
678
+ def joint_attention_kwargs(self):
679
+ return self._joint_attention_kwargs
680
+
681
+ @property
682
+ def num_timesteps(self):
683
+ return self._num_timesteps
684
+
685
+ @property
686
+ def current_timestep(self):
687
+ return self._current_timestep
688
+
689
+ @property
690
+ def interrupt(self):
691
+ return self._interrupt
692
+
693
+ @torch.no_grad()
694
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
695
+ def __call__(
696
+ self,
697
+ prompt: Union[str, List[str]] = None,
698
+ prompt_2: Optional[Union[str, List[str]]] = None,
699
+ negative_prompt: Union[str, List[str]] = None,
700
+ negative_prompt_2: Optional[Union[str, List[str]]] = None,
701
+ true_cfg_scale: float = 1.0,
702
+ height: Optional[int] = None,
703
+ width: Optional[int] = None,
704
+ num_inference_steps: int = 28,
705
+ sigmas: Optional[List[float]] = None,
706
+ guidance_scale: float = 3.5,
707
+ num_images_per_prompt: Optional[int] = 1,
708
+ generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
709
+ latents: Optional[torch.FloatTensor] = None,
710
+ prompt_embeds: Optional[torch.FloatTensor] = None,
711
+ pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
712
+ ip_adapter_image: Optional[PipelineImageInput] = None,
713
+ ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None,
714
+ negative_ip_adapter_image: Optional[PipelineImageInput] = None,
715
+ negative_ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None,
716
+ negative_prompt_embeds: Optional[torch.FloatTensor] = None,
717
+ negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
718
+ output_type: Optional[str] = "pil",
719
+ return_dict: bool = True,
720
+ joint_attention_kwargs: Optional[Dict[str, Any]] = None,
721
+ callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
722
+ callback_on_step_end_tensor_inputs: List[str] = ["latents"],
723
+ max_sequence_length: int = 512,
724
+ ):
725
+ r"""
726
+ Function invoked when calling the pipeline for generation.
727
+
728
+ Args:
729
+ prompt (`str` or `List[str]`, *optional*):
730
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
731
+ instead.
732
+ prompt_2 (`str` or `List[str]`, *optional*):
733
+ The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
734
+ will be used instead.
735
+ negative_prompt (`str` or `List[str]`, *optional*):
736
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
737
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is
738
+ not greater than `1`).
739
+ negative_prompt_2 (`str` or `List[str]`, *optional*):
740
+ The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
741
+ `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders.
742
+ true_cfg_scale (`float`, *optional*, defaults to 1.0):
743
+ True classifier-free guidance (guidance scale) is enabled when `true_cfg_scale` > 1 and
744
+ `negative_prompt` is provided.
745
+ height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
746
+ The height in pixels of the generated image. This is set to 1024 by default for the best results.
747
+ width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
748
+ The width in pixels of the generated image. This is set to 1024 by default for the best results.
749
+ num_inference_steps (`int`, *optional*, defaults to 50):
750
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
751
+ expense of slower inference.
752
+ sigmas (`List[float]`, *optional*):
753
+ Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
754
+ their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
755
+ will be used.
756
+ guidance_scale (`float`, *optional*, defaults to 3.5):
757
+ Embedded guiddance scale is enabled by setting `guidance_scale` > 1. Higher `guidance_scale` encourages
758
+ a model to generate images more aligned with `prompt` at the expense of lower image quality.
759
+
760
+ Guidance-distilled models approximates true classifer-free guidance for `guidance_scale` > 1. Refer to
761
+ the [paper](https://huggingface.co/papers/2210.03142) to learn more.
762
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
763
+ The number of images to generate per prompt.
764
+ generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
765
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
766
+ to make generation deterministic.
767
+ latents (`torch.FloatTensor`, *optional*):
768
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
769
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
770
+ tensor will be generated by sampling using the supplied random `generator`.
771
+ prompt_embeds (`torch.FloatTensor`, *optional*):
772
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
773
+ provided, text embeddings will be generated from `prompt` input argument.
774
+ pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
775
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
776
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
777
+ ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.
778
+ ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*):
779
+ Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of
780
+ IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not
781
+ provided, embeddings are computed from the `ip_adapter_image` input argument.
782
+ negative_ip_adapter_image:
783
+ (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.
784
+ negative_ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*):
785
+ Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of
786
+ IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not
787
+ provided, embeddings are computed from the `ip_adapter_image` input argument.
788
+ negative_prompt_embeds (`torch.FloatTensor`, *optional*):
789
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
790
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
791
+ argument.
792
+ negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
793
+ Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
794
+ weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
795
+ input argument.
796
+ output_type (`str`, *optional*, defaults to `"pil"`):
797
+ The output format of the generate image. Choose between
798
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
799
+ return_dict (`bool`, *optional*, defaults to `True`):
800
+ Whether or not to return a [`~pipelines.flux.FluxPipelineOutput`] instead of a plain tuple.
801
+ joint_attention_kwargs (`dict`, *optional*):
802
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
803
+ `self.processor` in
804
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
805
+ callback_on_step_end (`Callable`, *optional*):
806
+ A function that calls at the end of each denoising steps during the inference. The function is called
807
+ with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
808
+ callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
809
+ `callback_on_step_end_tensor_inputs`.
810
+ callback_on_step_end_tensor_inputs (`List`, *optional*):
811
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
812
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
813
+ `._callback_tensor_inputs` attribute of your pipeline class.
814
+ max_sequence_length (`int` defaults to 512): Maximum sequence length to use with the `prompt`.
815
+
816
+ Examples:
817
+
818
+ Returns:
819
+ [`~pipelines.flux.FluxPipelineOutput`] or `tuple`: [`~pipelines.flux.FluxPipelineOutput`] if `return_dict`
820
+ is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the generated
821
+ images.
822
+ """
823
+
824
+ height = height or self.default_sample_size * self.vae_scale_factor
825
+ width = width or self.default_sample_size * self.vae_scale_factor
826
+
827
+ # 1. Check inputs. Raise error if not correct
828
+ self.check_inputs(
829
+ prompt,
830
+ prompt_2,
831
+ height,
832
+ width,
833
+ negative_prompt=negative_prompt,
834
+ negative_prompt_2=negative_prompt_2,
835
+ prompt_embeds=prompt_embeds,
836
+ negative_prompt_embeds=negative_prompt_embeds,
837
+ pooled_prompt_embeds=pooled_prompt_embeds,
838
+ negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
839
+ callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs,
840
+ max_sequence_length=max_sequence_length,
841
+ )
842
+
843
+ self._guidance_scale = guidance_scale
844
+ self._joint_attention_kwargs = joint_attention_kwargs
845
+ self._current_timestep = None
846
+ self._interrupt = False
847
+
848
+ # 2. Define call parameters
849
+ if prompt is not None and isinstance(prompt, str):
850
+ batch_size = 1
851
+ elif prompt is not None and isinstance(prompt, list):
852
+ batch_size = len(prompt)
853
+ else:
854
+ batch_size = prompt_embeds.shape[0]
855
+
856
+ device = self._execution_device
857
+
858
+ lora_scale = (
859
+ self.joint_attention_kwargs.get("scale", None) if self.joint_attention_kwargs is not None else None
860
+ )
861
+ has_neg_prompt = negative_prompt is not None or (
862
+ negative_prompt_embeds is not None and negative_pooled_prompt_embeds is not None
863
+ )
864
+ do_true_cfg = true_cfg_scale > 1 and has_neg_prompt
865
+ (
866
+ prompt_embeds,
867
+ pooled_prompt_embeds,
868
+ text_ids,
869
+ ) = self.encode_prompt(
870
+ prompt=prompt,
871
+ prompt_2=prompt_2,
872
+ prompt_embeds=prompt_embeds,
873
+ pooled_prompt_embeds=pooled_prompt_embeds,
874
+ device=device,
875
+ num_images_per_prompt=num_images_per_prompt,
876
+ max_sequence_length=max_sequence_length,
877
+ lora_scale=lora_scale,
878
+ )
879
+ if do_true_cfg:
880
+ # For DC-Gen models with null embeddings in transformer, use them for proper CFG
881
+ if hasattr(self.transformer, 'has_null_embeddings') and self.transformer.has_null_embeddings():
882
+ # Use the null embeddings from the transformer for unconditional branch
883
+ batch_size_cfg = batch_size * num_images_per_prompt
884
+
885
+ negative_prompt_embeds, negative_pooled_prompt_embeds, negative_text_ids = \
886
+ self.transformer.get_null_embeddings(
887
+ batch_size=batch_size_cfg,
888
+ device=device,
889
+ dtype=prompt_embeds.dtype
890
+ )
891
+ else:
892
+ # Fallback to encoding negative prompts normally
893
+ (
894
+ negative_prompt_embeds,
895
+ negative_pooled_prompt_embeds,
896
+ negative_text_ids,
897
+ ) = self.encode_prompt(
898
+ prompt=negative_prompt,
899
+ prompt_2=negative_prompt_2,
900
+ prompt_embeds=negative_prompt_embeds,
901
+ pooled_prompt_embeds=negative_pooled_prompt_embeds,
902
+ device=device,
903
+ num_images_per_prompt=num_images_per_prompt,
904
+ max_sequence_length=max_sequence_length,
905
+ lora_scale=lora_scale,
906
+ )
907
+
908
+ # 4. Prepare latent variables
909
+ # AutoencoderDC doesn't use packing, so we don't divide by 4
910
+ is_dc_ae = hasattr(self.vae, "spatial_compression_ratio")
911
+ if is_dc_ae:
912
+ num_channels_latents = self.transformer.config.in_channels
913
+ else:
914
+ num_channels_latents = self.transformer.config.in_channels // 4
915
+
916
+ latents, latent_image_ids = self.prepare_latents(
917
+ batch_size * num_images_per_prompt,
918
+ num_channels_latents,
919
+ height,
920
+ width,
921
+ prompt_embeds.dtype,
922
+ device,
923
+ generator,
924
+ latents,
925
+ )
926
+
927
+ # 5. Prepare timesteps
928
+ sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas
929
+ if hasattr(self.scheduler.config, "use_flow_sigmas") and self.scheduler.config.use_flow_sigmas:
930
+ sigmas = None
931
+ image_seq_len = latents.shape[1]
932
+ mu = calculate_shift(
933
+ image_seq_len,
934
+ self.scheduler.config.get("base_image_seq_len", 256),
935
+ self.scheduler.config.get("max_image_seq_len", 4096),
936
+ self.scheduler.config.get("base_shift", 0.5),
937
+ self.scheduler.config.get("max_shift", 1.15),
938
+ )
939
+ timesteps, num_inference_steps = retrieve_timesteps(
940
+ self.scheduler,
941
+ num_inference_steps,
942
+ device,
943
+ sigmas=sigmas,
944
+ mu=mu,
945
+ )
946
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
947
+ self._num_timesteps = len(timesteps)
948
+
949
+ # handle guidance
950
+ if self.transformer.config.guidance_embeds:
951
+ guidance = torch.full([1], guidance_scale, device=device, dtype=torch.float32)
952
+ guidance = guidance.expand(latents.shape[0])
953
+ else:
954
+ guidance = None
955
+
956
+ if (ip_adapter_image is not None or ip_adapter_image_embeds is not None) and (
957
+ negative_ip_adapter_image is None and negative_ip_adapter_image_embeds is None
958
+ ):
959
+ negative_ip_adapter_image = np.zeros((width, height, 3), dtype=np.uint8)
960
+ negative_ip_adapter_image = [negative_ip_adapter_image] * self.transformer.encoder_hid_proj.num_ip_adapters
961
+
962
+ elif (ip_adapter_image is None and ip_adapter_image_embeds is None) and (
963
+ negative_ip_adapter_image is not None or negative_ip_adapter_image_embeds is not None
964
+ ):
965
+ ip_adapter_image = np.zeros((width, height, 3), dtype=np.uint8)
966
+ ip_adapter_image = [ip_adapter_image] * self.transformer.encoder_hid_proj.num_ip_adapters
967
+
968
+ if self.joint_attention_kwargs is None:
969
+ self._joint_attention_kwargs = {}
970
+
971
+ image_embeds = None
972
+ negative_image_embeds = None
973
+ if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
974
+ image_embeds = self.prepare_ip_adapter_image_embeds(
975
+ ip_adapter_image,
976
+ ip_adapter_image_embeds,
977
+ device,
978
+ batch_size * num_images_per_prompt,
979
+ )
980
+ if negative_ip_adapter_image is not None or negative_ip_adapter_image_embeds is not None:
981
+ negative_image_embeds = self.prepare_ip_adapter_image_embeds(
982
+ negative_ip_adapter_image,
983
+ negative_ip_adapter_image_embeds,
984
+ device,
985
+ batch_size * num_images_per_prompt,
986
+ )
987
+
988
+ # 6. Denoising loop
989
+ # We set the index here to remove DtoH sync, helpful especially during compilation.
990
+ # Check out more details here: https://github.com/huggingface/diffusers/pull/11696
991
+ self.scheduler.set_begin_index(0)
992
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
993
+ for i, t in enumerate(timesteps):
994
+ if self.interrupt:
995
+ continue
996
+
997
+ self._current_timestep = t
998
+ if image_embeds is not None:
999
+ self._joint_attention_kwargs["ip_adapter_image_embeds"] = image_embeds
1000
+ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
1001
+ timestep = t.expand(latents.shape[0]).to(latents.dtype)
1002
+
1003
+ with self.transformer.cache_context("cond"):
1004
+ noise_pred = self.transformer(
1005
+ hidden_states=latents,
1006
+ timestep=timestep / 1000,
1007
+ guidance=guidance,
1008
+ pooled_projections=pooled_prompt_embeds,
1009
+ encoder_hidden_states=prompt_embeds,
1010
+ txt_ids=text_ids,
1011
+ img_ids=latent_image_ids,
1012
+ joint_attention_kwargs=self.joint_attention_kwargs,
1013
+ return_dict=False,
1014
+ )[0]
1015
+
1016
+ if do_true_cfg:
1017
+ if negative_image_embeds is not None:
1018
+ self._joint_attention_kwargs["ip_adapter_image_embeds"] = negative_image_embeds
1019
+
1020
+ with self.transformer.cache_context("uncond"):
1021
+ neg_noise_pred = self.transformer(
1022
+ hidden_states=latents,
1023
+ timestep=timestep / 1000,
1024
+ guidance=guidance,
1025
+ pooled_projections=negative_pooled_prompt_embeds,
1026
+ encoder_hidden_states=negative_prompt_embeds,
1027
+ txt_ids=negative_text_ids,
1028
+ img_ids=latent_image_ids,
1029
+ joint_attention_kwargs=self.joint_attention_kwargs,
1030
+ return_dict=False,
1031
+ )[0]
1032
+ noise_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
1033
+
1034
+ # compute the previous noisy sample x_t -> x_t-1
1035
+ latents_dtype = latents.dtype
1036
+ latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
1037
+
1038
+ if latents.dtype != latents_dtype:
1039
+ if torch.backends.mps.is_available():
1040
+ # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
1041
+ latents = latents.to(latents_dtype)
1042
+
1043
+ if callback_on_step_end is not None:
1044
+ callback_kwargs = {}
1045
+ for k in callback_on_step_end_tensor_inputs:
1046
+ callback_kwargs[k] = locals()[k]
1047
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
1048
+
1049
+ latents = callback_outputs.pop("latents", latents)
1050
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
1051
+
1052
+ # call the callback, if provided
1053
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
1054
+ progress_bar.update()
1055
+
1056
+ if XLA_AVAILABLE:
1057
+ xm.mark_step()
1058
+
1059
+ self._current_timestep = None
1060
+
1061
+ if output_type == "latent":
1062
+ image = latents
1063
+ else:
1064
+ # AutoencoderDC doesn't use packing, so we skip unpacking for it
1065
+ is_dc_ae = hasattr(self.vae, "spatial_compression_ratio")
1066
+
1067
+ if not is_dc_ae:
1068
+ # Only unpack for standard AutoencoderKL
1069
+ latents = self._unpack_latents(latents, height, width, self.vae_scale_factor)
1070
+ else:
1071
+ # For DC-AE, reshape from (batch, height*width, channels) back to (batch, channels, height, width)
1072
+ batch_size = latents.shape[0]
1073
+ latent_height = int(height) // self.vae_scale_factor
1074
+ latent_width = int(width) // self.vae_scale_factor
1075
+ num_channels = latents.shape[2]
1076
+ latents = latents.permute(0, 2, 1) # (batch, channels, height*width)
1077
+ latents = latents.view(batch_size, num_channels, latent_height, latent_width)
1078
+
1079
+ # Handle both AutoencoderKL (has shift_factor) and AutoencoderDC (no shift_factor)
1080
+ scaling_factor = self.vae.config.scaling_factor
1081
+ shift_factor = getattr(self.vae.config, "shift_factor", 0.0)
1082
+ latents = (latents / scaling_factor) + shift_factor
1083
+ image = self.vae.decode(latents, return_dict=False)[0]
1084
+ image = self.image_processor.postprocess(image, output_type=output_type)
1085
+
1086
+ # Offload all models
1087
+ self.maybe_free_model_hooks()
1088
+
1089
+ if not return_dict:
1090
+ return (image,)
1091
+
1092
+ return FluxPipelineOutput(images=image)