luclima commited on
Commit
3840ce1
·
verified ·
1 Parent(s): a0c1ee2

Upload folder using huggingface_hub (part 13)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +12 -0
  2. western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_inpaint.py +1387 -0
  3. western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_xl/__init__.py +76 -0
  4. western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_xl/pipeline_flax_stable_diffusion_xl.py +307 -0
  5. western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_xl/pipeline_output.py +36 -0
  6. western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py +1308 -0
  7. western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +1492 -0
  8. western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +1734 -0
  9. western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py +977 -0
  10. western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_xl/watermark.py +42 -0
  11. western_ai/europe/diffusers/src/diffusers/pipelines/stable_video_diffusion/__init__.py +58 -0
  12. western_ai/europe/diffusers/src/diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py +743 -0
  13. western_ai/europe/diffusers/src/diffusers/pipelines/t2i_adapter/__init__.py +47 -0
  14. western_ai/europe/diffusers/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py +960 -0
  15. western_ai/europe/diffusers/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py +1303 -0
  16. western_ai/europe/diffusers/src/diffusers/pipelines/transformers_loading_utils.py +123 -0
  17. western_ai/europe/diffusers/src/diffusers/pipelines/visualcloze/__init__.py +52 -0
  18. western_ai/europe/diffusers/src/diffusers/pipelines/visualcloze/pipeline_visualcloze_combined.py +440 -0
  19. western_ai/europe/diffusers/src/diffusers/pipelines/visualcloze/pipeline_visualcloze_generation.py +977 -0
  20. western_ai/europe/diffusers/src/diffusers/pipelines/visualcloze/visualcloze_utils.py +250 -0
  21. western_ai/europe/diffusers/src/diffusers/pipelines/wan/__init__.py +54 -0
  22. western_ai/europe/diffusers/src/diffusers/pipelines/wan/image_processor.py +184 -0
  23. western_ai/europe/diffusers/src/diffusers/pipelines/wan/pipeline_output.py +20 -0
  24. western_ai/europe/diffusers/src/diffusers/pipelines/wan/pipeline_wan.py +678 -0
  25. western_ai/europe/diffusers/src/diffusers/pipelines/wan/pipeline_wan_animate.py +1202 -0
  26. western_ai/europe/diffusers/src/diffusers/pipelines/wan/pipeline_wan_i2v.py +841 -0
  27. western_ai/europe/diffusers/src/diffusers/pipelines/wan/pipeline_wan_vace.py +1046 -0
  28. western_ai/europe/diffusers/src/diffusers/pipelines/wan/pipeline_wan_video2video.py +741 -0
  29. western_ai/europe/diffusers/src/diffusers/pipelines/z_image/__init__.py +59 -0
  30. western_ai/europe/diffusers/src/diffusers/pipelines/z_image/pipeline_output.py +34 -0
  31. western_ai/europe/diffusers/src/diffusers/pipelines/z_image/pipeline_z_image.py +600 -0
  32. western_ai/europe/diffusers/src/diffusers/pipelines/z_image/pipeline_z_image_controlnet.py +740 -0
  33. western_ai/europe/diffusers/src/diffusers/pipelines/z_image/pipeline_z_image_controlnet_inpaint.py +767 -0
  34. western_ai/europe/diffusers/src/diffusers/pipelines/z_image/pipeline_z_image_img2img.py +715 -0
  35. western_ai/europe/diffusers/src/diffusers/pipelines/z_image/pipeline_z_image_inpaint.py +938 -0
  36. western_ai/europe/diffusers/src/diffusers/pipelines/z_image/pipeline_z_image_omni.py +749 -0
  37. western_ai/europe/diffusers/src/diffusers/py.typed +0 -0
  38. western_ai/europe/diffusers/src/diffusers/quantizers/__init__.py +18 -0
  39. western_ai/europe/diffusers/src/diffusers/quantizers/auto.py +166 -0
  40. western_ai/europe/diffusers/src/diffusers/quantizers/autoround/__init__.py +1 -0
  41. western_ai/europe/diffusers/src/diffusers/quantizers/autoround/autoround_quantizer.py +122 -0
  42. western_ai/europe/diffusers/src/diffusers/quantizers/base.py +265 -0
  43. western_ai/europe/diffusers/src/diffusers/quantizers/bitsandbytes/__init__.py +2 -0
  44. western_ai/europe/diffusers/src/diffusers/quantizers/bitsandbytes/bnb_quantizer.py +581 -0
  45. western_ai/europe/diffusers/src/diffusers/quantizers/bitsandbytes/utils.py +318 -0
  46. western_ai/europe/diffusers/src/diffusers/quantizers/gguf/__init__.py +1 -0
  47. western_ai/europe/diffusers/src/diffusers/quantizers/gguf/gguf_quantizer.py +190 -0
  48. western_ai/europe/diffusers/src/diffusers/quantizers/gguf/utils.py +612 -0
  49. western_ai/europe/diffusers/src/diffusers/quantizers/modelopt/__init__.py +1 -0
  50. western_ai/europe/diffusers/src/diffusers/quantizers/modelopt/modelopt_quantizer.py +190 -0
.gitattributes CHANGED
@@ -1102,3 +1102,15 @@ western_ai/canada/sentence-transformers/examples/sentence_transformer/applicatio
1102
  western_ai/europe/bigscience/experiments/images/gpipe.png filter=lfs diff=lfs merge=lfs -text
1103
  western_ai/europe/diffusers/docs/source/en/imgs/access_request.png filter=lfs diff=lfs merge=lfs -text
1104
  western_ai/europe/diffusers/examples/research_projects/gligen/generated-images-100000-00.png filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
1102
  western_ai/europe/bigscience/experiments/images/gpipe.png filter=lfs diff=lfs merge=lfs -text
1103
  western_ai/europe/diffusers/docs/source/en/imgs/access_request.png filter=lfs diff=lfs merge=lfs -text
1104
  western_ai/europe/diffusers/examples/research_projects/gligen/generated-images-100000-00.png filter=lfs diff=lfs merge=lfs -text
1105
+ western_ai/europe/flux/assets/cup.png filter=lfs diff=lfs merge=lfs -text
1106
+ western_ai/europe/flux/assets/dev_grid.jpg filter=lfs diff=lfs merge=lfs -text
1107
+ western_ai/europe/flux/assets/docs/canny.png filter=lfs diff=lfs merge=lfs -text
1108
+ western_ai/europe/flux/assets/docs/depth.png filter=lfs diff=lfs merge=lfs -text
1109
+ western_ai/europe/flux/assets/docs/inpainting.png filter=lfs diff=lfs merge=lfs -text
1110
+ western_ai/europe/flux/assets/docs/kontext.png filter=lfs diff=lfs merge=lfs -text
1111
+ western_ai/europe/flux/assets/docs/outpainting.png filter=lfs diff=lfs merge=lfs -text
1112
+ western_ai/europe/flux/assets/docs/redux.png filter=lfs diff=lfs merge=lfs -text
1113
+ western_ai/europe/flux/assets/flux-1-krea-dev-grid.png filter=lfs diff=lfs merge=lfs -text
1114
+ western_ai/europe/flux/assets/grid.jpg filter=lfs diff=lfs merge=lfs -text
1115
+ western_ai/europe/flux/assets/schnell_grid.jpg filter=lfs diff=lfs merge=lfs -text
1116
+ western_ai/europe/mistral/assets/smoe.png filter=lfs diff=lfs merge=lfs -text
western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_inpaint.py ADDED
@@ -0,0 +1,1387 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 Stability AI and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from typing import Any, Callable
17
+
18
+ import torch
19
+ from transformers import (
20
+ CLIPTextModelWithProjection,
21
+ CLIPTokenizer,
22
+ SiglipImageProcessor,
23
+ SiglipVisionModel,
24
+ T5EncoderModel,
25
+ T5TokenizerFast,
26
+ )
27
+
28
+ from ...callbacks import MultiPipelineCallbacks, PipelineCallback
29
+ from ...image_processor import PipelineImageInput, VaeImageProcessor
30
+ from ...loaders import FromSingleFileMixin, SD3IPAdapterMixin, SD3LoraLoaderMixin
31
+ from ...models.autoencoders import AutoencoderKL
32
+ from ...models.transformers import SD3Transformer2DModel
33
+ from ...schedulers import FlowMatchEulerDiscreteScheduler
34
+ from ...utils import (
35
+ USE_PEFT_BACKEND,
36
+ is_torch_xla_available,
37
+ logging,
38
+ replace_example_docstring,
39
+ scale_lora_layers,
40
+ unscale_lora_layers,
41
+ )
42
+ from ...utils.torch_utils import randn_tensor
43
+ from ..pipeline_utils import DiffusionPipeline
44
+ from .pipeline_output import StableDiffusion3PipelineOutput
45
+
46
+
47
+ if is_torch_xla_available():
48
+ import torch_xla.core.xla_model as xm
49
+
50
+ XLA_AVAILABLE = True
51
+ else:
52
+ XLA_AVAILABLE = False
53
+
54
+
55
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
56
+
57
+ EXAMPLE_DOC_STRING = """
58
+ Examples:
59
+ ```py
60
+ >>> import torch
61
+ >>> from diffusers import StableDiffusion3InpaintPipeline
62
+ >>> from diffusers.utils import load_image
63
+
64
+ >>> pipe = StableDiffusion3InpaintPipeline.from_pretrained(
65
+ ... "stabilityai/stable-diffusion-3-medium-diffusers", torch_dtype=torch.float16
66
+ ... )
67
+ >>> pipe.to("cuda")
68
+ >>> prompt = "Face of a yellow cat, high resolution, sitting on a park bench"
69
+ >>> img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png"
70
+ >>> mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png"
71
+ >>> source = load_image(img_url)
72
+ >>> mask = load_image(mask_url)
73
+ >>> image = pipe(prompt=prompt, image=source, mask_image=mask).images[0]
74
+ >>> image.save("sd3_inpainting.png")
75
+ ```
76
+ """
77
+
78
+
79
+ # Copied from diffusers.pipelines.flux.pipeline_flux.calculate_shift
80
+ def calculate_shift(
81
+ image_seq_len,
82
+ base_seq_len: int = 256,
83
+ max_seq_len: int = 4096,
84
+ base_shift: float = 0.5,
85
+ max_shift: float = 1.15,
86
+ ):
87
+ m = (max_shift - base_shift) / (max_seq_len - base_seq_len)
88
+ b = base_shift - m * base_seq_len
89
+ mu = image_seq_len * m + b
90
+ return mu
91
+
92
+
93
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
94
+ def retrieve_latents(
95
+ encoder_output: torch.Tensor, generator: torch.Generator | None = None, sample_mode: str = "sample"
96
+ ):
97
+ if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
98
+ return encoder_output.latent_dist.sample(generator)
99
+ elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
100
+ return encoder_output.latent_dist.mode()
101
+ elif hasattr(encoder_output, "latents"):
102
+ return encoder_output.latents
103
+ else:
104
+ raise AttributeError("Could not access latents of provided encoder_output")
105
+
106
+
107
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
108
+ def retrieve_timesteps(
109
+ scheduler,
110
+ num_inference_steps: int | None = None,
111
+ device: str | torch.device | None = None,
112
+ timesteps: list[int] | None = None,
113
+ sigmas: list[float] | None = None,
114
+ **kwargs,
115
+ ):
116
+ r"""
117
+ Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
118
+ custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
119
+
120
+ Args:
121
+ scheduler (`SchedulerMixin`):
122
+ The scheduler to get timesteps from.
123
+ num_inference_steps (`int`):
124
+ The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
125
+ must be `None`.
126
+ device (`str` or `torch.device`, *optional*):
127
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
128
+ timesteps (`list[int]`, *optional*):
129
+ Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
130
+ `num_inference_steps` and `sigmas` must be `None`.
131
+ sigmas (`list[float]`, *optional*):
132
+ Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
133
+ `num_inference_steps` and `timesteps` must be `None`.
134
+
135
+ Returns:
136
+ `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
137
+ second element is the number of inference steps.
138
+ """
139
+ if timesteps is not None and sigmas is not None:
140
+ raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
141
+ if timesteps is not None:
142
+ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
143
+ if not accepts_timesteps:
144
+ raise ValueError(
145
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
146
+ f" timestep schedules. Please check whether you are using the correct scheduler."
147
+ )
148
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
149
+ timesteps = scheduler.timesteps
150
+ num_inference_steps = len(timesteps)
151
+ elif sigmas is not None:
152
+ accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
153
+ if not accept_sigmas:
154
+ raise ValueError(
155
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
156
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
157
+ )
158
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
159
+ timesteps = scheduler.timesteps
160
+ num_inference_steps = len(timesteps)
161
+ else:
162
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
163
+ timesteps = scheduler.timesteps
164
+ return timesteps, num_inference_steps
165
+
166
+
167
+ class StableDiffusion3InpaintPipeline(DiffusionPipeline, SD3LoraLoaderMixin, FromSingleFileMixin, SD3IPAdapterMixin):
168
+ r"""
169
+ Args:
170
+ transformer ([`SD3Transformer2DModel`]):
171
+ Conditional Transformer (MMDiT) architecture to denoise the encoded image latents.
172
+ scheduler ([`FlowMatchEulerDiscreteScheduler`]):
173
+ A scheduler to be used in combination with `transformer` to denoise the encoded image latents.
174
+ vae ([`AutoencoderKL`]):
175
+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
176
+ text_encoder ([`CLIPTextModelWithProjection`]):
177
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection),
178
+ specifically the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant,
179
+ with an additional added projection layer that is initialized with a diagonal matrix with the `hidden_size`
180
+ as its dimension.
181
+ text_encoder_2 ([`CLIPTextModelWithProjection`]):
182
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection),
183
+ specifically the
184
+ [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k)
185
+ variant.
186
+ text_encoder_3 ([`T5EncoderModel`]):
187
+ Frozen text-encoder. Stable Diffusion 3 uses
188
+ [T5](https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5EncoderModel), specifically the
189
+ [t5-v1_1-xxl](https://huggingface.co/google/t5-v1_1-xxl) variant.
190
+ tokenizer (`CLIPTokenizer`):
191
+ Tokenizer of class
192
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
193
+ tokenizer_2 (`CLIPTokenizer`):
194
+ Second Tokenizer of class
195
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
196
+ tokenizer_3 (`T5TokenizerFast`):
197
+ Tokenizer of class
198
+ [T5Tokenizer](https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5Tokenizer).
199
+ image_encoder (`SiglipVisionModel`, *optional*):
200
+ Pre-trained Vision Model for IP Adapter.
201
+ feature_extractor (`SiglipImageProcessor`, *optional*):
202
+ Image processor for IP Adapter.
203
+ """
204
+
205
+ model_cpu_offload_seq = "text_encoder->text_encoder_2->text_encoder_3->image_encoder->transformer->vae"
206
+ _optional_components = ["image_encoder", "feature_extractor"]
207
+ _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds", "negative_pooled_prompt_embeds"]
208
+
209
+ def __init__(
210
+ self,
211
+ transformer: SD3Transformer2DModel,
212
+ scheduler: FlowMatchEulerDiscreteScheduler,
213
+ vae: AutoencoderKL,
214
+ text_encoder: CLIPTextModelWithProjection,
215
+ tokenizer: CLIPTokenizer,
216
+ text_encoder_2: CLIPTextModelWithProjection,
217
+ tokenizer_2: CLIPTokenizer,
218
+ text_encoder_3: T5EncoderModel,
219
+ tokenizer_3: T5TokenizerFast,
220
+ image_encoder: SiglipVisionModel | None = None,
221
+ feature_extractor: SiglipImageProcessor | None = None,
222
+ ):
223
+ super().__init__()
224
+
225
+ self.register_modules(
226
+ vae=vae,
227
+ text_encoder=text_encoder,
228
+ text_encoder_2=text_encoder_2,
229
+ text_encoder_3=text_encoder_3,
230
+ tokenizer=tokenizer,
231
+ tokenizer_2=tokenizer_2,
232
+ tokenizer_3=tokenizer_3,
233
+ transformer=transformer,
234
+ scheduler=scheduler,
235
+ image_encoder=image_encoder,
236
+ feature_extractor=feature_extractor,
237
+ )
238
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
239
+ latent_channels = self.vae.config.latent_channels if getattr(self, "vae", None) else 16
240
+ self.image_processor = VaeImageProcessor(
241
+ vae_scale_factor=self.vae_scale_factor, vae_latent_channels=latent_channels
242
+ )
243
+ self.mask_processor = VaeImageProcessor(
244
+ vae_scale_factor=self.vae_scale_factor,
245
+ vae_latent_channels=latent_channels,
246
+ do_normalize=False,
247
+ do_binarize=True,
248
+ do_convert_grayscale=True,
249
+ )
250
+ self.tokenizer_max_length = (
251
+ self.tokenizer.model_max_length if hasattr(self, "tokenizer") and self.tokenizer is not None else 77
252
+ )
253
+ self.default_sample_size = (
254
+ self.transformer.config.sample_size
255
+ if hasattr(self, "transformer") and self.transformer is not None
256
+ else 128
257
+ )
258
+ self.patch_size = (
259
+ self.transformer.config.patch_size if hasattr(self, "transformer") and self.transformer is not None else 2
260
+ )
261
+
262
+ # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline._get_t5_prompt_embeds
263
+ def _get_t5_prompt_embeds(
264
+ self,
265
+ prompt: str | list[str] = None,
266
+ num_images_per_prompt: int = 1,
267
+ max_sequence_length: int = 256,
268
+ device: torch.device | None = None,
269
+ dtype: torch.dtype | None = None,
270
+ ):
271
+ device = device or self._execution_device
272
+ dtype = dtype or self.text_encoder.dtype
273
+
274
+ prompt = [prompt] if isinstance(prompt, str) else prompt
275
+ batch_size = len(prompt)
276
+
277
+ if self.text_encoder_3 is None:
278
+ return torch.zeros(
279
+ (
280
+ batch_size * num_images_per_prompt,
281
+ max_sequence_length,
282
+ self.transformer.config.joint_attention_dim,
283
+ ),
284
+ device=device,
285
+ dtype=dtype,
286
+ )
287
+
288
+ text_inputs = self.tokenizer_3(
289
+ prompt,
290
+ padding="max_length",
291
+ max_length=max_sequence_length,
292
+ truncation=True,
293
+ add_special_tokens=True,
294
+ return_tensors="pt",
295
+ )
296
+ text_input_ids = text_inputs.input_ids
297
+ untruncated_ids = self.tokenizer_3(prompt, padding="longest", return_tensors="pt").input_ids
298
+
299
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids):
300
+ removed_text = self.tokenizer_3.batch_decode(untruncated_ids[:, self.tokenizer_max_length - 1 : -1])
301
+ logger.warning(
302
+ "The following part of your input was truncated because `max_sequence_length` is set to "
303
+ f" {max_sequence_length} tokens: {removed_text}"
304
+ )
305
+
306
+ prompt_embeds = self.text_encoder_3(text_input_ids.to(device))[0]
307
+
308
+ dtype = self.text_encoder_3.dtype
309
+ prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
310
+
311
+ _, seq_len, _ = prompt_embeds.shape
312
+
313
+ # duplicate text embeddings and attention mask for each generation per prompt, using mps friendly method
314
+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
315
+ prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
316
+
317
+ return prompt_embeds
318
+
319
+ # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline._get_clip_prompt_embeds
320
+ def _get_clip_prompt_embeds(
321
+ self,
322
+ prompt: str | list[str],
323
+ num_images_per_prompt: int = 1,
324
+ device: torch.device | None = None,
325
+ clip_skip: int | None = None,
326
+ clip_model_index: int = 0,
327
+ ):
328
+ device = device or self._execution_device
329
+
330
+ clip_tokenizers = [self.tokenizer, self.tokenizer_2]
331
+ clip_text_encoders = [self.text_encoder, self.text_encoder_2]
332
+
333
+ tokenizer = clip_tokenizers[clip_model_index]
334
+ text_encoder = clip_text_encoders[clip_model_index]
335
+
336
+ prompt = [prompt] if isinstance(prompt, str) else prompt
337
+ batch_size = len(prompt)
338
+
339
+ text_inputs = tokenizer(
340
+ prompt,
341
+ padding="max_length",
342
+ max_length=self.tokenizer_max_length,
343
+ truncation=True,
344
+ return_tensors="pt",
345
+ )
346
+
347
+ text_input_ids = text_inputs.input_ids
348
+ untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
349
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids):
350
+ removed_text = tokenizer.batch_decode(untruncated_ids[:, self.tokenizer_max_length - 1 : -1])
351
+ logger.warning(
352
+ "The following part of your input was truncated because CLIP can only handle sequences up to"
353
+ f" {self.tokenizer_max_length} tokens: {removed_text}"
354
+ )
355
+ prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True)
356
+ pooled_prompt_embeds = prompt_embeds[0]
357
+
358
+ if clip_skip is None:
359
+ prompt_embeds = prompt_embeds.hidden_states[-2]
360
+ else:
361
+ prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)]
362
+
363
+ prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
364
+
365
+ _, seq_len, _ = prompt_embeds.shape
366
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
367
+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
368
+ prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
369
+
370
+ pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt)
371
+ pooled_prompt_embeds = pooled_prompt_embeds.view(batch_size * num_images_per_prompt, -1)
372
+
373
+ return prompt_embeds, pooled_prompt_embeds
374
+
375
+ # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline.encode_prompt
376
+ def encode_prompt(
377
+ self,
378
+ prompt: str | list[str],
379
+ prompt_2: str | list[str],
380
+ prompt_3: str | list[str],
381
+ device: torch.device | None = None,
382
+ num_images_per_prompt: int = 1,
383
+ do_classifier_free_guidance: bool = True,
384
+ negative_prompt: str | list[str] | None = None,
385
+ negative_prompt_2: str | list[str] | None = None,
386
+ negative_prompt_3: str | list[str] | None = None,
387
+ prompt_embeds: torch.FloatTensor | None = None,
388
+ negative_prompt_embeds: torch.FloatTensor | None = None,
389
+ pooled_prompt_embeds: torch.FloatTensor | None = None,
390
+ negative_pooled_prompt_embeds: torch.FloatTensor | None = None,
391
+ clip_skip: int | None = None,
392
+ max_sequence_length: int = 256,
393
+ lora_scale: float | None = None,
394
+ ):
395
+ r"""
396
+
397
+ Args:
398
+ prompt (`str` or `list[str]`, *optional*):
399
+ prompt to be encoded
400
+ prompt_2 (`str` or `list[str]`, *optional*):
401
+ The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
402
+ used in all text-encoders
403
+ prompt_3 (`str` or `list[str]`, *optional*):
404
+ The prompt or prompts to be sent to the `tokenizer_3` and `text_encoder_3`. If not defined, `prompt` is
405
+ used in all text-encoders
406
+ device: (`torch.device`):
407
+ torch device
408
+ num_images_per_prompt (`int`):
409
+ number of images that should be generated per prompt
410
+ do_classifier_free_guidance (`bool`):
411
+ whether to use classifier free guidance or not
412
+ negative_prompt (`str` or `list[str]`, *optional*):
413
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
414
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
415
+ less than `1`).
416
+ negative_prompt_2 (`str` or `list[str]`, *optional*):
417
+ The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
418
+ `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders.
419
+ negative_prompt_3 (`str` or `list[str]`, *optional*):
420
+ The prompt or prompts not to guide the image generation to be sent to `tokenizer_3` and
421
+ `text_encoder_3`. If not defined, `negative_prompt` is used in all the text-encoders.
422
+ prompt_embeds (`torch.FloatTensor`, *optional*):
423
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
424
+ provided, text embeddings will be generated from `prompt` input argument.
425
+ negative_prompt_embeds (`torch.FloatTensor`, *optional*):
426
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
427
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
428
+ argument.
429
+ pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
430
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
431
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
432
+ negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
433
+ Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
434
+ weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
435
+ input argument.
436
+ clip_skip (`int`, *optional*):
437
+ Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
438
+ the output of the pre-final layer will be used for computing the prompt embeddings.
439
+ lora_scale (`float`, *optional*):
440
+ A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
441
+ """
442
+ device = device or self._execution_device
443
+
444
+ # set lora scale so that monkey patched LoRA
445
+ # function of text encoder can correctly access it
446
+ if lora_scale is not None and isinstance(self, SD3LoraLoaderMixin):
447
+ self._lora_scale = lora_scale
448
+
449
+ # dynamically adjust the LoRA scale
450
+ if self.text_encoder is not None and USE_PEFT_BACKEND:
451
+ scale_lora_layers(self.text_encoder, lora_scale)
452
+ if self.text_encoder_2 is not None and USE_PEFT_BACKEND:
453
+ scale_lora_layers(self.text_encoder_2, lora_scale)
454
+
455
+ prompt = [prompt] if isinstance(prompt, str) else prompt
456
+ if prompt is not None:
457
+ batch_size = len(prompt)
458
+ else:
459
+ batch_size = prompt_embeds.shape[0]
460
+
461
+ if prompt_embeds is None:
462
+ prompt_2 = prompt_2 or prompt
463
+ prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2
464
+
465
+ prompt_3 = prompt_3 or prompt
466
+ prompt_3 = [prompt_3] if isinstance(prompt_3, str) else prompt_3
467
+
468
+ prompt_embed, pooled_prompt_embed = self._get_clip_prompt_embeds(
469
+ prompt=prompt,
470
+ device=device,
471
+ num_images_per_prompt=num_images_per_prompt,
472
+ clip_skip=clip_skip,
473
+ clip_model_index=0,
474
+ )
475
+ prompt_2_embed, pooled_prompt_2_embed = self._get_clip_prompt_embeds(
476
+ prompt=prompt_2,
477
+ device=device,
478
+ num_images_per_prompt=num_images_per_prompt,
479
+ clip_skip=clip_skip,
480
+ clip_model_index=1,
481
+ )
482
+ clip_prompt_embeds = torch.cat([prompt_embed, prompt_2_embed], dim=-1)
483
+
484
+ t5_prompt_embed = self._get_t5_prompt_embeds(
485
+ prompt=prompt_3,
486
+ num_images_per_prompt=num_images_per_prompt,
487
+ max_sequence_length=max_sequence_length,
488
+ device=device,
489
+ )
490
+
491
+ clip_prompt_embeds = torch.nn.functional.pad(
492
+ clip_prompt_embeds, (0, t5_prompt_embed.shape[-1] - clip_prompt_embeds.shape[-1])
493
+ )
494
+
495
+ prompt_embeds = torch.cat([clip_prompt_embeds, t5_prompt_embed], dim=-2)
496
+ pooled_prompt_embeds = torch.cat([pooled_prompt_embed, pooled_prompt_2_embed], dim=-1)
497
+
498
+ if do_classifier_free_guidance and negative_prompt_embeds is None:
499
+ negative_prompt = negative_prompt or ""
500
+ negative_prompt_2 = negative_prompt_2 or negative_prompt
501
+ negative_prompt_3 = negative_prompt_3 or negative_prompt
502
+
503
+ # normalize str to list
504
+ negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
505
+ negative_prompt_2 = (
506
+ batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2
507
+ )
508
+ negative_prompt_3 = (
509
+ batch_size * [negative_prompt_3] if isinstance(negative_prompt_3, str) else negative_prompt_3
510
+ )
511
+
512
+ if prompt is not None and type(prompt) is not type(negative_prompt):
513
+ raise TypeError(
514
+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
515
+ f" {type(prompt)}."
516
+ )
517
+ elif batch_size != len(negative_prompt):
518
+ raise ValueError(
519
+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
520
+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
521
+ " the batch size of `prompt`."
522
+ )
523
+
524
+ negative_prompt_embed, negative_pooled_prompt_embed = self._get_clip_prompt_embeds(
525
+ negative_prompt,
526
+ device=device,
527
+ num_images_per_prompt=num_images_per_prompt,
528
+ clip_skip=None,
529
+ clip_model_index=0,
530
+ )
531
+ negative_prompt_2_embed, negative_pooled_prompt_2_embed = self._get_clip_prompt_embeds(
532
+ negative_prompt_2,
533
+ device=device,
534
+ num_images_per_prompt=num_images_per_prompt,
535
+ clip_skip=None,
536
+ clip_model_index=1,
537
+ )
538
+ negative_clip_prompt_embeds = torch.cat([negative_prompt_embed, negative_prompt_2_embed], dim=-1)
539
+
540
+ t5_negative_prompt_embed = self._get_t5_prompt_embeds(
541
+ prompt=negative_prompt_3,
542
+ num_images_per_prompt=num_images_per_prompt,
543
+ max_sequence_length=max_sequence_length,
544
+ device=device,
545
+ )
546
+
547
+ negative_clip_prompt_embeds = torch.nn.functional.pad(
548
+ negative_clip_prompt_embeds,
549
+ (0, t5_negative_prompt_embed.shape[-1] - negative_clip_prompt_embeds.shape[-1]),
550
+ )
551
+
552
+ negative_prompt_embeds = torch.cat([negative_clip_prompt_embeds, t5_negative_prompt_embed], dim=-2)
553
+ negative_pooled_prompt_embeds = torch.cat(
554
+ [negative_pooled_prompt_embed, negative_pooled_prompt_2_embed], dim=-1
555
+ )
556
+
557
+ if self.text_encoder is not None:
558
+ if isinstance(self, SD3LoraLoaderMixin) and USE_PEFT_BACKEND:
559
+ # Retrieve the original scale by scaling back the LoRA layers
560
+ unscale_lora_layers(self.text_encoder, lora_scale)
561
+
562
+ if self.text_encoder_2 is not None:
563
+ if isinstance(self, SD3LoraLoaderMixin) and USE_PEFT_BACKEND:
564
+ # Retrieve the original scale by scaling back the LoRA layers
565
+ unscale_lora_layers(self.text_encoder_2, lora_scale)
566
+
567
+ return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds
568
+
569
+ # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3_img2img.StableDiffusion3Img2ImgPipeline.check_inputs
570
+ def check_inputs(
571
+ self,
572
+ prompt,
573
+ prompt_2,
574
+ prompt_3,
575
+ height,
576
+ width,
577
+ strength,
578
+ negative_prompt=None,
579
+ negative_prompt_2=None,
580
+ negative_prompt_3=None,
581
+ prompt_embeds=None,
582
+ negative_prompt_embeds=None,
583
+ pooled_prompt_embeds=None,
584
+ negative_pooled_prompt_embeds=None,
585
+ callback_on_step_end_tensor_inputs=None,
586
+ max_sequence_length=None,
587
+ ):
588
+ if (
589
+ height % (self.vae_scale_factor * self.patch_size) != 0
590
+ or width % (self.vae_scale_factor * self.patch_size) != 0
591
+ ):
592
+ raise ValueError(
593
+ f"`height` and `width` have to be divisible by {self.vae_scale_factor * self.patch_size} but are {height} and {width}."
594
+ f"You can use height {height - height % (self.vae_scale_factor * self.patch_size)} and width {width - width % (self.vae_scale_factor * self.patch_size)}."
595
+ )
596
+
597
+ if strength < 0 or strength > 1:
598
+ raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}")
599
+
600
+ if callback_on_step_end_tensor_inputs is not None and not all(
601
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
602
+ ):
603
+ raise ValueError(
604
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
605
+ )
606
+
607
+ if prompt is not None and prompt_embeds is not None:
608
+ raise ValueError(
609
+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
610
+ " only forward one of the two."
611
+ )
612
+ elif prompt_2 is not None and prompt_embeds is not None:
613
+ raise ValueError(
614
+ f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
615
+ " only forward one of the two."
616
+ )
617
+ elif prompt_3 is not None and prompt_embeds is not None:
618
+ raise ValueError(
619
+ f"Cannot forward both `prompt_3`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
620
+ " only forward one of the two."
621
+ )
622
+ elif prompt is None and prompt_embeds is None:
623
+ raise ValueError(
624
+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
625
+ )
626
+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
627
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
628
+ elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)):
629
+ raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}")
630
+ elif prompt_3 is not None and (not isinstance(prompt_3, str) and not isinstance(prompt_3, list)):
631
+ raise ValueError(f"`prompt_3` has to be of type `str` or `list` but is {type(prompt_3)}")
632
+
633
+ if negative_prompt is not None and negative_prompt_embeds is not None:
634
+ raise ValueError(
635
+ f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
636
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
637
+ )
638
+ elif negative_prompt_2 is not None and negative_prompt_embeds is not None:
639
+ raise ValueError(
640
+ f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:"
641
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
642
+ )
643
+ elif negative_prompt_3 is not None and negative_prompt_embeds is not None:
644
+ raise ValueError(
645
+ f"Cannot forward both `negative_prompt_3`: {negative_prompt_3} and `negative_prompt_embeds`:"
646
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
647
+ )
648
+
649
+ if prompt_embeds is not None and negative_prompt_embeds is not None:
650
+ if prompt_embeds.shape != negative_prompt_embeds.shape:
651
+ raise ValueError(
652
+ "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"
653
+ f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"
654
+ f" {negative_prompt_embeds.shape}."
655
+ )
656
+
657
+ if prompt_embeds is not None and pooled_prompt_embeds is None:
658
+ raise ValueError(
659
+ "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`."
660
+ )
661
+
662
+ if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None:
663
+ raise ValueError(
664
+ "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`."
665
+ )
666
+
667
+ if max_sequence_length is not None and max_sequence_length > 512:
668
+ raise ValueError(f"`max_sequence_length` cannot be greater than 512 but is {max_sequence_length}")
669
+
670
+ # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3_img2img.StableDiffusion3Img2ImgPipeline.get_timesteps
671
+ def get_timesteps(self, num_inference_steps, strength, device):
672
+ # get the original timestep using init_timestep
673
+ init_timestep = min(num_inference_steps * strength, num_inference_steps)
674
+
675
+ t_start = int(max(num_inference_steps - init_timestep, 0))
676
+ timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :]
677
+ if hasattr(self.scheduler, "set_begin_index"):
678
+ self.scheduler.set_begin_index(t_start * self.scheduler.order)
679
+
680
+ return timesteps, num_inference_steps - t_start
681
+
682
+ def prepare_latents(
683
+ self,
684
+ batch_size,
685
+ num_channels_latents,
686
+ height,
687
+ width,
688
+ dtype,
689
+ device,
690
+ generator,
691
+ latents=None,
692
+ image=None,
693
+ timestep=None,
694
+ is_strength_max=True,
695
+ return_noise=False,
696
+ return_image_latents=False,
697
+ ):
698
+ shape = (
699
+ batch_size,
700
+ num_channels_latents,
701
+ int(height) // self.vae_scale_factor,
702
+ int(width) // self.vae_scale_factor,
703
+ )
704
+ if isinstance(generator, list) and len(generator) != batch_size:
705
+ raise ValueError(
706
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
707
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
708
+ )
709
+
710
+ if (image is None or timestep is None) and not is_strength_max:
711
+ raise ValueError(
712
+ "Since strength < 1. initial latents are to be initialised as a combination of Image + Noise."
713
+ "However, either the image or the noise timestep has not been provided."
714
+ )
715
+
716
+ if return_image_latents or (latents is None and not is_strength_max):
717
+ image = image.to(device=device, dtype=dtype)
718
+
719
+ if image.shape[1] == 16:
720
+ image_latents = image
721
+ else:
722
+ image_latents = self._encode_vae_image(image=image, generator=generator)
723
+ image_latents = image_latents.repeat(batch_size // image_latents.shape[0], 1, 1, 1)
724
+
725
+ if latents is None:
726
+ noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
727
+ # if strength is 1. then initialise the latents to noise, else initial to image + noise
728
+ latents = noise if is_strength_max else self.scheduler.scale_noise(image_latents, timestep, noise)
729
+ else:
730
+ noise = latents.to(device)
731
+ latents = noise
732
+
733
+ outputs = (latents,)
734
+
735
+ if return_noise:
736
+ outputs += (noise,)
737
+
738
+ if return_image_latents:
739
+ outputs += (image_latents,)
740
+
741
+ return outputs
742
+
743
+ def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator):
744
+ if isinstance(generator, list):
745
+ image_latents = [
746
+ retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i])
747
+ for i in range(image.shape[0])
748
+ ]
749
+ image_latents = torch.cat(image_latents, dim=0)
750
+ else:
751
+ image_latents = retrieve_latents(self.vae.encode(image), generator=generator)
752
+
753
+ image_latents = (image_latents - self.vae.config.shift_factor) * self.vae.config.scaling_factor
754
+
755
+ return image_latents
756
+
757
+ def prepare_mask_latents(
758
+ self,
759
+ mask,
760
+ masked_image,
761
+ batch_size,
762
+ num_images_per_prompt,
763
+ height,
764
+ width,
765
+ dtype,
766
+ device,
767
+ generator,
768
+ do_classifier_free_guidance,
769
+ ):
770
+ # resize the mask to latents shape as we concatenate the mask to the latents
771
+ # we do that before converting to dtype to avoid breaking in case we're using cpu_offload
772
+ # and half precision
773
+ mask = torch.nn.functional.interpolate(
774
+ mask, size=(height // self.vae_scale_factor, width // self.vae_scale_factor)
775
+ )
776
+ mask = mask.to(device=device, dtype=dtype)
777
+
778
+ batch_size = batch_size * num_images_per_prompt
779
+
780
+ masked_image = masked_image.to(device=device, dtype=dtype)
781
+
782
+ if masked_image.shape[1] == 16:
783
+ masked_image_latents = masked_image
784
+ else:
785
+ masked_image_latents = retrieve_latents(self.vae.encode(masked_image), generator=generator)
786
+
787
+ masked_image_latents = (masked_image_latents - self.vae.config.shift_factor) * self.vae.config.scaling_factor
788
+
789
+ # duplicate mask and masked_image_latents for each generation per prompt, using mps friendly method
790
+ if mask.shape[0] < batch_size:
791
+ if not batch_size % mask.shape[0] == 0:
792
+ raise ValueError(
793
+ "The passed mask and the required batch size don't match. Masks are supposed to be duplicated to"
794
+ f" a total batch size of {batch_size}, but {mask.shape[0]} masks were passed. Make sure the number"
795
+ " of masks that you pass is divisible by the total requested batch size."
796
+ )
797
+ mask = mask.repeat(batch_size // mask.shape[0], 1, 1, 1)
798
+ if masked_image_latents.shape[0] < batch_size:
799
+ if not batch_size % masked_image_latents.shape[0] == 0:
800
+ raise ValueError(
801
+ "The passed images and the required batch size don't match. Images are supposed to be duplicated"
802
+ f" to a total batch size of {batch_size}, but {masked_image_latents.shape[0]} images were passed."
803
+ " Make sure the number of images that you pass is divisible by the total requested batch size."
804
+ )
805
+ masked_image_latents = masked_image_latents.repeat(batch_size // masked_image_latents.shape[0], 1, 1, 1)
806
+
807
+ mask = torch.cat([mask] * 2) if do_classifier_free_guidance else mask
808
+ masked_image_latents = (
809
+ torch.cat([masked_image_latents] * 2) if do_classifier_free_guidance else masked_image_latents
810
+ )
811
+
812
+ # aligning device to prevent device errors when concating it with the latent model input
813
+ masked_image_latents = masked_image_latents.to(device=device, dtype=dtype)
814
+ return mask, masked_image_latents
815
+
816
+ @property
817
+ def guidance_scale(self):
818
+ return self._guidance_scale
819
+
820
+ @property
821
+ def clip_skip(self):
822
+ return self._clip_skip
823
+
824
+ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
825
+ # of the Imagen paper: https://huggingface.co/papers/2205.11487 . `guidance_scale = 1`
826
+ # corresponds to doing no classifier free guidance.
827
+ @property
828
+ def do_classifier_free_guidance(self):
829
+ return self._guidance_scale > 1
830
+
831
+ @property
832
+ def joint_attention_kwargs(self):
833
+ return self._joint_attention_kwargs
834
+
835
+ @property
836
+ def num_timesteps(self):
837
+ return self._num_timesteps
838
+
839
+ @property
840
+ def interrupt(self):
841
+ return self._interrupt
842
+
843
+ # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline.encode_image
844
+ def encode_image(self, image: PipelineImageInput, device: torch.device) -> torch.Tensor:
845
+ """Encodes the given image into a feature representation using a pre-trained image encoder.
846
+
847
+ Args:
848
+ image (`PipelineImageInput`):
849
+ Input image to be encoded.
850
+ device: (`torch.device`):
851
+ Torch device.
852
+
853
+ Returns:
854
+ `torch.Tensor`: The encoded image feature representation.
855
+ """
856
+ if not isinstance(image, torch.Tensor):
857
+ image = self.feature_extractor(image, return_tensors="pt").pixel_values
858
+
859
+ image = image.to(device=device, dtype=self.dtype)
860
+
861
+ return self.image_encoder(image, output_hidden_states=True).hidden_states[-2]
862
+
863
+ # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline.prepare_ip_adapter_image_embeds
864
+ def prepare_ip_adapter_image_embeds(
865
+ self,
866
+ ip_adapter_image: PipelineImageInput | None = None,
867
+ ip_adapter_image_embeds: torch.Tensor | None = None,
868
+ device: torch.device | None = None,
869
+ num_images_per_prompt: int = 1,
870
+ do_classifier_free_guidance: bool = True,
871
+ ) -> torch.Tensor:
872
+ """Prepares image embeddings for use in the IP-Adapter.
873
+
874
+ Either `ip_adapter_image` or `ip_adapter_image_embeds` must be passed.
875
+
876
+ Args:
877
+ ip_adapter_image (`PipelineImageInput`, *optional*):
878
+ The input image to extract features from for IP-Adapter.
879
+ ip_adapter_image_embeds (`torch.Tensor`, *optional*):
880
+ Precomputed image embeddings.
881
+ device: (`torch.device`, *optional*):
882
+ Torch device.
883
+ num_images_per_prompt (`int`, defaults to 1):
884
+ Number of images that should be generated per prompt.
885
+ do_classifier_free_guidance (`bool`, defaults to True):
886
+ Whether to use classifier free guidance or not.
887
+ """
888
+ device = device or self._execution_device
889
+
890
+ if ip_adapter_image_embeds is not None:
891
+ if do_classifier_free_guidance:
892
+ single_negative_image_embeds, single_image_embeds = ip_adapter_image_embeds.chunk(2)
893
+ else:
894
+ single_image_embeds = ip_adapter_image_embeds
895
+ elif ip_adapter_image is not None:
896
+ single_image_embeds = self.encode_image(ip_adapter_image, device)
897
+ if do_classifier_free_guidance:
898
+ single_negative_image_embeds = torch.zeros_like(single_image_embeds)
899
+ else:
900
+ raise ValueError("Neither `ip_adapter_image_embeds` or `ip_adapter_image_embeds` were provided.")
901
+
902
+ image_embeds = torch.cat([single_image_embeds] * num_images_per_prompt, dim=0)
903
+
904
+ if do_classifier_free_guidance:
905
+ negative_image_embeds = torch.cat([single_negative_image_embeds] * num_images_per_prompt, dim=0)
906
+ image_embeds = torch.cat([negative_image_embeds, image_embeds], dim=0)
907
+
908
+ return image_embeds.to(device=device)
909
+
910
+ # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3.StableDiffusion3Pipeline.enable_sequential_cpu_offload
911
+ def enable_sequential_cpu_offload(self, *args, **kwargs):
912
+ if self.image_encoder is not None and "image_encoder" not in self._exclude_from_cpu_offload:
913
+ logger.warning(
914
+ "`pipe.enable_sequential_cpu_offload()` might fail for `image_encoder` if it uses "
915
+ "`torch.nn.MultiheadAttention`. You can exclude `image_encoder` from CPU offloading by calling "
916
+ "`pipe._exclude_from_cpu_offload.append('image_encoder')` before `pipe.enable_sequential_cpu_offload()`."
917
+ )
918
+
919
+ super().enable_sequential_cpu_offload(*args, **kwargs)
920
+
921
+ @torch.no_grad()
922
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
923
+ def __call__(
924
+ self,
925
+ prompt: str | list[str] = None,
926
+ prompt_2: str | list[str] | None = None,
927
+ prompt_3: str | list[str] | None = None,
928
+ image: PipelineImageInput = None,
929
+ mask_image: PipelineImageInput = None,
930
+ masked_image_latents: PipelineImageInput = None,
931
+ height: int = None,
932
+ width: int = None,
933
+ padding_mask_crop: int | None = None,
934
+ strength: float = 0.6,
935
+ num_inference_steps: int = 50,
936
+ sigmas: list[float] | None = None,
937
+ guidance_scale: float = 7.0,
938
+ negative_prompt: str | list[str] | None = None,
939
+ negative_prompt_2: str | list[str] | None = None,
940
+ negative_prompt_3: str | list[str] | None = None,
941
+ num_images_per_prompt: int | None = 1,
942
+ generator: torch.Generator | list[torch.Generator] | None = None,
943
+ latents: torch.Tensor | None = None,
944
+ prompt_embeds: torch.Tensor | None = None,
945
+ negative_prompt_embeds: torch.Tensor | None = None,
946
+ pooled_prompt_embeds: torch.Tensor | None = None,
947
+ negative_pooled_prompt_embeds: torch.Tensor | None = None,
948
+ ip_adapter_image: PipelineImageInput | None = None,
949
+ ip_adapter_image_embeds: torch.Tensor | None = None,
950
+ output_type: str | None = "pil",
951
+ return_dict: bool = True,
952
+ joint_attention_kwargs: dict[str, Any] | None = None,
953
+ clip_skip: int | None = None,
954
+ callback_on_step_end: Callable[[int, int], None] | None = None,
955
+ callback_on_step_end_tensor_inputs: list[str] = ["latents"],
956
+ max_sequence_length: int = 256,
957
+ mu: float | None = None,
958
+ ):
959
+ r"""
960
+ Function invoked when calling the pipeline for generation.
961
+
962
+ Args:
963
+ prompt (`str` or `list[str]`, *optional*):
964
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
965
+ instead.
966
+ prompt_2 (`str` or `list[str]`, *optional*):
967
+ The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
968
+ will be used instead
969
+ prompt_3 (`str` or `list[str]`, *optional*):
970
+ The prompt or prompts to be sent to `tokenizer_3` and `text_encoder_3`. If not defined, `prompt` is
971
+ will be used instead
972
+ image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`):
973
+ `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both
974
+ numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list
975
+ or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a
976
+ list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image
977
+ latents as `image`, but if passing latents directly it is not encoded again.
978
+ mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`):
979
+ `Image`, numpy array or tensor representing an image batch to mask `image`. White pixels in the mask
980
+ are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a
981
+ single channel (luminance) before use. If it's a numpy array or pytorch tensor, it should contain one
982
+ color channel (L) instead of 3, so the expected shape for pytorch tensor would be `(B, 1, H, W)`, `(B,
983
+ H, W)`, `(1, H, W)`, `(H, W)`. And for numpy array would be for `(B, H, W, 1)`, `(B, H, W)`, `(H, W,
984
+ 1)`, or `(H, W)`.
985
+ masked_image_latents (`torch.Tensor`, *optional*):
986
+ Pre-encoded latent of the masked image (for inpainting). If not provided, the masked image latents are
987
+ generated from `mask_image` and `image`.
988
+ height (`int`, *optional*, defaults to self.transformer.config.sample_size * self.vae_scale_factor):
989
+ The height in pixels of the generated image. This is set to 1024 by default for the best results.
990
+ width (`int`, *optional*, defaults to self.transformer.config.sample_size * self.vae_scale_factor):
991
+ The width in pixels of the generated image. This is set to 1024 by default for the best results.
992
+ padding_mask_crop (`int`, *optional*, defaults to `None`):
993
+ The size of margin in the crop to be applied to the image and masking. If `None`, no crop is applied to
994
+ image and mask_image. If `padding_mask_crop` is not `None`, it will first find a rectangular region
995
+ with the same aspect ration of the image and contains all masked area, and then expand that area based
996
+ on `padding_mask_crop`. The image and mask_image will then be cropped based on the expanded area before
997
+ resizing to the original image size for inpainting. This is useful when the masked area is small while
998
+ the image is large and contain information irrelevant for inpainting, such as background.
999
+ strength (`float`, *optional*, defaults to 1.0):
1000
+ Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a
1001
+ starting point and more noise is added the higher the `strength`. The number of denoising steps depends
1002
+ on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising
1003
+ process runs for the full number of iterations specified in `num_inference_steps`. A value of 1
1004
+ essentially ignores `image`.
1005
+ num_inference_steps (`int`, *optional*, defaults to 50):
1006
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
1007
+ expense of slower inference.
1008
+ sigmas (`list[float]`, *optional*):
1009
+ Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
1010
+ their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
1011
+ will be used.
1012
+ guidance_scale (`float`, *optional*, defaults to 7.0):
1013
+ Guidance scale as defined in [Classifier-Free Diffusion
1014
+ Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2.
1015
+ of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
1016
+ `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to
1017
+ the text `prompt`, usually at the expense of lower image quality.
1018
+ negative_prompt (`str` or `list[str]`, *optional*):
1019
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
1020
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
1021
+ less than `1`).
1022
+ negative_prompt_2 (`str` or `list[str]`, *optional*):
1023
+ The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
1024
+ `text_encoder_2`. If not defined, `negative_prompt` is used instead
1025
+ negative_prompt_3 (`str` or `list[str]`, *optional*):
1026
+ The prompt or prompts not to guide the image generation to be sent to `tokenizer_3` and
1027
+ `text_encoder_3`. If not defined, `negative_prompt` is used instead
1028
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
1029
+ The number of images to generate per prompt.
1030
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
1031
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
1032
+ to make generation deterministic.
1033
+ latents (`torch.FloatTensor`, *optional*):
1034
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
1035
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
1036
+ tensor will be generated by sampling using the supplied random `generator`.
1037
+ prompt_embeds (`torch.FloatTensor`, *optional*):
1038
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
1039
+ provided, text embeddings will be generated from `prompt` input argument.
1040
+ negative_prompt_embeds (`torch.FloatTensor`, *optional*):
1041
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
1042
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
1043
+ argument.
1044
+ pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
1045
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
1046
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
1047
+ negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
1048
+ Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
1049
+ weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
1050
+ input argument.
1051
+ ip_adapter_image (`PipelineImageInput`, *optional*):
1052
+ Optional image input to work with IP Adapters.
1053
+ ip_adapter_image_embeds (`torch.Tensor`, *optional*):
1054
+ Pre-generated image embeddings for IP-Adapter. Should be a tensor of shape `(batch_size, num_images,
1055
+ emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to
1056
+ `True`. If not provided, embeddings are computed from the `ip_adapter_image` input argument.
1057
+ output_type (`str`, *optional*, defaults to `"pil"`):
1058
+ The output format of the generate image. Choose between
1059
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
1060
+ return_dict (`bool`, *optional*, defaults to `True`):
1061
+ Whether or not to return a [`~pipelines.stable_diffusion_3.StableDiffusion3PipelineOutput`] instead of
1062
+ a plain tuple.
1063
+ joint_attention_kwargs (`dict`, *optional*):
1064
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
1065
+ `self.processor` in
1066
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
1067
+ clip_skip (`int`, *optional*):
1068
+ Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
1069
+ the output of the pre-final layer will be used for computing the prompt embeddings.
1070
+ callback_on_step_end (`Callable`, *optional*):
1071
+ A function that calls at the end of each denoising steps during the inference. The function is called
1072
+ with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
1073
+ callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
1074
+ `callback_on_step_end_tensor_inputs`.
1075
+ callback_on_step_end_tensor_inputs (`list`, *optional*):
1076
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
1077
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
1078
+ `._callback_tensor_inputs` attribute of your pipeline class.
1079
+ max_sequence_length (`int` defaults to 256): Maximum sequence length to use with the `prompt`.
1080
+ mu (`float`, *optional*): `mu` value used for `dynamic_shifting`.
1081
+
1082
+ Examples:
1083
+
1084
+ Returns:
1085
+ [`~pipelines.stable_diffusion_3.StableDiffusion3PipelineOutput`] or `tuple`:
1086
+ [`~pipelines.stable_diffusion_3.StableDiffusion3PipelineOutput`] if `return_dict` is True, otherwise a
1087
+ `tuple`. When returning a tuple, the first element is a list with the generated images.
1088
+ """
1089
+
1090
+ if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
1091
+ callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
1092
+
1093
+ height = height or self.transformer.config.sample_size * self.vae_scale_factor
1094
+ width = width or self.transformer.config.sample_size * self.vae_scale_factor
1095
+
1096
+ # 1. Check inputs. Raise error if not correct
1097
+ self.check_inputs(
1098
+ prompt,
1099
+ prompt_2,
1100
+ prompt_3,
1101
+ height,
1102
+ width,
1103
+ strength,
1104
+ negative_prompt=negative_prompt,
1105
+ negative_prompt_2=negative_prompt_2,
1106
+ negative_prompt_3=negative_prompt_3,
1107
+ prompt_embeds=prompt_embeds,
1108
+ negative_prompt_embeds=negative_prompt_embeds,
1109
+ pooled_prompt_embeds=pooled_prompt_embeds,
1110
+ negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
1111
+ callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs,
1112
+ max_sequence_length=max_sequence_length,
1113
+ )
1114
+
1115
+ self._guidance_scale = guidance_scale
1116
+ self._clip_skip = clip_skip
1117
+ self._joint_attention_kwargs = joint_attention_kwargs
1118
+ self._interrupt = False
1119
+
1120
+ # 2. Define call parameters
1121
+ if prompt is not None and isinstance(prompt, str):
1122
+ batch_size = 1
1123
+ elif prompt is not None and isinstance(prompt, list):
1124
+ batch_size = len(prompt)
1125
+ else:
1126
+ batch_size = prompt_embeds.shape[0]
1127
+
1128
+ device = self._execution_device
1129
+
1130
+ (
1131
+ prompt_embeds,
1132
+ negative_prompt_embeds,
1133
+ pooled_prompt_embeds,
1134
+ negative_pooled_prompt_embeds,
1135
+ ) = self.encode_prompt(
1136
+ prompt=prompt,
1137
+ prompt_2=prompt_2,
1138
+ prompt_3=prompt_3,
1139
+ negative_prompt=negative_prompt,
1140
+ negative_prompt_2=negative_prompt_2,
1141
+ negative_prompt_3=negative_prompt_3,
1142
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
1143
+ prompt_embeds=prompt_embeds,
1144
+ negative_prompt_embeds=negative_prompt_embeds,
1145
+ pooled_prompt_embeds=pooled_prompt_embeds,
1146
+ negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
1147
+ device=device,
1148
+ clip_skip=self.clip_skip,
1149
+ num_images_per_prompt=num_images_per_prompt,
1150
+ max_sequence_length=max_sequence_length,
1151
+ )
1152
+
1153
+ if self.do_classifier_free_guidance:
1154
+ prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
1155
+ pooled_prompt_embeds = torch.cat([negative_pooled_prompt_embeds, pooled_prompt_embeds], dim=0)
1156
+
1157
+ # 3. Prepare timesteps
1158
+ scheduler_kwargs = {}
1159
+ if self.scheduler.config.get("use_dynamic_shifting", None) and mu is None:
1160
+ image_seq_len = (int(height) // self.vae_scale_factor // self.transformer.config.patch_size) * (
1161
+ int(width) // self.vae_scale_factor // self.transformer.config.patch_size
1162
+ )
1163
+ mu = calculate_shift(
1164
+ image_seq_len,
1165
+ self.scheduler.config.get("base_image_seq_len", 256),
1166
+ self.scheduler.config.get("max_image_seq_len", 4096),
1167
+ self.scheduler.config.get("base_shift", 0.5),
1168
+ self.scheduler.config.get("max_shift", 1.16),
1169
+ )
1170
+ scheduler_kwargs["mu"] = mu
1171
+ elif mu is not None:
1172
+ scheduler_kwargs["mu"] = mu
1173
+
1174
+ if XLA_AVAILABLE:
1175
+ timestep_device = "cpu"
1176
+ else:
1177
+ timestep_device = device
1178
+ timesteps, num_inference_steps = retrieve_timesteps(
1179
+ self.scheduler, num_inference_steps, timestep_device, sigmas=sigmas, **scheduler_kwargs
1180
+ )
1181
+ timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
1182
+ # check that number of inference steps is not < 1 - as this doesn't make sense
1183
+ if num_inference_steps < 1:
1184
+ raise ValueError(
1185
+ f"After adjusting the num_inference_steps by strength parameter: {strength}, the number of pipeline"
1186
+ f"steps is {num_inference_steps} which is < 1 and not appropriate for this pipeline."
1187
+ )
1188
+ latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
1189
+
1190
+ # create a boolean to check if the strength is set to 1. if so then initialise the latents with pure noise
1191
+ is_strength_max = strength == 1.0
1192
+
1193
+ # 4. Preprocess mask and image
1194
+ if padding_mask_crop is not None:
1195
+ crops_coords = self.mask_processor.get_crop_region(mask_image, width, height, pad=padding_mask_crop)
1196
+ resize_mode = "fill"
1197
+ else:
1198
+ crops_coords = None
1199
+ resize_mode = "default"
1200
+
1201
+ original_image = image
1202
+ init_image = self.image_processor.preprocess(
1203
+ image, height=height, width=width, crops_coords=crops_coords, resize_mode=resize_mode
1204
+ )
1205
+ init_image = init_image.to(dtype=torch.float32)
1206
+
1207
+ # 5. Prepare latent variables
1208
+ num_channels_latents = self.vae.config.latent_channels
1209
+ num_channels_transformer = self.transformer.config.in_channels
1210
+ return_image_latents = num_channels_transformer == 16
1211
+
1212
+ latents_outputs = self.prepare_latents(
1213
+ batch_size * num_images_per_prompt,
1214
+ num_channels_latents,
1215
+ height,
1216
+ width,
1217
+ prompt_embeds.dtype,
1218
+ device,
1219
+ generator,
1220
+ latents,
1221
+ image=init_image,
1222
+ timestep=latent_timestep,
1223
+ is_strength_max=is_strength_max,
1224
+ return_noise=True,
1225
+ return_image_latents=return_image_latents,
1226
+ )
1227
+
1228
+ if return_image_latents:
1229
+ latents, noise, image_latents = latents_outputs
1230
+ else:
1231
+ latents, noise = latents_outputs
1232
+
1233
+ # 6. Prepare mask latent variables
1234
+ mask_condition = self.mask_processor.preprocess(
1235
+ mask_image, height=height, width=width, resize_mode=resize_mode, crops_coords=crops_coords
1236
+ )
1237
+
1238
+ if masked_image_latents is None:
1239
+ masked_image = init_image * (mask_condition < 0.5)
1240
+ else:
1241
+ masked_image = masked_image_latents
1242
+
1243
+ mask, masked_image_latents = self.prepare_mask_latents(
1244
+ mask_condition,
1245
+ masked_image,
1246
+ batch_size,
1247
+ num_images_per_prompt,
1248
+ height,
1249
+ width,
1250
+ prompt_embeds.dtype,
1251
+ device,
1252
+ generator,
1253
+ self.do_classifier_free_guidance,
1254
+ )
1255
+
1256
+ # match the inpainting pipeline and will be updated with input + mask inpainting model later
1257
+ if num_channels_transformer == 33:
1258
+ # default case for stable-diffusion-v1-5/stable-diffusion-inpainting
1259
+ num_channels_mask = mask.shape[1]
1260
+ num_channels_masked_image = masked_image_latents.shape[1]
1261
+ if (
1262
+ num_channels_latents + num_channels_mask + num_channels_masked_image
1263
+ != self.transformer.config.in_channels
1264
+ ):
1265
+ raise ValueError(
1266
+ f"Incorrect configuration settings! The config of `pipeline.transformer`: {self.transformer.config} expects"
1267
+ f" {self.transformer.config.in_channels} but received `num_channels_latents`: {num_channels_latents} +"
1268
+ f" `num_channels_mask`: {num_channels_mask} + `num_channels_masked_image`: {num_channels_masked_image}"
1269
+ f" = {num_channels_latents + num_channels_masked_image + num_channels_mask}. Please verify the config of"
1270
+ " `pipeline.transformer` or your `mask_image` or `image` input."
1271
+ )
1272
+ elif num_channels_transformer != 16:
1273
+ raise ValueError(
1274
+ f"The transformer {self.transformer.__class__} should have 16 input channels or 33 input channels, not {self.transformer.config.in_channels}."
1275
+ )
1276
+
1277
+ # 7. Prepare image embeddings
1278
+ if (ip_adapter_image is not None and self.is_ip_adapter_active) or ip_adapter_image_embeds is not None:
1279
+ ip_adapter_image_embeds = self.prepare_ip_adapter_image_embeds(
1280
+ ip_adapter_image,
1281
+ ip_adapter_image_embeds,
1282
+ device,
1283
+ batch_size * num_images_per_prompt,
1284
+ self.do_classifier_free_guidance,
1285
+ )
1286
+
1287
+ if self.joint_attention_kwargs is None:
1288
+ self._joint_attention_kwargs = {"ip_adapter_image_embeds": ip_adapter_image_embeds}
1289
+ else:
1290
+ self._joint_attention_kwargs.update(ip_adapter_image_embeds=ip_adapter_image_embeds)
1291
+
1292
+ # 8. Denoising loop
1293
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
1294
+ self._num_timesteps = len(timesteps)
1295
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
1296
+ for i, t in enumerate(timesteps):
1297
+ if self.interrupt:
1298
+ continue
1299
+
1300
+ # expand the latents if we are doing classifier free guidance
1301
+ latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
1302
+ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
1303
+ timestep = t.expand(latent_model_input.shape[0])
1304
+
1305
+ if num_channels_transformer == 33:
1306
+ latent_model_input = torch.cat([latent_model_input, mask, masked_image_latents], dim=1)
1307
+
1308
+ noise_pred = self.transformer(
1309
+ hidden_states=latent_model_input,
1310
+ timestep=timestep,
1311
+ encoder_hidden_states=prompt_embeds,
1312
+ pooled_projections=pooled_prompt_embeds,
1313
+ joint_attention_kwargs=self.joint_attention_kwargs,
1314
+ return_dict=False,
1315
+ )[0]
1316
+
1317
+ # perform guidance
1318
+ if self.do_classifier_free_guidance:
1319
+ noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
1320
+ noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)
1321
+
1322
+ # compute the previous noisy sample x_t -> x_t-1
1323
+ latents_dtype = latents.dtype
1324
+ latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
1325
+ if num_channels_transformer == 16:
1326
+ init_latents_proper = image_latents
1327
+ if self.do_classifier_free_guidance:
1328
+ init_mask, _ = mask.chunk(2)
1329
+ else:
1330
+ init_mask = mask
1331
+
1332
+ if i < len(timesteps) - 1:
1333
+ noise_timestep = timesteps[i + 1]
1334
+ init_latents_proper = self.scheduler.scale_noise(
1335
+ init_latents_proper, torch.tensor([noise_timestep]), noise
1336
+ )
1337
+
1338
+ latents = (1 - init_mask) * init_latents_proper + init_mask * latents
1339
+
1340
+ if latents.dtype != latents_dtype:
1341
+ if torch.backends.mps.is_available():
1342
+ # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
1343
+ latents = latents.to(latents_dtype)
1344
+
1345
+ if callback_on_step_end is not None:
1346
+ callback_kwargs = {}
1347
+ for k in callback_on_step_end_tensor_inputs:
1348
+ callback_kwargs[k] = locals()[k]
1349
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
1350
+
1351
+ latents = callback_outputs.pop("latents", latents)
1352
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
1353
+ negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
1354
+ negative_pooled_prompt_embeds = callback_outputs.pop(
1355
+ "negative_pooled_prompt_embeds", negative_pooled_prompt_embeds
1356
+ )
1357
+ mask = callback_outputs.pop("mask", mask)
1358
+ masked_image_latents = callback_outputs.pop("masked_image_latents", masked_image_latents)
1359
+
1360
+ # call the callback, if provided
1361
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
1362
+ progress_bar.update()
1363
+
1364
+ if XLA_AVAILABLE:
1365
+ xm.mark_step()
1366
+
1367
+ if not output_type == "latent":
1368
+ image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[
1369
+ 0
1370
+ ]
1371
+ else:
1372
+ image = latents
1373
+
1374
+ do_denormalize = [True] * image.shape[0]
1375
+
1376
+ image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize)
1377
+
1378
+ if padding_mask_crop is not None:
1379
+ image = [self.image_processor.apply_overlay(mask_image, original_image, i, crops_coords) for i in image]
1380
+
1381
+ # Offload all models
1382
+ self.maybe_free_model_hooks()
1383
+
1384
+ if not return_dict:
1385
+ return (image,)
1386
+
1387
+ return StableDiffusion3PipelineOutput(images=image)
western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_xl/__init__.py ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import TYPE_CHECKING
2
+
3
+ from ...utils import (
4
+ DIFFUSERS_SLOW_IMPORT,
5
+ OptionalDependencyNotAvailable,
6
+ _LazyModule,
7
+ get_objects_from_module,
8
+ is_torch_available,
9
+ is_transformers_available,
10
+ is_transformers_flax_compatible,
11
+ )
12
+
13
+
14
+ _dummy_objects = {}
15
+ _additional_imports = {}
16
+ _import_structure = {"pipeline_output": ["StableDiffusionXLPipelineOutput"]}
17
+
18
+ if is_transformers_flax_compatible():
19
+ _import_structure["pipeline_output"].extend(["FlaxStableDiffusionXLPipelineOutput"])
20
+ try:
21
+ if not (is_transformers_available() and is_torch_available()):
22
+ raise OptionalDependencyNotAvailable()
23
+ except OptionalDependencyNotAvailable:
24
+ from ...utils import dummy_torch_and_transformers_objects # noqa F403
25
+
26
+ _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects))
27
+ else:
28
+ _import_structure["pipeline_stable_diffusion_xl"] = ["StableDiffusionXLPipeline"]
29
+ _import_structure["pipeline_stable_diffusion_xl_img2img"] = ["StableDiffusionXLImg2ImgPipeline"]
30
+ _import_structure["pipeline_stable_diffusion_xl_inpaint"] = ["StableDiffusionXLInpaintPipeline"]
31
+ _import_structure["pipeline_stable_diffusion_xl_instruct_pix2pix"] = ["StableDiffusionXLInstructPix2PixPipeline"]
32
+
33
+ if is_transformers_flax_compatible():
34
+ from ...schedulers.scheduling_pndm_flax import PNDMSchedulerState
35
+
36
+ _additional_imports.update({"PNDMSchedulerState": PNDMSchedulerState})
37
+ _import_structure["pipeline_flax_stable_diffusion_xl"] = ["FlaxStableDiffusionXLPipeline"]
38
+
39
+
40
+ if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT:
41
+ try:
42
+ if not (is_transformers_available() and is_torch_available()):
43
+ raise OptionalDependencyNotAvailable()
44
+ except OptionalDependencyNotAvailable:
45
+ from ...utils.dummy_torch_and_transformers_objects import * # noqa F403
46
+ else:
47
+ from .pipeline_stable_diffusion_xl import StableDiffusionXLPipeline
48
+ from .pipeline_stable_diffusion_xl_img2img import StableDiffusionXLImg2ImgPipeline
49
+ from .pipeline_stable_diffusion_xl_inpaint import StableDiffusionXLInpaintPipeline
50
+ from .pipeline_stable_diffusion_xl_instruct_pix2pix import StableDiffusionXLInstructPix2PixPipeline
51
+
52
+ try:
53
+ if not is_transformers_flax_compatible():
54
+ raise OptionalDependencyNotAvailable()
55
+ except OptionalDependencyNotAvailable:
56
+ from ...utils.dummy_flax_objects import *
57
+ else:
58
+ from .pipeline_flax_stable_diffusion_xl import (
59
+ FlaxStableDiffusionXLPipeline,
60
+ )
61
+ from .pipeline_output import FlaxStableDiffusionXLPipelineOutput
62
+
63
+ else:
64
+ import sys
65
+
66
+ sys.modules[__name__] = _LazyModule(
67
+ __name__,
68
+ globals()["__file__"],
69
+ _import_structure,
70
+ module_spec=__spec__,
71
+ )
72
+
73
+ for name, value in _dummy_objects.items():
74
+ setattr(sys.modules[__name__], name, value)
75
+ for name, value in _additional_imports.items():
76
+ setattr(sys.modules[__name__], name, value)
western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_xl/pipeline_flax_stable_diffusion_xl.py ADDED
@@ -0,0 +1,307 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2026 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from __future__ import annotations
16
+
17
+ from functools import partial
18
+
19
+ import jax
20
+ import jax.numpy as jnp
21
+ from flax.core.frozen_dict import FrozenDict
22
+ from transformers import CLIPTokenizer, FlaxCLIPTextModel
23
+
24
+ from diffusers.utils import logging
25
+
26
+ from ...models import FlaxAutoencoderKL, FlaxUNet2DConditionModel
27
+ from ...schedulers import (
28
+ FlaxDDIMScheduler,
29
+ FlaxDPMSolverMultistepScheduler,
30
+ FlaxLMSDiscreteScheduler,
31
+ FlaxPNDMScheduler,
32
+ )
33
+ from ..pipeline_flax_utils import FlaxDiffusionPipeline
34
+ from .pipeline_output import FlaxStableDiffusionXLPipelineOutput
35
+
36
+
37
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
38
+
39
+ # Set to True to use python for loop instead of jax.fori_loop for easier debugging
40
+ DEBUG = False
41
+
42
+
43
+ class FlaxStableDiffusionXLPipeline(FlaxDiffusionPipeline):
44
+ def __init__(
45
+ self,
46
+ text_encoder: FlaxCLIPTextModel,
47
+ text_encoder_2: FlaxCLIPTextModel,
48
+ vae: FlaxAutoencoderKL,
49
+ tokenizer: CLIPTokenizer,
50
+ tokenizer_2: CLIPTokenizer,
51
+ unet: FlaxUNet2DConditionModel,
52
+ scheduler: FlaxDDIMScheduler | FlaxPNDMScheduler | FlaxLMSDiscreteScheduler | FlaxDPMSolverMultistepScheduler,
53
+ dtype: jnp.dtype = jnp.float32,
54
+ ):
55
+ super().__init__()
56
+ self.dtype = dtype
57
+
58
+ self.register_modules(
59
+ vae=vae,
60
+ text_encoder=text_encoder,
61
+ text_encoder_2=text_encoder_2,
62
+ tokenizer=tokenizer,
63
+ tokenizer_2=tokenizer_2,
64
+ unet=unet,
65
+ scheduler=scheduler,
66
+ )
67
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
68
+
69
+ def prepare_inputs(self, prompt: str | list[str]):
70
+ if not isinstance(prompt, (str, list)):
71
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
72
+
73
+ # Assume we have the two encoders
74
+ inputs = []
75
+ for tokenizer in [self.tokenizer, self.tokenizer_2]:
76
+ text_inputs = tokenizer(
77
+ prompt,
78
+ padding="max_length",
79
+ max_length=self.tokenizer.model_max_length,
80
+ truncation=True,
81
+ return_tensors="np",
82
+ )
83
+ inputs.append(text_inputs.input_ids)
84
+ inputs = jnp.stack(inputs, axis=1)
85
+ return inputs
86
+
87
+ def __call__(
88
+ self,
89
+ prompt_ids: jax.Array,
90
+ params: dict | FrozenDict,
91
+ prng_seed: jax.Array,
92
+ num_inference_steps: int = 50,
93
+ guidance_scale: float | jax.Array = 7.5,
94
+ height: int | None = None,
95
+ width: int | None = None,
96
+ latents: jnp.array = None,
97
+ neg_prompt_ids: jnp.array = None,
98
+ return_dict: bool = True,
99
+ output_type: str = None,
100
+ jit: bool = False,
101
+ ):
102
+ # 0. Default height and width to unet
103
+ height = height or self.unet.config.sample_size * self.vae_scale_factor
104
+ width = width or self.unet.config.sample_size * self.vae_scale_factor
105
+
106
+ if isinstance(guidance_scale, float) and jit:
107
+ # Convert to a tensor so each device gets a copy.
108
+ guidance_scale = jnp.array([guidance_scale] * prompt_ids.shape[0])
109
+ guidance_scale = guidance_scale[:, None]
110
+
111
+ return_latents = output_type == "latent"
112
+
113
+ if jit:
114
+ images = _p_generate(
115
+ self,
116
+ prompt_ids,
117
+ params,
118
+ prng_seed,
119
+ num_inference_steps,
120
+ height,
121
+ width,
122
+ guidance_scale,
123
+ latents,
124
+ neg_prompt_ids,
125
+ return_latents,
126
+ )
127
+ else:
128
+ images = self._generate(
129
+ prompt_ids,
130
+ params,
131
+ prng_seed,
132
+ num_inference_steps,
133
+ height,
134
+ width,
135
+ guidance_scale,
136
+ latents,
137
+ neg_prompt_ids,
138
+ return_latents,
139
+ )
140
+
141
+ if not return_dict:
142
+ return (images,)
143
+
144
+ return FlaxStableDiffusionXLPipelineOutput(images=images)
145
+
146
+ def get_embeddings(self, prompt_ids: jnp.array, params):
147
+ # We assume we have the two encoders
148
+
149
+ # bs, encoder_input, seq_length
150
+ te_1_inputs = prompt_ids[:, 0, :]
151
+ te_2_inputs = prompt_ids[:, 1, :]
152
+
153
+ prompt_embeds = self.text_encoder(te_1_inputs, params=params["text_encoder"], output_hidden_states=True)
154
+ prompt_embeds = prompt_embeds["hidden_states"][-2]
155
+ prompt_embeds_2_out = self.text_encoder_2(
156
+ te_2_inputs, params=params["text_encoder_2"], output_hidden_states=True
157
+ )
158
+ prompt_embeds_2 = prompt_embeds_2_out["hidden_states"][-2]
159
+ text_embeds = prompt_embeds_2_out["text_embeds"]
160
+ prompt_embeds = jnp.concatenate([prompt_embeds, prompt_embeds_2], axis=-1)
161
+ return prompt_embeds, text_embeds
162
+
163
+ def _get_add_time_ids(self, original_size, crops_coords_top_left, target_size, bs, dtype):
164
+ add_time_ids = list(original_size + crops_coords_top_left + target_size)
165
+ add_time_ids = jnp.array([add_time_ids] * bs, dtype=dtype)
166
+ return add_time_ids
167
+
168
+ def _generate(
169
+ self,
170
+ prompt_ids: jnp.array,
171
+ params: dict | FrozenDict,
172
+ prng_seed: jax.Array,
173
+ num_inference_steps: int,
174
+ height: int,
175
+ width: int,
176
+ guidance_scale: float,
177
+ latents: jnp.array | None = None,
178
+ neg_prompt_ids: jnp.array | None = None,
179
+ return_latents=False,
180
+ ):
181
+ if height % 8 != 0 or width % 8 != 0:
182
+ raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
183
+
184
+ # Encode input prompt
185
+ prompt_embeds, pooled_embeds = self.get_embeddings(prompt_ids, params)
186
+
187
+ # Get unconditional embeddings
188
+ batch_size = prompt_embeds.shape[0]
189
+ if neg_prompt_ids is None:
190
+ neg_prompt_embeds = jnp.zeros_like(prompt_embeds)
191
+ negative_pooled_embeds = jnp.zeros_like(pooled_embeds)
192
+ else:
193
+ neg_prompt_embeds, negative_pooled_embeds = self.get_embeddings(neg_prompt_ids, params)
194
+
195
+ add_time_ids = self._get_add_time_ids(
196
+ (height, width), (0, 0), (height, width), prompt_embeds.shape[0], dtype=prompt_embeds.dtype
197
+ )
198
+
199
+ prompt_embeds = jnp.concatenate([neg_prompt_embeds, prompt_embeds], axis=0) # (2, 77, 2048)
200
+ add_text_embeds = jnp.concatenate([negative_pooled_embeds, pooled_embeds], axis=0)
201
+ add_time_ids = jnp.concatenate([add_time_ids, add_time_ids], axis=0)
202
+
203
+ # Ensure model output will be `float32` before going into the scheduler
204
+ guidance_scale = jnp.array([guidance_scale], dtype=jnp.float32)
205
+
206
+ # Create random latents
207
+ latents_shape = (
208
+ batch_size,
209
+ self.unet.config.in_channels,
210
+ height // self.vae_scale_factor,
211
+ width // self.vae_scale_factor,
212
+ )
213
+ if latents is None:
214
+ latents = jax.random.normal(prng_seed, shape=latents_shape, dtype=jnp.float32)
215
+ else:
216
+ if latents.shape != latents_shape:
217
+ raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}")
218
+
219
+ # Prepare scheduler state
220
+ scheduler_state = self.scheduler.set_timesteps(
221
+ params["scheduler"], num_inference_steps=num_inference_steps, shape=latents.shape
222
+ )
223
+
224
+ # scale the initial noise by the standard deviation required by the scheduler
225
+ latents = latents * scheduler_state.init_noise_sigma
226
+
227
+ added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
228
+
229
+ # Denoising loop
230
+ def loop_body(step, args):
231
+ latents, scheduler_state = args
232
+ # For classifier free guidance, we need to do two forward passes.
233
+ # Here we concatenate the unconditional and text embeddings into a single batch
234
+ # to avoid doing two forward passes
235
+ latents_input = jnp.concatenate([latents] * 2)
236
+
237
+ t = jnp.array(scheduler_state.timesteps, dtype=jnp.int32)[step]
238
+ timestep = jnp.broadcast_to(t, latents_input.shape[0])
239
+
240
+ latents_input = self.scheduler.scale_model_input(scheduler_state, latents_input, t)
241
+
242
+ # predict the noise residual
243
+ noise_pred = self.unet.apply(
244
+ {"params": params["unet"]},
245
+ jnp.array(latents_input),
246
+ jnp.array(timestep, dtype=jnp.int32),
247
+ encoder_hidden_states=prompt_embeds,
248
+ added_cond_kwargs=added_cond_kwargs,
249
+ ).sample
250
+ # perform guidance
251
+ noise_pred_uncond, noise_prediction_text = jnp.split(noise_pred, 2, axis=0)
252
+ noise_pred = noise_pred_uncond + guidance_scale * (noise_prediction_text - noise_pred_uncond)
253
+
254
+ # compute the previous noisy sample x_t -> x_t-1
255
+ latents, scheduler_state = self.scheduler.step(scheduler_state, noise_pred, t, latents).to_tuple()
256
+ return latents, scheduler_state
257
+
258
+ if DEBUG:
259
+ # run with python for loop
260
+ for i in range(num_inference_steps):
261
+ latents, scheduler_state = loop_body(i, (latents, scheduler_state))
262
+ else:
263
+ latents, _ = jax.lax.fori_loop(0, num_inference_steps, loop_body, (latents, scheduler_state))
264
+
265
+ if return_latents:
266
+ return latents
267
+
268
+ # Decode latents
269
+ latents = 1 / self.vae.config.scaling_factor * latents
270
+ image = self.vae.apply({"params": params["vae"]}, latents, method=self.vae.decode).sample
271
+
272
+ image = (image / 2 + 0.5).clip(0, 1).transpose(0, 2, 3, 1)
273
+ return image
274
+
275
+
276
+ # Static argnums are pipe, num_inference_steps, height, width, return_latents. A change would trigger recompilation.
277
+ # Non-static args are (sharded) input tensors mapped over their first dimension (hence, `0`).
278
+ @partial(
279
+ jax.pmap,
280
+ in_axes=(None, 0, 0, 0, None, None, None, 0, 0, 0, None),
281
+ static_broadcasted_argnums=(0, 4, 5, 6, 10),
282
+ )
283
+ def _p_generate(
284
+ pipe,
285
+ prompt_ids,
286
+ params,
287
+ prng_seed,
288
+ num_inference_steps,
289
+ height,
290
+ width,
291
+ guidance_scale,
292
+ latents,
293
+ neg_prompt_ids,
294
+ return_latents,
295
+ ):
296
+ return pipe._generate(
297
+ prompt_ids,
298
+ params,
299
+ prng_seed,
300
+ num_inference_steps,
301
+ height,
302
+ width,
303
+ guidance_scale,
304
+ latents,
305
+ neg_prompt_ids,
306
+ return_latents,
307
+ )
western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_xl/pipeline_output.py ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+
3
+ import numpy as np
4
+ import PIL.Image
5
+
6
+ from ...utils import BaseOutput, is_flax_available
7
+
8
+
9
+ @dataclass
10
+ class StableDiffusionXLPipelineOutput(BaseOutput):
11
+ """
12
+ Output class for Stable Diffusion pipelines.
13
+
14
+ Args:
15
+ images (`list[PIL.Image.Image]` or `np.ndarray`)
16
+ list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width,
17
+ num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline.
18
+ """
19
+
20
+ images: list[PIL.Image.Image] | np.ndarray
21
+
22
+
23
+ if is_flax_available():
24
+ import flax
25
+
26
+ @flax.struct.dataclass
27
+ class FlaxStableDiffusionXLPipelineOutput(BaseOutput):
28
+ """
29
+ Output class for Flax Stable Diffusion XL pipelines.
30
+
31
+ Args:
32
+ images (`np.ndarray`)
33
+ Array of shape `(batch_size, height, width, num_channels)` with images from the diffusion pipeline.
34
+ """
35
+
36
+ images: np.ndarray
western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py ADDED
@@ -0,0 +1,1308 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2026 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from typing import Any, Callable
17
+
18
+ import torch
19
+ from transformers import (
20
+ CLIPImageProcessor,
21
+ CLIPTextModel,
22
+ CLIPTextModelWithProjection,
23
+ CLIPTokenizer,
24
+ CLIPVisionModelWithProjection,
25
+ )
26
+
27
+ from ...callbacks import MultiPipelineCallbacks, PipelineCallback
28
+ from ...image_processor import PipelineImageInput, VaeImageProcessor
29
+ from ...loaders import (
30
+ FromSingleFileMixin,
31
+ IPAdapterMixin,
32
+ StableDiffusionXLLoraLoaderMixin,
33
+ TextualInversionLoaderMixin,
34
+ )
35
+ from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel
36
+ from ...models.lora import adjust_lora_scale_text_encoder
37
+ from ...schedulers import KarrasDiffusionSchedulers
38
+ from ...utils import (
39
+ USE_PEFT_BACKEND,
40
+ deprecate,
41
+ is_invisible_watermark_available,
42
+ is_torch_xla_available,
43
+ logging,
44
+ replace_example_docstring,
45
+ scale_lora_layers,
46
+ unscale_lora_layers,
47
+ )
48
+ from ...utils.torch_utils import randn_tensor
49
+ from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin
50
+ from .pipeline_output import StableDiffusionXLPipelineOutput
51
+
52
+
53
+ if is_invisible_watermark_available():
54
+ from .watermark import StableDiffusionXLWatermarker
55
+
56
+ if is_torch_xla_available():
57
+ import torch_xla.core.xla_model as xm
58
+
59
+ XLA_AVAILABLE = True
60
+ else:
61
+ XLA_AVAILABLE = False
62
+
63
+
64
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
65
+
66
+ EXAMPLE_DOC_STRING = """
67
+ Examples:
68
+ ```py
69
+ >>> import torch
70
+ >>> from diffusers import StableDiffusionXLPipeline
71
+
72
+ >>> pipe = StableDiffusionXLPipeline.from_pretrained(
73
+ ... "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16
74
+ ... )
75
+ >>> pipe = pipe.to("cuda")
76
+
77
+ >>> prompt = "a photo of an astronaut riding a horse on mars"
78
+ >>> image = pipe(prompt).images[0]
79
+ ```
80
+ """
81
+
82
+
83
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg
84
+ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):
85
+ r"""
86
+ Rescales `noise_cfg` tensor based on `guidance_rescale` to improve image quality and fix overexposure. Based on
87
+ Section 3.4 from [Common Diffusion Noise Schedules and Sample Steps are
88
+ Flawed](https://huggingface.co/papers/2305.08891).
89
+
90
+ Args:
91
+ noise_cfg (`torch.Tensor`):
92
+ The predicted noise tensor for the guided diffusion process.
93
+ noise_pred_text (`torch.Tensor`):
94
+ The predicted noise tensor for the text-guided diffusion process.
95
+ guidance_rescale (`float`, *optional*, defaults to 0.0):
96
+ A rescale factor applied to the noise predictions.
97
+
98
+ Returns:
99
+ noise_cfg (`torch.Tensor`): The rescaled noise prediction tensor.
100
+ """
101
+ std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)
102
+ std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)
103
+ # rescale the results from guidance (fixes overexposure)
104
+ noise_pred_rescaled = noise_cfg * (std_text / std_cfg)
105
+ # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images
106
+ noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg
107
+ return noise_cfg
108
+
109
+
110
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
111
+ def retrieve_timesteps(
112
+ scheduler,
113
+ num_inference_steps: int | None = None,
114
+ device: str | torch.device | None = None,
115
+ timesteps: list[int] | None = None,
116
+ sigmas: list[float] | None = None,
117
+ **kwargs,
118
+ ):
119
+ r"""
120
+ Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
121
+ custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
122
+
123
+ Args:
124
+ scheduler (`SchedulerMixin`):
125
+ The scheduler to get timesteps from.
126
+ num_inference_steps (`int`):
127
+ The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
128
+ must be `None`.
129
+ device (`str` or `torch.device`, *optional*):
130
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
131
+ timesteps (`list[int]`, *optional*):
132
+ Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
133
+ `num_inference_steps` and `sigmas` must be `None`.
134
+ sigmas (`list[float]`, *optional*):
135
+ Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
136
+ `num_inference_steps` and `timesteps` must be `None`.
137
+
138
+ Returns:
139
+ `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
140
+ second element is the number of inference steps.
141
+ """
142
+ if timesteps is not None and sigmas is not None:
143
+ raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
144
+ if timesteps is not None:
145
+ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
146
+ if not accepts_timesteps:
147
+ raise ValueError(
148
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
149
+ f" timestep schedules. Please check whether you are using the correct scheduler."
150
+ )
151
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
152
+ timesteps = scheduler.timesteps
153
+ num_inference_steps = len(timesteps)
154
+ elif sigmas is not None:
155
+ accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
156
+ if not accept_sigmas:
157
+ raise ValueError(
158
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
159
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
160
+ )
161
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
162
+ timesteps = scheduler.timesteps
163
+ num_inference_steps = len(timesteps)
164
+ else:
165
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
166
+ timesteps = scheduler.timesteps
167
+ return timesteps, num_inference_steps
168
+
169
+
170
+ class StableDiffusionXLPipeline(
171
+ DiffusionPipeline,
172
+ StableDiffusionMixin,
173
+ FromSingleFileMixin,
174
+ StableDiffusionXLLoraLoaderMixin,
175
+ TextualInversionLoaderMixin,
176
+ IPAdapterMixin,
177
+ ):
178
+ r"""
179
+ Pipeline for text-to-image generation using Stable Diffusion XL.
180
+
181
+ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
182
+ library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
183
+
184
+ The pipeline also inherits the following loading methods:
185
+ - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings
186
+ - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files
187
+ - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights
188
+ - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights
189
+ - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters
190
+
191
+ Args:
192
+ vae ([`AutoencoderKL`]):
193
+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
194
+ text_encoder ([`CLIPTextModel`]):
195
+ Frozen text-encoder. Stable Diffusion XL uses the text portion of
196
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
197
+ the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
198
+ text_encoder_2 ([` CLIPTextModelWithProjection`]):
199
+ Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of
200
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection),
201
+ specifically the
202
+ [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k)
203
+ variant.
204
+ tokenizer (`CLIPTokenizer`):
205
+ Tokenizer of class
206
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
207
+ tokenizer_2 (`CLIPTokenizer`):
208
+ Second Tokenizer of class
209
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
210
+ unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.
211
+ scheduler ([`SchedulerMixin`]):
212
+ A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
213
+ [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
214
+ force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`):
215
+ Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of
216
+ `stabilityai/stable-diffusion-xl-base-1-0`.
217
+ add_watermarker (`bool`, *optional*):
218
+ Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to
219
+ watermark output images. If not defined, it will default to True if the package is installed, otherwise no
220
+ watermarker will be used.
221
+ """
222
+
223
+ model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->unet->vae"
224
+ _optional_components = [
225
+ "tokenizer",
226
+ "tokenizer_2",
227
+ "text_encoder",
228
+ "text_encoder_2",
229
+ "image_encoder",
230
+ "feature_extractor",
231
+ ]
232
+ _callback_tensor_inputs = [
233
+ "latents",
234
+ "prompt_embeds",
235
+ "add_text_embeds",
236
+ "add_time_ids",
237
+ ]
238
+
239
+ def __init__(
240
+ self,
241
+ vae: AutoencoderKL,
242
+ text_encoder: CLIPTextModel,
243
+ text_encoder_2: CLIPTextModelWithProjection,
244
+ tokenizer: CLIPTokenizer,
245
+ tokenizer_2: CLIPTokenizer,
246
+ unet: UNet2DConditionModel,
247
+ scheduler: KarrasDiffusionSchedulers,
248
+ image_encoder: CLIPVisionModelWithProjection = None,
249
+ feature_extractor: CLIPImageProcessor = None,
250
+ force_zeros_for_empty_prompt: bool = True,
251
+ add_watermarker: bool | None = None,
252
+ ):
253
+ super().__init__()
254
+
255
+ self.register_modules(
256
+ vae=vae,
257
+ text_encoder=text_encoder,
258
+ text_encoder_2=text_encoder_2,
259
+ tokenizer=tokenizer,
260
+ tokenizer_2=tokenizer_2,
261
+ unet=unet,
262
+ scheduler=scheduler,
263
+ image_encoder=image_encoder,
264
+ feature_extractor=feature_extractor,
265
+ )
266
+ self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt)
267
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
268
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
269
+
270
+ self.default_sample_size = (
271
+ self.unet.config.sample_size
272
+ if hasattr(self, "unet") and self.unet is not None and hasattr(self.unet.config, "sample_size")
273
+ else 128
274
+ )
275
+
276
+ add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available()
277
+
278
+ if add_watermarker:
279
+ self.watermark = StableDiffusionXLWatermarker()
280
+ else:
281
+ self.watermark = None
282
+
283
+ def encode_prompt(
284
+ self,
285
+ prompt: str,
286
+ prompt_2: str | None = None,
287
+ device: torch.device | None = None,
288
+ num_images_per_prompt: int = 1,
289
+ do_classifier_free_guidance: bool = True,
290
+ negative_prompt: str | None = None,
291
+ negative_prompt_2: str | None = None,
292
+ prompt_embeds: torch.Tensor | None = None,
293
+ negative_prompt_embeds: torch.Tensor | None = None,
294
+ pooled_prompt_embeds: torch.Tensor | None = None,
295
+ negative_pooled_prompt_embeds: torch.Tensor | None = None,
296
+ lora_scale: float | None = None,
297
+ clip_skip: int | None = None,
298
+ ):
299
+ r"""
300
+ Encodes the prompt into text encoder hidden states.
301
+
302
+ Args:
303
+ prompt (`str` or `list[str]`, *optional*):
304
+ prompt to be encoded
305
+ prompt_2 (`str` or `list[str]`, *optional*):
306
+ The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
307
+ used in both text-encoders
308
+ device: (`torch.device`):
309
+ torch device
310
+ num_images_per_prompt (`int`):
311
+ number of images that should be generated per prompt
312
+ do_classifier_free_guidance (`bool`):
313
+ whether to use classifier free guidance or not
314
+ negative_prompt (`str` or `list[str]`, *optional*):
315
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
316
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
317
+ less than `1`).
318
+ negative_prompt_2 (`str` or `list[str]`, *optional*):
319
+ The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
320
+ `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
321
+ prompt_embeds (`torch.Tensor`, *optional*):
322
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
323
+ provided, text embeddings will be generated from `prompt` input argument.
324
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
325
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
326
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
327
+ argument.
328
+ pooled_prompt_embeds (`torch.Tensor`, *optional*):
329
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
330
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
331
+ negative_pooled_prompt_embeds (`torch.Tensor`, *optional*):
332
+ Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
333
+ weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
334
+ input argument.
335
+ lora_scale (`float`, *optional*):
336
+ A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
337
+ clip_skip (`int`, *optional*):
338
+ Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
339
+ the output of the pre-final layer will be used for computing the prompt embeddings.
340
+ """
341
+ device = device or self._execution_device
342
+
343
+ # set lora scale so that monkey patched LoRA
344
+ # function of text encoder can correctly access it
345
+ if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin):
346
+ self._lora_scale = lora_scale
347
+
348
+ # dynamically adjust the LoRA scale
349
+ if self.text_encoder is not None:
350
+ if not USE_PEFT_BACKEND:
351
+ adjust_lora_scale_text_encoder(self.text_encoder, lora_scale)
352
+ else:
353
+ scale_lora_layers(self.text_encoder, lora_scale)
354
+
355
+ if self.text_encoder_2 is not None:
356
+ if not USE_PEFT_BACKEND:
357
+ adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale)
358
+ else:
359
+ scale_lora_layers(self.text_encoder_2, lora_scale)
360
+
361
+ prompt = [prompt] if isinstance(prompt, str) else prompt
362
+
363
+ if prompt is not None:
364
+ batch_size = len(prompt)
365
+ else:
366
+ batch_size = prompt_embeds.shape[0]
367
+
368
+ # Define tokenizers and text encoders
369
+ tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2]
370
+ text_encoders = (
371
+ [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2]
372
+ )
373
+
374
+ if prompt_embeds is None:
375
+ prompt_2 = prompt_2 or prompt
376
+ prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2
377
+
378
+ # textual inversion: process multi-vector tokens if necessary
379
+ prompt_embeds_list = []
380
+ prompts = [prompt, prompt_2]
381
+ for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders):
382
+ if isinstance(self, TextualInversionLoaderMixin):
383
+ prompt = self.maybe_convert_prompt(prompt, tokenizer)
384
+
385
+ text_inputs = tokenizer(
386
+ prompt,
387
+ padding="max_length",
388
+ max_length=tokenizer.model_max_length,
389
+ truncation=True,
390
+ return_tensors="pt",
391
+ )
392
+
393
+ text_input_ids = text_inputs.input_ids
394
+ untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
395
+
396
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(
397
+ text_input_ids, untruncated_ids
398
+ ):
399
+ removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1])
400
+ logger.warning(
401
+ "The following part of your input was truncated because CLIP can only handle sequences up to"
402
+ f" {tokenizer.model_max_length} tokens: {removed_text}"
403
+ )
404
+
405
+ prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True)
406
+
407
+ # We are only ALWAYS interested in the pooled output of the final text encoder
408
+ if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2:
409
+ pooled_prompt_embeds = prompt_embeds[0]
410
+
411
+ if clip_skip is None:
412
+ prompt_embeds = prompt_embeds.hidden_states[-2]
413
+ else:
414
+ # "2" because SDXL always indexes from the penultimate layer.
415
+ prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)]
416
+
417
+ prompt_embeds_list.append(prompt_embeds)
418
+
419
+ prompt_embeds = torch.concat(prompt_embeds_list, dim=-1)
420
+
421
+ # get unconditional embeddings for classifier free guidance
422
+ zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt
423
+ if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt:
424
+ negative_prompt_embeds = torch.zeros_like(prompt_embeds)
425
+ negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds)
426
+ elif do_classifier_free_guidance and negative_prompt_embeds is None:
427
+ negative_prompt = negative_prompt or ""
428
+ negative_prompt_2 = negative_prompt_2 or negative_prompt
429
+
430
+ # normalize str to list
431
+ negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
432
+ negative_prompt_2 = (
433
+ batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2
434
+ )
435
+
436
+ uncond_tokens: list[str]
437
+ if prompt is not None and type(prompt) is not type(negative_prompt):
438
+ raise TypeError(
439
+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
440
+ f" {type(prompt)}."
441
+ )
442
+ elif batch_size != len(negative_prompt):
443
+ raise ValueError(
444
+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
445
+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
446
+ " the batch size of `prompt`."
447
+ )
448
+ else:
449
+ uncond_tokens = [negative_prompt, negative_prompt_2]
450
+
451
+ negative_prompt_embeds_list = []
452
+ for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders):
453
+ if isinstance(self, TextualInversionLoaderMixin):
454
+ negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer)
455
+
456
+ max_length = prompt_embeds.shape[1]
457
+ uncond_input = tokenizer(
458
+ negative_prompt,
459
+ padding="max_length",
460
+ max_length=max_length,
461
+ truncation=True,
462
+ return_tensors="pt",
463
+ )
464
+
465
+ negative_prompt_embeds = text_encoder(
466
+ uncond_input.input_ids.to(device),
467
+ output_hidden_states=True,
468
+ )
469
+
470
+ # We are only ALWAYS interested in the pooled output of the final text encoder
471
+ if negative_pooled_prompt_embeds is None and negative_prompt_embeds[0].ndim == 2:
472
+ negative_pooled_prompt_embeds = negative_prompt_embeds[0]
473
+ negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2]
474
+
475
+ negative_prompt_embeds_list.append(negative_prompt_embeds)
476
+
477
+ negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1)
478
+
479
+ if self.text_encoder_2 is not None:
480
+ prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)
481
+ else:
482
+ prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device)
483
+
484
+ bs_embed, seq_len, _ = prompt_embeds.shape
485
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
486
+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
487
+ prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
488
+
489
+ if do_classifier_free_guidance:
490
+ # duplicate unconditional embeddings for each generation per prompt, using mps friendly method
491
+ seq_len = negative_prompt_embeds.shape[1]
492
+
493
+ if self.text_encoder_2 is not None:
494
+ negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)
495
+ else:
496
+ negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device)
497
+
498
+ negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
499
+ negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
500
+
501
+ pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
502
+ bs_embed * num_images_per_prompt, -1
503
+ )
504
+ if do_classifier_free_guidance:
505
+ negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
506
+ bs_embed * num_images_per_prompt, -1
507
+ )
508
+
509
+ if self.text_encoder is not None:
510
+ if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND:
511
+ # Retrieve the original scale by scaling back the LoRA layers
512
+ unscale_lora_layers(self.text_encoder, lora_scale)
513
+
514
+ if self.text_encoder_2 is not None:
515
+ if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND:
516
+ # Retrieve the original scale by scaling back the LoRA layers
517
+ unscale_lora_layers(self.text_encoder_2, lora_scale)
518
+
519
+ return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds
520
+
521
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image
522
+ def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None):
523
+ dtype = next(self.image_encoder.parameters()).dtype
524
+
525
+ if not isinstance(image, torch.Tensor):
526
+ image = self.feature_extractor(image, return_tensors="pt").pixel_values
527
+
528
+ image = image.to(device=device, dtype=dtype)
529
+ if output_hidden_states:
530
+ image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2]
531
+ image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0)
532
+ uncond_image_enc_hidden_states = self.image_encoder(
533
+ torch.zeros_like(image), output_hidden_states=True
534
+ ).hidden_states[-2]
535
+ uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave(
536
+ num_images_per_prompt, dim=0
537
+ )
538
+ return image_enc_hidden_states, uncond_image_enc_hidden_states
539
+ else:
540
+ image_embeds = self.image_encoder(image).image_embeds
541
+ image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0)
542
+ uncond_image_embeds = torch.zeros_like(image_embeds)
543
+
544
+ return image_embeds, uncond_image_embeds
545
+
546
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds
547
+ def prepare_ip_adapter_image_embeds(
548
+ self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance
549
+ ):
550
+ image_embeds = []
551
+ if do_classifier_free_guidance:
552
+ negative_image_embeds = []
553
+ if ip_adapter_image_embeds is None:
554
+ if not isinstance(ip_adapter_image, list):
555
+ ip_adapter_image = [ip_adapter_image]
556
+
557
+ if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers):
558
+ raise ValueError(
559
+ f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters."
560
+ )
561
+
562
+ for single_ip_adapter_image, image_proj_layer in zip(
563
+ ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers
564
+ ):
565
+ output_hidden_state = not isinstance(image_proj_layer, ImageProjection)
566
+ single_image_embeds, single_negative_image_embeds = self.encode_image(
567
+ single_ip_adapter_image, device, 1, output_hidden_state
568
+ )
569
+
570
+ image_embeds.append(single_image_embeds[None, :])
571
+ if do_classifier_free_guidance:
572
+ negative_image_embeds.append(single_negative_image_embeds[None, :])
573
+ else:
574
+ for single_image_embeds in ip_adapter_image_embeds:
575
+ if do_classifier_free_guidance:
576
+ single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2)
577
+ negative_image_embeds.append(single_negative_image_embeds)
578
+ image_embeds.append(single_image_embeds)
579
+
580
+ ip_adapter_image_embeds = []
581
+ for i, single_image_embeds in enumerate(image_embeds):
582
+ single_image_embeds = torch.cat([single_image_embeds] * num_images_per_prompt, dim=0)
583
+ if do_classifier_free_guidance:
584
+ single_negative_image_embeds = torch.cat([negative_image_embeds[i]] * num_images_per_prompt, dim=0)
585
+ single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds], dim=0)
586
+
587
+ single_image_embeds = single_image_embeds.to(device=device)
588
+ ip_adapter_image_embeds.append(single_image_embeds)
589
+
590
+ return ip_adapter_image_embeds
591
+
592
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs
593
+ def prepare_extra_step_kwargs(self, generator, eta):
594
+ # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
595
+ # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
596
+ # eta corresponds to η in DDIM paper: https://huggingface.co/papers/2010.02502
597
+ # and should be between [0, 1]
598
+
599
+ accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
600
+ extra_step_kwargs = {}
601
+ if accepts_eta:
602
+ extra_step_kwargs["eta"] = eta
603
+
604
+ # check if the scheduler accepts generator
605
+ accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
606
+ if accepts_generator:
607
+ extra_step_kwargs["generator"] = generator
608
+ return extra_step_kwargs
609
+
610
+ def check_inputs(
611
+ self,
612
+ prompt,
613
+ prompt_2,
614
+ height,
615
+ width,
616
+ callback_steps,
617
+ negative_prompt=None,
618
+ negative_prompt_2=None,
619
+ prompt_embeds=None,
620
+ negative_prompt_embeds=None,
621
+ pooled_prompt_embeds=None,
622
+ negative_pooled_prompt_embeds=None,
623
+ ip_adapter_image=None,
624
+ ip_adapter_image_embeds=None,
625
+ callback_on_step_end_tensor_inputs=None,
626
+ ):
627
+ if height % 8 != 0 or width % 8 != 0:
628
+ raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
629
+
630
+ if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0):
631
+ raise ValueError(
632
+ f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
633
+ f" {type(callback_steps)}."
634
+ )
635
+
636
+ if callback_on_step_end_tensor_inputs is not None and not all(
637
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
638
+ ):
639
+ raise ValueError(
640
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
641
+ )
642
+
643
+ if prompt is not None and prompt_embeds is not None:
644
+ raise ValueError(
645
+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
646
+ " only forward one of the two."
647
+ )
648
+ elif prompt_2 is not None and prompt_embeds is not None:
649
+ raise ValueError(
650
+ f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
651
+ " only forward one of the two."
652
+ )
653
+ elif prompt is None and prompt_embeds is None:
654
+ raise ValueError(
655
+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
656
+ )
657
+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
658
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
659
+ elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)):
660
+ raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}")
661
+
662
+ if negative_prompt is not None and negative_prompt_embeds is not None:
663
+ raise ValueError(
664
+ f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
665
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
666
+ )
667
+ elif negative_prompt_2 is not None and negative_prompt_embeds is not None:
668
+ raise ValueError(
669
+ f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:"
670
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
671
+ )
672
+
673
+ if prompt_embeds is not None and negative_prompt_embeds is not None:
674
+ if prompt_embeds.shape != negative_prompt_embeds.shape:
675
+ raise ValueError(
676
+ "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"
677
+ f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"
678
+ f" {negative_prompt_embeds.shape}."
679
+ )
680
+
681
+ if prompt_embeds is not None and pooled_prompt_embeds is None:
682
+ raise ValueError(
683
+ "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`."
684
+ )
685
+
686
+ if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None:
687
+ raise ValueError(
688
+ "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`."
689
+ )
690
+
691
+ if ip_adapter_image is not None and ip_adapter_image_embeds is not None:
692
+ raise ValueError(
693
+ "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined."
694
+ )
695
+
696
+ if ip_adapter_image_embeds is not None:
697
+ if not isinstance(ip_adapter_image_embeds, list):
698
+ raise ValueError(
699
+ f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}"
700
+ )
701
+ elif ip_adapter_image_embeds[0].ndim not in [3, 4]:
702
+ raise ValueError(
703
+ f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D"
704
+ )
705
+
706
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents
707
+ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):
708
+ shape = (
709
+ batch_size,
710
+ num_channels_latents,
711
+ int(height) // self.vae_scale_factor,
712
+ int(width) // self.vae_scale_factor,
713
+ )
714
+ if isinstance(generator, list) and len(generator) != batch_size:
715
+ raise ValueError(
716
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
717
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
718
+ )
719
+
720
+ if latents is None:
721
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
722
+ else:
723
+ latents = latents.to(device)
724
+
725
+ # scale the initial noise by the standard deviation required by the scheduler
726
+ latents = latents * self.scheduler.init_noise_sigma
727
+ return latents
728
+
729
+ def _get_add_time_ids(
730
+ self, original_size, crops_coords_top_left, target_size, dtype, text_encoder_projection_dim=None
731
+ ):
732
+ add_time_ids = list(original_size + crops_coords_top_left + target_size)
733
+
734
+ passed_add_embed_dim = (
735
+ self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim
736
+ )
737
+ expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features
738
+
739
+ if expected_add_embed_dim != passed_add_embed_dim:
740
+ raise ValueError(
741
+ f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`."
742
+ )
743
+
744
+ add_time_ids = torch.tensor([add_time_ids], dtype=dtype)
745
+ return add_time_ids
746
+
747
+ def upcast_vae(self):
748
+ deprecate(
749
+ "upcast_vae",
750
+ "1.0.0",
751
+ "`upcast_vae` is deprecated. Please use `pipe.vae.to(torch.float32)`. For more details, please refer to: https://github.com/huggingface/diffusers/pull/12619#issue-3606633695.",
752
+ )
753
+ self.vae.to(dtype=torch.float32)
754
+
755
+ # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding
756
+ def get_guidance_scale_embedding(
757
+ self, w: torch.Tensor, embedding_dim: int = 512, dtype: torch.dtype = torch.float32
758
+ ) -> torch.Tensor:
759
+ """
760
+ See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298
761
+
762
+ Args:
763
+ w (`torch.Tensor`):
764
+ Generate embedding vectors with a specified guidance scale to subsequently enrich timestep embeddings.
765
+ embedding_dim (`int`, *optional*, defaults to 512):
766
+ Dimension of the embeddings to generate.
767
+ dtype (`torch.dtype`, *optional*, defaults to `torch.float32`):
768
+ Data type of the generated embeddings.
769
+
770
+ Returns:
771
+ `torch.Tensor`: Embedding vectors with shape `(len(w), embedding_dim)`.
772
+ """
773
+ assert len(w.shape) == 1
774
+ w = w * 1000.0
775
+
776
+ half_dim = embedding_dim // 2
777
+ emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1)
778
+ emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb)
779
+ emb = w.to(dtype)[:, None] * emb[None, :]
780
+ emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)
781
+ if embedding_dim % 2 == 1: # zero pad
782
+ emb = torch.nn.functional.pad(emb, (0, 1))
783
+ assert emb.shape == (w.shape[0], embedding_dim)
784
+ return emb
785
+
786
+ @property
787
+ def guidance_scale(self):
788
+ return self._guidance_scale
789
+
790
+ @property
791
+ def guidance_rescale(self):
792
+ return self._guidance_rescale
793
+
794
+ @property
795
+ def clip_skip(self):
796
+ return self._clip_skip
797
+
798
+ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
799
+ # of the Imagen paper: https://huggingface.co/papers/2205.11487 . `guidance_scale = 1`
800
+ # corresponds to doing no classifier free guidance.
801
+ @property
802
+ def do_classifier_free_guidance(self):
803
+ return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None
804
+
805
+ @property
806
+ def cross_attention_kwargs(self):
807
+ return self._cross_attention_kwargs
808
+
809
+ @property
810
+ def denoising_end(self):
811
+ return self._denoising_end
812
+
813
+ @property
814
+ def num_timesteps(self):
815
+ return self._num_timesteps
816
+
817
+ @property
818
+ def interrupt(self):
819
+ return self._interrupt
820
+
821
+ @torch.no_grad()
822
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
823
+ def __call__(
824
+ self,
825
+ prompt: str | list[str] = None,
826
+ prompt_2: str | list[str] | None = None,
827
+ height: int | None = None,
828
+ width: int | None = None,
829
+ num_inference_steps: int = 50,
830
+ timesteps: list[int] = None,
831
+ sigmas: list[float] = None,
832
+ denoising_end: float | None = None,
833
+ guidance_scale: float = 5.0,
834
+ negative_prompt: str | list[str] | None = None,
835
+ negative_prompt_2: str | list[str] | None = None,
836
+ num_images_per_prompt: int | None = 1,
837
+ eta: float = 0.0,
838
+ generator: torch.Generator | list[torch.Generator] | None = None,
839
+ latents: torch.Tensor | None = None,
840
+ prompt_embeds: torch.Tensor | None = None,
841
+ negative_prompt_embeds: torch.Tensor | None = None,
842
+ pooled_prompt_embeds: torch.Tensor | None = None,
843
+ negative_pooled_prompt_embeds: torch.Tensor | None = None,
844
+ ip_adapter_image: PipelineImageInput | None = None,
845
+ ip_adapter_image_embeds: list[torch.Tensor] | None = None,
846
+ output_type: str | None = "pil",
847
+ return_dict: bool = True,
848
+ cross_attention_kwargs: dict[str, Any] | None = None,
849
+ guidance_rescale: float = 0.0,
850
+ original_size: tuple[int, int] | None = None,
851
+ crops_coords_top_left: tuple[int, int] = (0, 0),
852
+ target_size: tuple[int, int] | None = None,
853
+ negative_original_size: tuple[int, int] | None = None,
854
+ negative_crops_coords_top_left: tuple[int, int] = (0, 0),
855
+ negative_target_size: tuple[int, int] | None = None,
856
+ clip_skip: int | None = None,
857
+ callback_on_step_end: Callable[[int, int], None] | PipelineCallback | MultiPipelineCallbacks | None = None,
858
+ callback_on_step_end_tensor_inputs: list[str] = ["latents"],
859
+ **kwargs,
860
+ ):
861
+ r"""
862
+ Function invoked when calling the pipeline for generation.
863
+
864
+ Args:
865
+ prompt (`str` or `list[str]`, *optional*):
866
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
867
+ instead.
868
+ prompt_2 (`str` or `list[str]`, *optional*):
869
+ The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
870
+ used in both text-encoders
871
+ height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
872
+ The height in pixels of the generated image. This is set to 1024 by default for the best results.
873
+ Anything below 512 pixels won't work well for
874
+ [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
875
+ and checkpoints that are not specifically fine-tuned on low resolutions.
876
+ width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
877
+ The width in pixels of the generated image. This is set to 1024 by default for the best results.
878
+ Anything below 512 pixels won't work well for
879
+ [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
880
+ and checkpoints that are not specifically fine-tuned on low resolutions.
881
+ num_inference_steps (`int`, *optional*, defaults to 50):
882
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
883
+ expense of slower inference.
884
+ timesteps (`list[int]`, *optional*):
885
+ Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument
886
+ in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is
887
+ passed will be used. Must be in descending order.
888
+ sigmas (`list[float]`, *optional*):
889
+ Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
890
+ their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
891
+ will be used.
892
+ denoising_end (`float`, *optional*):
893
+ When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be
894
+ completed before it is intentionally prematurely terminated. As a result, the returned sample will
895
+ still retain a substantial amount of noise as determined by the discrete timesteps selected by the
896
+ scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a
897
+ "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image
898
+ Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output)
899
+ guidance_scale (`float`, *optional*, defaults to 5.0):
900
+ Guidance scale as defined in [Classifier-Free Diffusion
901
+ Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2.
902
+ of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
903
+ `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to
904
+ the text `prompt`, usually at the expense of lower image quality.
905
+ negative_prompt (`str` or `list[str]`, *optional*):
906
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
907
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
908
+ less than `1`).
909
+ negative_prompt_2 (`str` or `list[str]`, *optional*):
910
+ The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
911
+ `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
912
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
913
+ The number of images to generate per prompt.
914
+ eta (`float`, *optional*, defaults to 0.0):
915
+ Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only
916
+ applies to [`schedulers.DDIMScheduler`], will be ignored for others.
917
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
918
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
919
+ to make generation deterministic.
920
+ latents (`torch.Tensor`, *optional*):
921
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
922
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
923
+ tensor will be generated by sampling using the supplied random `generator`.
924
+ prompt_embeds (`torch.Tensor`, *optional*):
925
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
926
+ provided, text embeddings will be generated from `prompt` input argument.
927
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
928
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
929
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
930
+ argument.
931
+ pooled_prompt_embeds (`torch.Tensor`, *optional*):
932
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
933
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
934
+ negative_pooled_prompt_embeds (`torch.Tensor`, *optional*):
935
+ Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
936
+ weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
937
+ input argument.
938
+ ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.
939
+ ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*):
940
+ Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of
941
+ IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should
942
+ contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not
943
+ provided, embeddings are computed from the `ip_adapter_image` input argument.
944
+ output_type (`str`, *optional*, defaults to `"pil"`):
945
+ The output format of the generate image. Choose between
946
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
947
+ return_dict (`bool`, *optional*, defaults to `True`):
948
+ Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead
949
+ of a plain tuple.
950
+ cross_attention_kwargs (`dict`, *optional*):
951
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
952
+ `self.processor` in
953
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
954
+ guidance_rescale (`float`, *optional*, defaults to 0.0):
955
+ Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are
956
+ Flawed](https://huggingface.co/papers/2305.08891) `guidance_scale` is defined as `φ` in equation 16. of
957
+ [Common Diffusion Noise Schedules and Sample Steps are
958
+ Flawed](https://huggingface.co/papers/2305.08891). Guidance rescale factor should fix overexposure when
959
+ using zero terminal SNR.
960
+ original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
961
+ If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled.
962
+ `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as
963
+ explained in section 2.2 of
964
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
965
+ crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)):
966
+ `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position
967
+ `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting
968
+ `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of
969
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
970
+ target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
971
+ For most cases, `target_size` should be set to the desired height and width of the generated image. If
972
+ not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in
973
+ section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
974
+ negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
975
+ To negatively condition the generation process based on a specific image resolution. Part of SDXL's
976
+ micro-conditioning as explained in section 2.2 of
977
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
978
+ information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
979
+ negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)):
980
+ To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's
981
+ micro-conditioning as explained in section 2.2 of
982
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
983
+ information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
984
+ negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
985
+ To negatively condition the generation process based on a target image resolution. It should be as same
986
+ as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of
987
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
988
+ information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
989
+ clip_skip (`int`, *optional*):
990
+ Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
991
+ the output of the pre-final layer will be used for computing the prompt embeddings.
992
+ callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*):
993
+ A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of
994
+ each denoising step during the inference. with the following arguments: `callback_on_step_end(self:
995
+ DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a
996
+ list of all tensors as specified by `callback_on_step_end_tensor_inputs`.
997
+ callback_on_step_end_tensor_inputs (`list`, *optional*):
998
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
999
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
1000
+ `._callback_tensor_inputs` attribute of your pipeline class.
1001
+
1002
+ Examples:
1003
+
1004
+ Returns:
1005
+ [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`:
1006
+ [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a
1007
+ `tuple`. When returning a tuple, the first element is a list with the generated images.
1008
+ """
1009
+
1010
+ callback = kwargs.pop("callback", None)
1011
+ callback_steps = kwargs.pop("callback_steps", None)
1012
+
1013
+ if callback is not None:
1014
+ deprecate(
1015
+ "callback",
1016
+ "1.0.0",
1017
+ "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
1018
+ )
1019
+ if callback_steps is not None:
1020
+ deprecate(
1021
+ "callback_steps",
1022
+ "1.0.0",
1023
+ "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
1024
+ )
1025
+
1026
+ if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
1027
+ callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
1028
+
1029
+ # 0. Default height and width to unet
1030
+ height = height or self.default_sample_size * self.vae_scale_factor
1031
+ width = width or self.default_sample_size * self.vae_scale_factor
1032
+
1033
+ original_size = original_size or (height, width)
1034
+ target_size = target_size or (height, width)
1035
+
1036
+ # 1. Check inputs. Raise error if not correct
1037
+ self.check_inputs(
1038
+ prompt,
1039
+ prompt_2,
1040
+ height,
1041
+ width,
1042
+ callback_steps,
1043
+ negative_prompt,
1044
+ negative_prompt_2,
1045
+ prompt_embeds,
1046
+ negative_prompt_embeds,
1047
+ pooled_prompt_embeds,
1048
+ negative_pooled_prompt_embeds,
1049
+ ip_adapter_image,
1050
+ ip_adapter_image_embeds,
1051
+ callback_on_step_end_tensor_inputs,
1052
+ )
1053
+
1054
+ self._guidance_scale = guidance_scale
1055
+ self._guidance_rescale = guidance_rescale
1056
+ self._clip_skip = clip_skip
1057
+ self._cross_attention_kwargs = cross_attention_kwargs
1058
+ self._denoising_end = denoising_end
1059
+ self._interrupt = False
1060
+
1061
+ # 2. Define call parameters
1062
+ if prompt is not None and isinstance(prompt, str):
1063
+ batch_size = 1
1064
+ elif prompt is not None and isinstance(prompt, list):
1065
+ batch_size = len(prompt)
1066
+ else:
1067
+ batch_size = prompt_embeds.shape[0]
1068
+
1069
+ device = self._execution_device
1070
+
1071
+ # 3. Encode input prompt
1072
+ lora_scale = (
1073
+ self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None
1074
+ )
1075
+
1076
+ (
1077
+ prompt_embeds,
1078
+ negative_prompt_embeds,
1079
+ pooled_prompt_embeds,
1080
+ negative_pooled_prompt_embeds,
1081
+ ) = self.encode_prompt(
1082
+ prompt=prompt,
1083
+ prompt_2=prompt_2,
1084
+ device=device,
1085
+ num_images_per_prompt=num_images_per_prompt,
1086
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
1087
+ negative_prompt=negative_prompt,
1088
+ negative_prompt_2=negative_prompt_2,
1089
+ prompt_embeds=prompt_embeds,
1090
+ negative_prompt_embeds=negative_prompt_embeds,
1091
+ pooled_prompt_embeds=pooled_prompt_embeds,
1092
+ negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
1093
+ lora_scale=lora_scale,
1094
+ clip_skip=self.clip_skip,
1095
+ )
1096
+
1097
+ # 4. Prepare timesteps
1098
+ is_neuron_device = device.type == "neuron"
1099
+ if XLA_AVAILABLE or is_neuron_device:
1100
+ timestep_device = "cpu"
1101
+ else:
1102
+ timestep_device = device
1103
+ timesteps, num_inference_steps = retrieve_timesteps(
1104
+ self.scheduler, num_inference_steps, timestep_device, timesteps, sigmas
1105
+ )
1106
+
1107
+ # 5. Prepare latent variables
1108
+ num_channels_latents = self.unet.config.in_channels
1109
+ latents = self.prepare_latents(
1110
+ batch_size * num_images_per_prompt,
1111
+ num_channels_latents,
1112
+ height,
1113
+ width,
1114
+ prompt_embeds.dtype,
1115
+ device,
1116
+ generator,
1117
+ latents,
1118
+ )
1119
+
1120
+ # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
1121
+ extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
1122
+
1123
+ # 7. Prepare added time ids & embeddings
1124
+ add_text_embeds = pooled_prompt_embeds
1125
+ if self.text_encoder_2 is None:
1126
+ text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1])
1127
+ else:
1128
+ text_encoder_projection_dim = self.text_encoder_2.config.projection_dim
1129
+
1130
+ add_time_ids = self._get_add_time_ids(
1131
+ original_size,
1132
+ crops_coords_top_left,
1133
+ target_size,
1134
+ dtype=prompt_embeds.dtype,
1135
+ text_encoder_projection_dim=text_encoder_projection_dim,
1136
+ )
1137
+ if negative_original_size is not None and negative_target_size is not None:
1138
+ negative_add_time_ids = self._get_add_time_ids(
1139
+ negative_original_size,
1140
+ negative_crops_coords_top_left,
1141
+ negative_target_size,
1142
+ dtype=prompt_embeds.dtype,
1143
+ text_encoder_projection_dim=text_encoder_projection_dim,
1144
+ )
1145
+ else:
1146
+ negative_add_time_ids = add_time_ids
1147
+
1148
+ if self.do_classifier_free_guidance:
1149
+ prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
1150
+ add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0)
1151
+ add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], dim=0)
1152
+
1153
+ prompt_embeds = prompt_embeds.to(device)
1154
+ add_text_embeds = add_text_embeds.to(device)
1155
+ add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1)
1156
+
1157
+ if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
1158
+ image_embeds = self.prepare_ip_adapter_image_embeds(
1159
+ ip_adapter_image,
1160
+ ip_adapter_image_embeds,
1161
+ device,
1162
+ batch_size * num_images_per_prompt,
1163
+ self.do_classifier_free_guidance,
1164
+ )
1165
+
1166
+ # 8. Denoising loop
1167
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
1168
+
1169
+ # 8.1 Apply denoising_end
1170
+ if (
1171
+ self.denoising_end is not None
1172
+ and isinstance(self.denoising_end, float)
1173
+ and self.denoising_end > 0
1174
+ and self.denoising_end < 1
1175
+ ):
1176
+ discrete_timestep_cutoff = int(
1177
+ round(
1178
+ self.scheduler.config.num_train_timesteps
1179
+ - (self.denoising_end * self.scheduler.config.num_train_timesteps)
1180
+ )
1181
+ )
1182
+ num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps)))
1183
+ timesteps = timesteps[:num_inference_steps]
1184
+
1185
+ # 9. Optionally get Guidance Scale Embedding
1186
+ timestep_cond = None
1187
+ if self.unet.config.time_cond_proj_dim is not None:
1188
+ guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)
1189
+ timestep_cond = self.get_guidance_scale_embedding(
1190
+ guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim
1191
+ ).to(device=device, dtype=latents.dtype)
1192
+
1193
+ self._num_timesteps = len(timesteps)
1194
+ if hasattr(self.scheduler, "set_begin_index"):
1195
+ self.scheduler.set_begin_index(0)
1196
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
1197
+ for i, t in enumerate(timesteps):
1198
+ if self.interrupt:
1199
+ continue
1200
+
1201
+ # expand the latents if we are doing classifier free guidance
1202
+ latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
1203
+ latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
1204
+
1205
+ # predict the noise residual
1206
+ added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
1207
+ if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
1208
+ added_cond_kwargs["image_embeds"] = image_embeds
1209
+ # [Neuron] pre-cast timestep to float32 on device. Neuron XLA does not support
1210
+ # int64 ops; the compiled UNet graph requires a float32 timestep input on-device.
1211
+ t_unet = t.to(torch.float32).to(device) if is_neuron_device else t
1212
+ noise_pred = self.unet(
1213
+ latent_model_input,
1214
+ t_unet,
1215
+ encoder_hidden_states=prompt_embeds,
1216
+ timestep_cond=timestep_cond,
1217
+ cross_attention_kwargs=self.cross_attention_kwargs,
1218
+ added_cond_kwargs=added_cond_kwargs,
1219
+ return_dict=False,
1220
+ )[0]
1221
+
1222
+ # perform guidance
1223
+ if self.do_classifier_free_guidance:
1224
+ noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
1225
+ noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)
1226
+
1227
+ if self.do_classifier_free_guidance and self.guidance_rescale > 0.0:
1228
+ # Based on 3.4. in https://huggingface.co/papers/2305.08891
1229
+ noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale)
1230
+
1231
+ # compute the previous noisy sample x_t -> x_t-1
1232
+ latents_dtype = latents.dtype
1233
+ latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
1234
+ if latents.dtype != latents_dtype:
1235
+ if torch.backends.mps.is_available():
1236
+ # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
1237
+ latents = latents.to(latents_dtype)
1238
+
1239
+ if callback_on_step_end is not None:
1240
+ callback_kwargs = {}
1241
+ for k in callback_on_step_end_tensor_inputs:
1242
+ callback_kwargs[k] = locals()[k]
1243
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
1244
+
1245
+ latents = callback_outputs.pop("latents", latents)
1246
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
1247
+ add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds)
1248
+ add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids)
1249
+
1250
+ # call the callback, if provided
1251
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
1252
+ progress_bar.update()
1253
+ if callback is not None and i % callback_steps == 0:
1254
+ step_idx = i // getattr(self.scheduler, "order", 1)
1255
+ callback(step_idx, t, latents)
1256
+
1257
+ if XLA_AVAILABLE:
1258
+ xm.mark_step()
1259
+
1260
+ if not output_type == "latent":
1261
+ # make sure the VAE is in float32 mode, as it overflows in float16
1262
+ needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast
1263
+
1264
+ if needs_upcasting:
1265
+ self.upcast_vae()
1266
+ latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)
1267
+ elif latents.dtype != self.vae.dtype:
1268
+ if torch.backends.mps.is_available():
1269
+ # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
1270
+ self.vae = self.vae.to(latents.dtype)
1271
+
1272
+ # unscale/denormalize the latents
1273
+ # denormalize with the mean and std if available and not None
1274
+ has_latents_mean = hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None
1275
+ has_latents_std = hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None
1276
+ if has_latents_mean and has_latents_std:
1277
+ latents_mean = (
1278
+ torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype)
1279
+ )
1280
+ latents_std = (
1281
+ torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype)
1282
+ )
1283
+ latents = latents * latents_std / self.vae.config.scaling_factor + latents_mean
1284
+ else:
1285
+ latents = latents / self.vae.config.scaling_factor
1286
+
1287
+ image = self.vae.decode(latents, return_dict=False)[0]
1288
+
1289
+ # cast back to fp16 if needed
1290
+ if needs_upcasting:
1291
+ self.vae.to(dtype=torch.float16)
1292
+ else:
1293
+ image = latents
1294
+
1295
+ if not output_type == "latent":
1296
+ # apply watermark if available
1297
+ if self.watermark is not None:
1298
+ image = self.watermark.apply_watermark(image)
1299
+
1300
+ image = self.image_processor.postprocess(image, output_type=output_type)
1301
+
1302
+ # Offload all models
1303
+ self.maybe_free_model_hooks()
1304
+
1305
+ if not return_dict:
1306
+ return (image,)
1307
+
1308
+ return StableDiffusionXLPipelineOutput(images=image)
western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py ADDED
@@ -0,0 +1,1492 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2026 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from typing import Any, Callable
17
+
18
+ import PIL.Image
19
+ import torch
20
+ from transformers import (
21
+ CLIPImageProcessor,
22
+ CLIPTextModel,
23
+ CLIPTextModelWithProjection,
24
+ CLIPTokenizer,
25
+ CLIPVisionModelWithProjection,
26
+ )
27
+
28
+ from ...callbacks import MultiPipelineCallbacks, PipelineCallback
29
+ from ...image_processor import PipelineImageInput, VaeImageProcessor
30
+ from ...loaders import (
31
+ FromSingleFileMixin,
32
+ IPAdapterMixin,
33
+ StableDiffusionXLLoraLoaderMixin,
34
+ TextualInversionLoaderMixin,
35
+ )
36
+ from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel
37
+ from ...models.lora import adjust_lora_scale_text_encoder
38
+ from ...schedulers import KarrasDiffusionSchedulers
39
+ from ...utils import (
40
+ USE_PEFT_BACKEND,
41
+ deprecate,
42
+ is_invisible_watermark_available,
43
+ is_torch_xla_available,
44
+ logging,
45
+ replace_example_docstring,
46
+ scale_lora_layers,
47
+ unscale_lora_layers,
48
+ )
49
+ from ...utils.torch_utils import empty_device_cache, randn_tensor
50
+ from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin
51
+ from .pipeline_output import StableDiffusionXLPipelineOutput
52
+
53
+
54
+ if is_invisible_watermark_available():
55
+ from .watermark import StableDiffusionXLWatermarker
56
+
57
+ if is_torch_xla_available():
58
+ import torch_xla.core.xla_model as xm
59
+
60
+ XLA_AVAILABLE = True
61
+ else:
62
+ XLA_AVAILABLE = False
63
+
64
+
65
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
66
+
67
+ EXAMPLE_DOC_STRING = """
68
+ Examples:
69
+ ```py
70
+ >>> import torch
71
+ >>> from diffusers import StableDiffusionXLImg2ImgPipeline
72
+ >>> from diffusers.utils import load_image
73
+
74
+ >>> pipe = StableDiffusionXLImg2ImgPipeline.from_pretrained(
75
+ ... "stabilityai/stable-diffusion-xl-refiner-1.0", torch_dtype=torch.float16
76
+ ... )
77
+ >>> pipe = pipe.to("cuda")
78
+ >>> url = "https://huggingface.co/datasets/patrickvonplaten/images/resolve/main/aa_xl/000000009.png"
79
+
80
+ >>> init_image = load_image(url).convert("RGB")
81
+ >>> prompt = "a photo of an astronaut riding a horse on mars"
82
+ >>> image = pipe(prompt, image=init_image).images[0]
83
+ ```
84
+ """
85
+
86
+
87
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg
88
+ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):
89
+ r"""
90
+ Rescales `noise_cfg` tensor based on `guidance_rescale` to improve image quality and fix overexposure. Based on
91
+ Section 3.4 from [Common Diffusion Noise Schedules and Sample Steps are
92
+ Flawed](https://huggingface.co/papers/2305.08891).
93
+
94
+ Args:
95
+ noise_cfg (`torch.Tensor`):
96
+ The predicted noise tensor for the guided diffusion process.
97
+ noise_pred_text (`torch.Tensor`):
98
+ The predicted noise tensor for the text-guided diffusion process.
99
+ guidance_rescale (`float`, *optional*, defaults to 0.0):
100
+ A rescale factor applied to the noise predictions.
101
+
102
+ Returns:
103
+ noise_cfg (`torch.Tensor`): The rescaled noise prediction tensor.
104
+ """
105
+ std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)
106
+ std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)
107
+ # rescale the results from guidance (fixes overexposure)
108
+ noise_pred_rescaled = noise_cfg * (std_text / std_cfg)
109
+ # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images
110
+ noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg
111
+ return noise_cfg
112
+
113
+
114
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
115
+ def retrieve_latents(
116
+ encoder_output: torch.Tensor, generator: torch.Generator | None = None, sample_mode: str = "sample"
117
+ ):
118
+ if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
119
+ return encoder_output.latent_dist.sample(generator)
120
+ elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
121
+ return encoder_output.latent_dist.mode()
122
+ elif hasattr(encoder_output, "latents"):
123
+ return encoder_output.latents
124
+ else:
125
+ raise AttributeError("Could not access latents of provided encoder_output")
126
+
127
+
128
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
129
+ def retrieve_timesteps(
130
+ scheduler,
131
+ num_inference_steps: int | None = None,
132
+ device: str | torch.device | None = None,
133
+ timesteps: list[int] | None = None,
134
+ sigmas: list[float] | None = None,
135
+ **kwargs,
136
+ ):
137
+ r"""
138
+ Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
139
+ custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
140
+
141
+ Args:
142
+ scheduler (`SchedulerMixin`):
143
+ The scheduler to get timesteps from.
144
+ num_inference_steps (`int`):
145
+ The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
146
+ must be `None`.
147
+ device (`str` or `torch.device`, *optional*):
148
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
149
+ timesteps (`list[int]`, *optional*):
150
+ Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
151
+ `num_inference_steps` and `sigmas` must be `None`.
152
+ sigmas (`list[float]`, *optional*):
153
+ Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
154
+ `num_inference_steps` and `timesteps` must be `None`.
155
+
156
+ Returns:
157
+ `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
158
+ second element is the number of inference steps.
159
+ """
160
+ if timesteps is not None and sigmas is not None:
161
+ raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
162
+ if timesteps is not None:
163
+ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
164
+ if not accepts_timesteps:
165
+ raise ValueError(
166
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
167
+ f" timestep schedules. Please check whether you are using the correct scheduler."
168
+ )
169
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
170
+ timesteps = scheduler.timesteps
171
+ num_inference_steps = len(timesteps)
172
+ elif sigmas is not None:
173
+ accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
174
+ if not accept_sigmas:
175
+ raise ValueError(
176
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
177
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
178
+ )
179
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
180
+ timesteps = scheduler.timesteps
181
+ num_inference_steps = len(timesteps)
182
+ else:
183
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
184
+ timesteps = scheduler.timesteps
185
+ return timesteps, num_inference_steps
186
+
187
+
188
+ class StableDiffusionXLImg2ImgPipeline(
189
+ DiffusionPipeline,
190
+ StableDiffusionMixin,
191
+ TextualInversionLoaderMixin,
192
+ FromSingleFileMixin,
193
+ StableDiffusionXLLoraLoaderMixin,
194
+ IPAdapterMixin,
195
+ ):
196
+ r"""
197
+ Pipeline for text-to-image generation using Stable Diffusion XL.
198
+
199
+ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
200
+ library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
201
+
202
+ The pipeline also inherits the following loading methods:
203
+ - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings
204
+ - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files
205
+ - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights
206
+ - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights
207
+ - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters
208
+
209
+ Args:
210
+ vae ([`AutoencoderKL`]):
211
+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
212
+ text_encoder ([`CLIPTextModel`]):
213
+ Frozen text-encoder. Stable Diffusion XL uses the text portion of
214
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
215
+ the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
216
+ text_encoder_2 ([` CLIPTextModelWithProjection`]):
217
+ Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of
218
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection),
219
+ specifically the
220
+ [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k)
221
+ variant.
222
+ tokenizer (`CLIPTokenizer`):
223
+ Tokenizer of class
224
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
225
+ tokenizer_2 (`CLIPTokenizer`):
226
+ Second Tokenizer of class
227
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
228
+ unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.
229
+ scheduler ([`SchedulerMixin`]):
230
+ A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
231
+ [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
232
+ requires_aesthetics_score (`bool`, *optional*, defaults to `"False"`):
233
+ Whether the `unet` requires an `aesthetic_score` condition to be passed during inference. Also see the
234
+ config of `stabilityai/stable-diffusion-xl-refiner-1-0`.
235
+ force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`):
236
+ Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of
237
+ `stabilityai/stable-diffusion-xl-base-1-0`.
238
+ add_watermarker (`bool`, *optional*):
239
+ Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to
240
+ watermark output images. If not defined, it will default to True if the package is installed, otherwise no
241
+ watermarker will be used.
242
+ """
243
+
244
+ model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->unet->vae"
245
+ _optional_components = [
246
+ "tokenizer",
247
+ "tokenizer_2",
248
+ "text_encoder",
249
+ "text_encoder_2",
250
+ "image_encoder",
251
+ "feature_extractor",
252
+ ]
253
+ _callback_tensor_inputs = [
254
+ "latents",
255
+ "prompt_embeds",
256
+ "add_text_embeds",
257
+ "add_time_ids",
258
+ ]
259
+
260
+ def __init__(
261
+ self,
262
+ vae: AutoencoderKL,
263
+ text_encoder: CLIPTextModel,
264
+ text_encoder_2: CLIPTextModelWithProjection,
265
+ tokenizer: CLIPTokenizer,
266
+ tokenizer_2: CLIPTokenizer,
267
+ unet: UNet2DConditionModel,
268
+ scheduler: KarrasDiffusionSchedulers,
269
+ image_encoder: CLIPVisionModelWithProjection = None,
270
+ feature_extractor: CLIPImageProcessor = None,
271
+ requires_aesthetics_score: bool = False,
272
+ force_zeros_for_empty_prompt: bool = True,
273
+ add_watermarker: bool | None = None,
274
+ ):
275
+ super().__init__()
276
+
277
+ self.register_modules(
278
+ vae=vae,
279
+ text_encoder=text_encoder,
280
+ text_encoder_2=text_encoder_2,
281
+ tokenizer=tokenizer,
282
+ tokenizer_2=tokenizer_2,
283
+ unet=unet,
284
+ image_encoder=image_encoder,
285
+ feature_extractor=feature_extractor,
286
+ scheduler=scheduler,
287
+ )
288
+ self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt)
289
+ self.register_to_config(requires_aesthetics_score=requires_aesthetics_score)
290
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
291
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
292
+
293
+ add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available()
294
+
295
+ if add_watermarker:
296
+ self.watermark = StableDiffusionXLWatermarker()
297
+ else:
298
+ self.watermark = None
299
+
300
+ # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.encode_prompt
301
+ def encode_prompt(
302
+ self,
303
+ prompt: str,
304
+ prompt_2: str | None = None,
305
+ device: torch.device | None = None,
306
+ num_images_per_prompt: int = 1,
307
+ do_classifier_free_guidance: bool = True,
308
+ negative_prompt: str | None = None,
309
+ negative_prompt_2: str | None = None,
310
+ prompt_embeds: torch.Tensor | None = None,
311
+ negative_prompt_embeds: torch.Tensor | None = None,
312
+ pooled_prompt_embeds: torch.Tensor | None = None,
313
+ negative_pooled_prompt_embeds: torch.Tensor | None = None,
314
+ lora_scale: float | None = None,
315
+ clip_skip: int | None = None,
316
+ ):
317
+ r"""
318
+ Encodes the prompt into text encoder hidden states.
319
+
320
+ Args:
321
+ prompt (`str` or `list[str]`, *optional*):
322
+ prompt to be encoded
323
+ prompt_2 (`str` or `list[str]`, *optional*):
324
+ The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
325
+ used in both text-encoders
326
+ device: (`torch.device`):
327
+ torch device
328
+ num_images_per_prompt (`int`):
329
+ number of images that should be generated per prompt
330
+ do_classifier_free_guidance (`bool`):
331
+ whether to use classifier free guidance or not
332
+ negative_prompt (`str` or `list[str]`, *optional*):
333
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
334
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
335
+ less than `1`).
336
+ negative_prompt_2 (`str` or `list[str]`, *optional*):
337
+ The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
338
+ `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
339
+ prompt_embeds (`torch.Tensor`, *optional*):
340
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
341
+ provided, text embeddings will be generated from `prompt` input argument.
342
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
343
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
344
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
345
+ argument.
346
+ pooled_prompt_embeds (`torch.Tensor`, *optional*):
347
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
348
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
349
+ negative_pooled_prompt_embeds (`torch.Tensor`, *optional*):
350
+ Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
351
+ weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
352
+ input argument.
353
+ lora_scale (`float`, *optional*):
354
+ A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
355
+ clip_skip (`int`, *optional*):
356
+ Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
357
+ the output of the pre-final layer will be used for computing the prompt embeddings.
358
+ """
359
+ device = device or self._execution_device
360
+
361
+ # set lora scale so that monkey patched LoRA
362
+ # function of text encoder can correctly access it
363
+ if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin):
364
+ self._lora_scale = lora_scale
365
+
366
+ # dynamically adjust the LoRA scale
367
+ if self.text_encoder is not None:
368
+ if not USE_PEFT_BACKEND:
369
+ adjust_lora_scale_text_encoder(self.text_encoder, lora_scale)
370
+ else:
371
+ scale_lora_layers(self.text_encoder, lora_scale)
372
+
373
+ if self.text_encoder_2 is not None:
374
+ if not USE_PEFT_BACKEND:
375
+ adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale)
376
+ else:
377
+ scale_lora_layers(self.text_encoder_2, lora_scale)
378
+
379
+ prompt = [prompt] if isinstance(prompt, str) else prompt
380
+
381
+ if prompt is not None:
382
+ batch_size = len(prompt)
383
+ else:
384
+ batch_size = prompt_embeds.shape[0]
385
+
386
+ # Define tokenizers and text encoders
387
+ tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2]
388
+ text_encoders = (
389
+ [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2]
390
+ )
391
+
392
+ if prompt_embeds is None:
393
+ prompt_2 = prompt_2 or prompt
394
+ prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2
395
+
396
+ # textual inversion: process multi-vector tokens if necessary
397
+ prompt_embeds_list = []
398
+ prompts = [prompt, prompt_2]
399
+ for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders):
400
+ if isinstance(self, TextualInversionLoaderMixin):
401
+ prompt = self.maybe_convert_prompt(prompt, tokenizer)
402
+
403
+ text_inputs = tokenizer(
404
+ prompt,
405
+ padding="max_length",
406
+ max_length=tokenizer.model_max_length,
407
+ truncation=True,
408
+ return_tensors="pt",
409
+ )
410
+
411
+ text_input_ids = text_inputs.input_ids
412
+ untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
413
+
414
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(
415
+ text_input_ids, untruncated_ids
416
+ ):
417
+ removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1])
418
+ logger.warning(
419
+ "The following part of your input was truncated because CLIP can only handle sequences up to"
420
+ f" {tokenizer.model_max_length} tokens: {removed_text}"
421
+ )
422
+
423
+ prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True)
424
+
425
+ # We are only ALWAYS interested in the pooled output of the final text encoder
426
+ if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2:
427
+ pooled_prompt_embeds = prompt_embeds[0]
428
+
429
+ if clip_skip is None:
430
+ prompt_embeds = prompt_embeds.hidden_states[-2]
431
+ else:
432
+ # "2" because SDXL always indexes from the penultimate layer.
433
+ prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)]
434
+
435
+ prompt_embeds_list.append(prompt_embeds)
436
+
437
+ prompt_embeds = torch.concat(prompt_embeds_list, dim=-1)
438
+
439
+ # get unconditional embeddings for classifier free guidance
440
+ zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt
441
+ if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt:
442
+ negative_prompt_embeds = torch.zeros_like(prompt_embeds)
443
+ negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds)
444
+ elif do_classifier_free_guidance and negative_prompt_embeds is None:
445
+ negative_prompt = negative_prompt or ""
446
+ negative_prompt_2 = negative_prompt_2 or negative_prompt
447
+
448
+ # normalize str to list
449
+ negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
450
+ negative_prompt_2 = (
451
+ batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2
452
+ )
453
+
454
+ uncond_tokens: list[str]
455
+ if prompt is not None and type(prompt) is not type(negative_prompt):
456
+ raise TypeError(
457
+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
458
+ f" {type(prompt)}."
459
+ )
460
+ elif batch_size != len(negative_prompt):
461
+ raise ValueError(
462
+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
463
+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
464
+ " the batch size of `prompt`."
465
+ )
466
+ else:
467
+ uncond_tokens = [negative_prompt, negative_prompt_2]
468
+
469
+ negative_prompt_embeds_list = []
470
+ for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders):
471
+ if isinstance(self, TextualInversionLoaderMixin):
472
+ negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer)
473
+
474
+ max_length = prompt_embeds.shape[1]
475
+ uncond_input = tokenizer(
476
+ negative_prompt,
477
+ padding="max_length",
478
+ max_length=max_length,
479
+ truncation=True,
480
+ return_tensors="pt",
481
+ )
482
+
483
+ negative_prompt_embeds = text_encoder(
484
+ uncond_input.input_ids.to(device),
485
+ output_hidden_states=True,
486
+ )
487
+
488
+ # We are only ALWAYS interested in the pooled output of the final text encoder
489
+ if negative_pooled_prompt_embeds is None and negative_prompt_embeds[0].ndim == 2:
490
+ negative_pooled_prompt_embeds = negative_prompt_embeds[0]
491
+ negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2]
492
+
493
+ negative_prompt_embeds_list.append(negative_prompt_embeds)
494
+
495
+ negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1)
496
+
497
+ if self.text_encoder_2 is not None:
498
+ prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)
499
+ else:
500
+ prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device)
501
+
502
+ bs_embed, seq_len, _ = prompt_embeds.shape
503
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
504
+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
505
+ prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
506
+
507
+ if do_classifier_free_guidance:
508
+ # duplicate unconditional embeddings for each generation per prompt, using mps friendly method
509
+ seq_len = negative_prompt_embeds.shape[1]
510
+
511
+ if self.text_encoder_2 is not None:
512
+ negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)
513
+ else:
514
+ negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device)
515
+
516
+ negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
517
+ negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
518
+
519
+ pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
520
+ bs_embed * num_images_per_prompt, -1
521
+ )
522
+ if do_classifier_free_guidance:
523
+ negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
524
+ bs_embed * num_images_per_prompt, -1
525
+ )
526
+
527
+ if self.text_encoder is not None:
528
+ if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND:
529
+ # Retrieve the original scale by scaling back the LoRA layers
530
+ unscale_lora_layers(self.text_encoder, lora_scale)
531
+
532
+ if self.text_encoder_2 is not None:
533
+ if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND:
534
+ # Retrieve the original scale by scaling back the LoRA layers
535
+ unscale_lora_layers(self.text_encoder_2, lora_scale)
536
+
537
+ return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds
538
+
539
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs
540
+ def prepare_extra_step_kwargs(self, generator, eta):
541
+ # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
542
+ # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
543
+ # eta corresponds to η in DDIM paper: https://huggingface.co/papers/2010.02502
544
+ # and should be between [0, 1]
545
+
546
+ accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
547
+ extra_step_kwargs = {}
548
+ if accepts_eta:
549
+ extra_step_kwargs["eta"] = eta
550
+
551
+ # check if the scheduler accepts generator
552
+ accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
553
+ if accepts_generator:
554
+ extra_step_kwargs["generator"] = generator
555
+ return extra_step_kwargs
556
+
557
+ def check_inputs(
558
+ self,
559
+ prompt,
560
+ prompt_2,
561
+ strength,
562
+ num_inference_steps,
563
+ callback_steps,
564
+ negative_prompt=None,
565
+ negative_prompt_2=None,
566
+ prompt_embeds=None,
567
+ negative_prompt_embeds=None,
568
+ ip_adapter_image=None,
569
+ ip_adapter_image_embeds=None,
570
+ callback_on_step_end_tensor_inputs=None,
571
+ ):
572
+ if strength < 0 or strength > 1:
573
+ raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}")
574
+ if num_inference_steps is None:
575
+ raise ValueError("`num_inference_steps` cannot be None.")
576
+ elif not isinstance(num_inference_steps, int) or num_inference_steps <= 0:
577
+ raise ValueError(
578
+ f"`num_inference_steps` has to be a positive integer but is {num_inference_steps} of type"
579
+ f" {type(num_inference_steps)}."
580
+ )
581
+ if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0):
582
+ raise ValueError(
583
+ f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
584
+ f" {type(callback_steps)}."
585
+ )
586
+
587
+ if callback_on_step_end_tensor_inputs is not None and not all(
588
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
589
+ ):
590
+ raise ValueError(
591
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
592
+ )
593
+
594
+ if prompt is not None and prompt_embeds is not None:
595
+ raise ValueError(
596
+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
597
+ " only forward one of the two."
598
+ )
599
+ elif prompt_2 is not None and prompt_embeds is not None:
600
+ raise ValueError(
601
+ f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
602
+ " only forward one of the two."
603
+ )
604
+ elif prompt is None and prompt_embeds is None:
605
+ raise ValueError(
606
+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
607
+ )
608
+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
609
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
610
+ elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)):
611
+ raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}")
612
+
613
+ if negative_prompt is not None and negative_prompt_embeds is not None:
614
+ raise ValueError(
615
+ f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
616
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
617
+ )
618
+ elif negative_prompt_2 is not None and negative_prompt_embeds is not None:
619
+ raise ValueError(
620
+ f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:"
621
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
622
+ )
623
+
624
+ if prompt_embeds is not None and negative_prompt_embeds is not None:
625
+ if prompt_embeds.shape != negative_prompt_embeds.shape:
626
+ raise ValueError(
627
+ "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"
628
+ f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"
629
+ f" {negative_prompt_embeds.shape}."
630
+ )
631
+
632
+ if ip_adapter_image is not None and ip_adapter_image_embeds is not None:
633
+ raise ValueError(
634
+ "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined."
635
+ )
636
+
637
+ if ip_adapter_image_embeds is not None:
638
+ if not isinstance(ip_adapter_image_embeds, list):
639
+ raise ValueError(
640
+ f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}"
641
+ )
642
+ elif ip_adapter_image_embeds[0].ndim not in [3, 4]:
643
+ raise ValueError(
644
+ f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D"
645
+ )
646
+
647
+ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=None):
648
+ # get the original timestep using init_timestep
649
+ if denoising_start is None:
650
+ init_timestep = min(int(num_inference_steps * strength), num_inference_steps)
651
+ t_start = max(num_inference_steps - init_timestep, 0)
652
+
653
+ timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :]
654
+ if hasattr(self.scheduler, "set_begin_index"):
655
+ self.scheduler.set_begin_index(t_start * self.scheduler.order)
656
+
657
+ return timesteps, num_inference_steps - t_start
658
+
659
+ else:
660
+ # Strength is irrelevant if we directly request a timestep to start at;
661
+ # that is, strength is determined by the denoising_start instead.
662
+ discrete_timestep_cutoff = int(
663
+ round(
664
+ self.scheduler.config.num_train_timesteps
665
+ - (denoising_start * self.scheduler.config.num_train_timesteps)
666
+ )
667
+ )
668
+
669
+ num_inference_steps = (self.scheduler.timesteps < discrete_timestep_cutoff).sum().item()
670
+ if self.scheduler.order == 2 and num_inference_steps % 2 == 0:
671
+ # if the scheduler is a 2nd order scheduler we might have to do +1
672
+ # because `num_inference_steps` might be even given that every timestep
673
+ # (except the highest one) is duplicated. If `num_inference_steps` is even it would
674
+ # mean that we cut the timesteps in the middle of the denoising step
675
+ # (between 1st and 2nd derivative) which leads to incorrect results. By adding 1
676
+ # we ensure that the denoising process always ends after the 2nd derivate step of the scheduler
677
+ num_inference_steps = num_inference_steps + 1
678
+
679
+ # because t_n+1 >= t_n, we slice the timesteps starting from the end
680
+ t_start = len(self.scheduler.timesteps) - num_inference_steps
681
+ timesteps = self.scheduler.timesteps[t_start:]
682
+ if hasattr(self.scheduler, "set_begin_index"):
683
+ self.scheduler.set_begin_index(t_start)
684
+ return timesteps, num_inference_steps
685
+
686
+ def prepare_latents(
687
+ self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None, add_noise=True
688
+ ):
689
+ if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)):
690
+ raise ValueError(
691
+ f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}"
692
+ )
693
+
694
+ latents_mean = latents_std = None
695
+ if hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None:
696
+ latents_mean = torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1)
697
+ if hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None:
698
+ latents_std = torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1)
699
+
700
+ # Offload text encoder if `enable_model_cpu_offload` was enabled
701
+ if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:
702
+ self.text_encoder_2.to("cpu")
703
+ empty_device_cache()
704
+
705
+ image = image.to(device=device, dtype=dtype)
706
+
707
+ batch_size = batch_size * num_images_per_prompt
708
+
709
+ if image.shape[1] == 4:
710
+ init_latents = image
711
+
712
+ else:
713
+ # make sure the VAE is in float32 mode, as it overflows in float16
714
+ if self.vae.config.force_upcast:
715
+ image = image.float()
716
+ self.vae.to(dtype=torch.float32)
717
+
718
+ if isinstance(generator, list) and len(generator) != batch_size:
719
+ raise ValueError(
720
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
721
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
722
+ )
723
+
724
+ elif isinstance(generator, list):
725
+ if image.shape[0] < batch_size and batch_size % image.shape[0] == 0:
726
+ image = torch.cat([image] * (batch_size // image.shape[0]), dim=0)
727
+ elif image.shape[0] < batch_size and batch_size % image.shape[0] != 0:
728
+ raise ValueError(
729
+ f"Cannot duplicate `image` of batch size {image.shape[0]} to effective batch_size {batch_size} "
730
+ )
731
+
732
+ init_latents = [
733
+ retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i])
734
+ for i in range(batch_size)
735
+ ]
736
+ init_latents = torch.cat(init_latents, dim=0)
737
+ else:
738
+ init_latents = retrieve_latents(self.vae.encode(image), generator=generator)
739
+
740
+ if self.vae.config.force_upcast:
741
+ self.vae.to(dtype)
742
+
743
+ init_latents = init_latents.to(dtype)
744
+ if latents_mean is not None and latents_std is not None:
745
+ latents_mean = latents_mean.to(device=device, dtype=dtype)
746
+ latents_std = latents_std.to(device=device, dtype=dtype)
747
+ init_latents = (init_latents - latents_mean) * self.vae.config.scaling_factor / latents_std
748
+ else:
749
+ init_latents = self.vae.config.scaling_factor * init_latents
750
+
751
+ if batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] == 0:
752
+ # expand init_latents for batch_size
753
+ additional_image_per_prompt = batch_size // init_latents.shape[0]
754
+ init_latents = torch.cat([init_latents] * additional_image_per_prompt, dim=0)
755
+ elif batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] != 0:
756
+ raise ValueError(
757
+ f"Cannot duplicate `image` of batch size {init_latents.shape[0]} to {batch_size} text prompts."
758
+ )
759
+ else:
760
+ init_latents = torch.cat([init_latents], dim=0)
761
+
762
+ if add_noise:
763
+ shape = init_latents.shape
764
+ noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
765
+ # get latents
766
+ init_latents = self.scheduler.add_noise(init_latents, noise, timestep)
767
+
768
+ latents = init_latents
769
+
770
+ return latents
771
+
772
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image
773
+ def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None):
774
+ dtype = next(self.image_encoder.parameters()).dtype
775
+
776
+ if not isinstance(image, torch.Tensor):
777
+ image = self.feature_extractor(image, return_tensors="pt").pixel_values
778
+
779
+ image = image.to(device=device, dtype=dtype)
780
+ if output_hidden_states:
781
+ image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2]
782
+ image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0)
783
+ uncond_image_enc_hidden_states = self.image_encoder(
784
+ torch.zeros_like(image), output_hidden_states=True
785
+ ).hidden_states[-2]
786
+ uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave(
787
+ num_images_per_prompt, dim=0
788
+ )
789
+ return image_enc_hidden_states, uncond_image_enc_hidden_states
790
+ else:
791
+ image_embeds = self.image_encoder(image).image_embeds
792
+ image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0)
793
+ uncond_image_embeds = torch.zeros_like(image_embeds)
794
+
795
+ return image_embeds, uncond_image_embeds
796
+
797
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds
798
+ def prepare_ip_adapter_image_embeds(
799
+ self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance
800
+ ):
801
+ image_embeds = []
802
+ if do_classifier_free_guidance:
803
+ negative_image_embeds = []
804
+ if ip_adapter_image_embeds is None:
805
+ if not isinstance(ip_adapter_image, list):
806
+ ip_adapter_image = [ip_adapter_image]
807
+
808
+ if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers):
809
+ raise ValueError(
810
+ f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters."
811
+ )
812
+
813
+ for single_ip_adapter_image, image_proj_layer in zip(
814
+ ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers
815
+ ):
816
+ output_hidden_state = not isinstance(image_proj_layer, ImageProjection)
817
+ single_image_embeds, single_negative_image_embeds = self.encode_image(
818
+ single_ip_adapter_image, device, 1, output_hidden_state
819
+ )
820
+
821
+ image_embeds.append(single_image_embeds[None, :])
822
+ if do_classifier_free_guidance:
823
+ negative_image_embeds.append(single_negative_image_embeds[None, :])
824
+ else:
825
+ for single_image_embeds in ip_adapter_image_embeds:
826
+ if do_classifier_free_guidance:
827
+ single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2)
828
+ negative_image_embeds.append(single_negative_image_embeds)
829
+ image_embeds.append(single_image_embeds)
830
+
831
+ ip_adapter_image_embeds = []
832
+ for i, single_image_embeds in enumerate(image_embeds):
833
+ single_image_embeds = torch.cat([single_image_embeds] * num_images_per_prompt, dim=0)
834
+ if do_classifier_free_guidance:
835
+ single_negative_image_embeds = torch.cat([negative_image_embeds[i]] * num_images_per_prompt, dim=0)
836
+ single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds], dim=0)
837
+
838
+ single_image_embeds = single_image_embeds.to(device=device)
839
+ ip_adapter_image_embeds.append(single_image_embeds)
840
+
841
+ return ip_adapter_image_embeds
842
+
843
+ def _get_add_time_ids(
844
+ self,
845
+ original_size,
846
+ crops_coords_top_left,
847
+ target_size,
848
+ aesthetic_score,
849
+ negative_aesthetic_score,
850
+ negative_original_size,
851
+ negative_crops_coords_top_left,
852
+ negative_target_size,
853
+ dtype,
854
+ text_encoder_projection_dim=None,
855
+ ):
856
+ if self.config.requires_aesthetics_score:
857
+ add_time_ids = list(original_size + crops_coords_top_left + (aesthetic_score,))
858
+ add_neg_time_ids = list(
859
+ negative_original_size + negative_crops_coords_top_left + (negative_aesthetic_score,)
860
+ )
861
+ else:
862
+ add_time_ids = list(original_size + crops_coords_top_left + target_size)
863
+ add_neg_time_ids = list(negative_original_size + crops_coords_top_left + negative_target_size)
864
+
865
+ passed_add_embed_dim = (
866
+ self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim
867
+ )
868
+ expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features
869
+
870
+ if (
871
+ expected_add_embed_dim > passed_add_embed_dim
872
+ and (expected_add_embed_dim - passed_add_embed_dim) == self.unet.config.addition_time_embed_dim
873
+ ):
874
+ raise ValueError(
875
+ f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. Please make sure to enable `requires_aesthetics_score` with `pipe.register_to_config(requires_aesthetics_score=True)` to make sure `aesthetic_score` {aesthetic_score} and `negative_aesthetic_score` {negative_aesthetic_score} is correctly used by the model."
876
+ )
877
+ elif (
878
+ expected_add_embed_dim < passed_add_embed_dim
879
+ and (passed_add_embed_dim - expected_add_embed_dim) == self.unet.config.addition_time_embed_dim
880
+ ):
881
+ raise ValueError(
882
+ f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. Please make sure to disable `requires_aesthetics_score` with `pipe.register_to_config(requires_aesthetics_score=False)` to make sure `target_size` {target_size} is correctly used by the model."
883
+ )
884
+ elif expected_add_embed_dim != passed_add_embed_dim:
885
+ raise ValueError(
886
+ f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`."
887
+ )
888
+
889
+ add_time_ids = torch.tensor([add_time_ids], dtype=dtype)
890
+ add_neg_time_ids = torch.tensor([add_neg_time_ids], dtype=dtype)
891
+
892
+ return add_time_ids, add_neg_time_ids
893
+
894
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.StableDiffusionUpscalePipeline.upcast_vae
895
+ def upcast_vae(self):
896
+ deprecate(
897
+ "upcast_vae",
898
+ "1.0.0",
899
+ "`upcast_vae` is deprecated. Please use `pipe.vae.to(torch.float32)`. For more details, please refer to: https://github.com/huggingface/diffusers/pull/12619#issue-3606633695.",
900
+ )
901
+ self.vae.to(dtype=torch.float32)
902
+
903
+ # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding
904
+ def get_guidance_scale_embedding(
905
+ self, w: torch.Tensor, embedding_dim: int = 512, dtype: torch.dtype = torch.float32
906
+ ) -> torch.Tensor:
907
+ """
908
+ See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298
909
+
910
+ Args:
911
+ w (`torch.Tensor`):
912
+ Generate embedding vectors with a specified guidance scale to subsequently enrich timestep embeddings.
913
+ embedding_dim (`int`, *optional*, defaults to 512):
914
+ Dimension of the embeddings to generate.
915
+ dtype (`torch.dtype`, *optional*, defaults to `torch.float32`):
916
+ Data type of the generated embeddings.
917
+
918
+ Returns:
919
+ `torch.Tensor`: Embedding vectors with shape `(len(w), embedding_dim)`.
920
+ """
921
+ assert len(w.shape) == 1
922
+ w = w * 1000.0
923
+
924
+ half_dim = embedding_dim // 2
925
+ emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1)
926
+ emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb)
927
+ emb = w.to(dtype)[:, None] * emb[None, :]
928
+ emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)
929
+ if embedding_dim % 2 == 1: # zero pad
930
+ emb = torch.nn.functional.pad(emb, (0, 1))
931
+ assert emb.shape == (w.shape[0], embedding_dim)
932
+ return emb
933
+
934
+ @property
935
+ def guidance_scale(self):
936
+ return self._guidance_scale
937
+
938
+ @property
939
+ def guidance_rescale(self):
940
+ return self._guidance_rescale
941
+
942
+ @property
943
+ def clip_skip(self):
944
+ return self._clip_skip
945
+
946
+ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
947
+ # of the Imagen paper: https://huggingface.co/papers/2205.11487 . `guidance_scale = 1`
948
+ # corresponds to doing no classifier free guidance.
949
+ @property
950
+ def do_classifier_free_guidance(self):
951
+ return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None
952
+
953
+ @property
954
+ def cross_attention_kwargs(self):
955
+ return self._cross_attention_kwargs
956
+
957
+ @property
958
+ def denoising_end(self):
959
+ return self._denoising_end
960
+
961
+ @property
962
+ def denoising_start(self):
963
+ return self._denoising_start
964
+
965
+ @property
966
+ def num_timesteps(self):
967
+ return self._num_timesteps
968
+
969
+ @property
970
+ def interrupt(self):
971
+ return self._interrupt
972
+
973
+ @torch.no_grad()
974
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
975
+ def __call__(
976
+ self,
977
+ prompt: str | list[str] = None,
978
+ prompt_2: str | list[str] | None = None,
979
+ image: PipelineImageInput = None,
980
+ strength: float = 0.3,
981
+ num_inference_steps: int = 50,
982
+ timesteps: list[int] = None,
983
+ sigmas: list[float] = None,
984
+ denoising_start: float | None = None,
985
+ denoising_end: float | None = None,
986
+ guidance_scale: float = 5.0,
987
+ negative_prompt: str | list[str] | None = None,
988
+ negative_prompt_2: str | list[str] | None = None,
989
+ num_images_per_prompt: int | None = 1,
990
+ eta: float = 0.0,
991
+ generator: torch.Generator | list[torch.Generator] | None = None,
992
+ latents: torch.Tensor | None = None,
993
+ prompt_embeds: torch.Tensor | None = None,
994
+ negative_prompt_embeds: torch.Tensor | None = None,
995
+ pooled_prompt_embeds: torch.Tensor | None = None,
996
+ negative_pooled_prompt_embeds: torch.Tensor | None = None,
997
+ ip_adapter_image: PipelineImageInput | None = None,
998
+ ip_adapter_image_embeds: list[torch.Tensor] | None = None,
999
+ output_type: str | None = "pil",
1000
+ return_dict: bool = True,
1001
+ cross_attention_kwargs: dict[str, Any] | None = None,
1002
+ guidance_rescale: float = 0.0,
1003
+ original_size: tuple[int, int] = None,
1004
+ crops_coords_top_left: tuple[int, int] = (0, 0),
1005
+ target_size: tuple[int, int] = None,
1006
+ negative_original_size: tuple[int, int] | None = None,
1007
+ negative_crops_coords_top_left: tuple[int, int] = (0, 0),
1008
+ negative_target_size: tuple[int, int] | None = None,
1009
+ aesthetic_score: float = 6.0,
1010
+ negative_aesthetic_score: float = 2.5,
1011
+ clip_skip: int | None = None,
1012
+ callback_on_step_end: Callable[[int, int], None] | PipelineCallback | MultiPipelineCallbacks | None = None,
1013
+ callback_on_step_end_tensor_inputs: list[str] = ["latents"],
1014
+ **kwargs,
1015
+ ):
1016
+ r"""
1017
+ Function invoked when calling the pipeline for generation.
1018
+
1019
+ Args:
1020
+ prompt (`str` or `list[str]`, *optional*):
1021
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
1022
+ instead.
1023
+ prompt_2 (`str` or `list[str]`, *optional*):
1024
+ The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
1025
+ used in both text-encoders
1026
+ image (`torch.Tensor` or `PIL.Image.Image` or `np.ndarray` or `list[torch.Tensor]` or `list[PIL.Image.Image]` or `list[np.ndarray]`):
1027
+ The image(s) to modify with the pipeline.
1028
+ strength (`float`, *optional*, defaults to 0.3):
1029
+ Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image`
1030
+ will be used as a starting point, adding more noise to it the larger the `strength`. The number of
1031
+ denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will
1032
+ be maximum and the denoising process will run for the full number of iterations specified in
1033
+ `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. Note that in the case of
1034
+ `denoising_start` being declared as an integer, the value of `strength` will be ignored.
1035
+ num_inference_steps (`int`, *optional*, defaults to 50):
1036
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
1037
+ expense of slower inference.
1038
+ timesteps (`list[int]`, *optional*):
1039
+ Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument
1040
+ in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is
1041
+ passed will be used. Must be in descending order.
1042
+ sigmas (`list[float]`, *optional*):
1043
+ Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
1044
+ their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
1045
+ will be used.
1046
+ denoising_start (`float`, *optional*):
1047
+ When specified, indicates the fraction (between 0.0 and 1.0) of the total denoising process to be
1048
+ bypassed before it is initiated. Consequently, the initial part of the denoising process is skipped and
1049
+ it is assumed that the passed `image` is a partly denoised image. Note that when this is specified,
1050
+ strength will be ignored. The `denoising_start` parameter is particularly beneficial when this pipeline
1051
+ is integrated into a "Mixture of Denoisers" multi-pipeline setup, as detailed in [**Refine Image
1052
+ Quality**](https://huggingface.co/docs/diffusers/using-diffusers/sdxl#refine-image-quality).
1053
+ denoising_end (`float`, *optional*):
1054
+ When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be
1055
+ completed before it is intentionally prematurely terminated. As a result, the returned sample will
1056
+ still retain a substantial amount of noise (ca. final 20% of timesteps still needed) and should be
1057
+ denoised by a successor pipeline that has `denoising_start` set to 0.8 so that it only denoises the
1058
+ final 20% of the scheduler. The denoising_end parameter should ideally be utilized when this pipeline
1059
+ forms a part of a "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refine Image
1060
+ Quality**](https://huggingface.co/docs/diffusers/using-diffusers/sdxl#refine-image-quality).
1061
+ guidance_scale (`float`, *optional*, defaults to 7.5):
1062
+ Guidance scale as defined in [Classifier-Free Diffusion
1063
+ Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2.
1064
+ of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
1065
+ `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to
1066
+ the text `prompt`, usually at the expense of lower image quality.
1067
+ negative_prompt (`str` or `list[str]`, *optional*):
1068
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
1069
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
1070
+ less than `1`).
1071
+ negative_prompt_2 (`str` or `list[str]`, *optional*):
1072
+ The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
1073
+ `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
1074
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
1075
+ The number of images to generate per prompt.
1076
+ eta (`float`, *optional*, defaults to 0.0):
1077
+ Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only
1078
+ applies to [`schedulers.DDIMScheduler`], will be ignored for others.
1079
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
1080
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
1081
+ to make generation deterministic.
1082
+ latents (`torch.Tensor`, *optional*):
1083
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
1084
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
1085
+ tensor will be generated by sampling using the supplied random `generator`.
1086
+ prompt_embeds (`torch.Tensor`, *optional*):
1087
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
1088
+ provided, text embeddings will be generated from `prompt` input argument.
1089
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
1090
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
1091
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
1092
+ argument.
1093
+ pooled_prompt_embeds (`torch.Tensor`, *optional*):
1094
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
1095
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
1096
+ negative_pooled_prompt_embeds (`torch.Tensor`, *optional*):
1097
+ Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
1098
+ weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
1099
+ input argument.
1100
+ ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.
1101
+ ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*):
1102
+ Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of
1103
+ IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should
1104
+ contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not
1105
+ provided, embeddings are computed from the `ip_adapter_image` input argument.
1106
+ output_type (`str`, *optional*, defaults to `"pil"`):
1107
+ The output format of the generate image. Choose between
1108
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
1109
+ return_dict (`bool`, *optional*, defaults to `True`):
1110
+ Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] instead of a
1111
+ plain tuple.
1112
+ cross_attention_kwargs (`dict`, *optional*):
1113
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
1114
+ `self.processor` in
1115
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
1116
+ guidance_rescale (`float`, *optional*, defaults to 0.0):
1117
+ Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are
1118
+ Flawed](https://huggingface.co/papers/2305.08891) `guidance_scale` is defined as `φ` in equation 16. of
1119
+ [Common Diffusion Noise Schedules and Sample Steps are
1120
+ Flawed](https://huggingface.co/papers/2305.08891). Guidance rescale factor should fix overexposure when
1121
+ using zero terminal SNR.
1122
+ original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
1123
+ If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled.
1124
+ `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as
1125
+ explained in section 2.2 of
1126
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
1127
+ crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)):
1128
+ `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position
1129
+ `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting
1130
+ `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of
1131
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
1132
+ target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
1133
+ For most cases, `target_size` should be set to the desired height and width of the generated image. If
1134
+ not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in
1135
+ section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
1136
+ negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
1137
+ To negatively condition the generation process based on a specific image resolution. Part of SDXL's
1138
+ micro-conditioning as explained in section 2.2 of
1139
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
1140
+ information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
1141
+ negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)):
1142
+ To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's
1143
+ micro-conditioning as explained in section 2.2 of
1144
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
1145
+ information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
1146
+ negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
1147
+ To negatively condition the generation process based on a target image resolution. It should be as same
1148
+ as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of
1149
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
1150
+ information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
1151
+ aesthetic_score (`float`, *optional*, defaults to 6.0):
1152
+ Used to simulate an aesthetic score of the generated image by influencing the positive text condition.
1153
+ Part of SDXL's micro-conditioning as explained in section 2.2 of
1154
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
1155
+ negative_aesthetic_score (`float`, *optional*, defaults to 2.5):
1156
+ Part of SDXL's micro-conditioning as explained in section 2.2 of
1157
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). Can be used to
1158
+ simulate an aesthetic score of the generated image by influencing the negative text condition.
1159
+ clip_skip (`int`, *optional*):
1160
+ Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
1161
+ the output of the pre-final layer will be used for computing the prompt embeddings.
1162
+ callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*):
1163
+ A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of
1164
+ each denoising step during the inference. with the following arguments: `callback_on_step_end(self:
1165
+ DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a
1166
+ list of all tensors as specified by `callback_on_step_end_tensor_inputs`.
1167
+ callback_on_step_end_tensor_inputs (`list`, *optional*):
1168
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
1169
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
1170
+ `._callback_tensor_inputs` attribute of your pipeline class.
1171
+
1172
+ Examples:
1173
+
1174
+ Returns:
1175
+ [`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] or `tuple`:
1176
+ [`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a
1177
+ `tuple. When returning a tuple, the first element is a list with the generated images.
1178
+ """
1179
+
1180
+ callback = kwargs.pop("callback", None)
1181
+ callback_steps = kwargs.pop("callback_steps", None)
1182
+
1183
+ if callback is not None:
1184
+ deprecate(
1185
+ "callback",
1186
+ "1.0.0",
1187
+ "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
1188
+ )
1189
+ if callback_steps is not None:
1190
+ deprecate(
1191
+ "callback_steps",
1192
+ "1.0.0",
1193
+ "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
1194
+ )
1195
+
1196
+ if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
1197
+ callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
1198
+
1199
+ # 1. Check inputs. Raise error if not correct
1200
+ self.check_inputs(
1201
+ prompt,
1202
+ prompt_2,
1203
+ strength,
1204
+ num_inference_steps,
1205
+ callback_steps,
1206
+ negative_prompt,
1207
+ negative_prompt_2,
1208
+ prompt_embeds,
1209
+ negative_prompt_embeds,
1210
+ ip_adapter_image,
1211
+ ip_adapter_image_embeds,
1212
+ callback_on_step_end_tensor_inputs,
1213
+ )
1214
+
1215
+ self._guidance_scale = guidance_scale
1216
+ self._guidance_rescale = guidance_rescale
1217
+ self._clip_skip = clip_skip
1218
+ self._cross_attention_kwargs = cross_attention_kwargs
1219
+ self._denoising_end = denoising_end
1220
+ self._denoising_start = denoising_start
1221
+ self._interrupt = False
1222
+
1223
+ # 2. Define call parameters
1224
+ if prompt is not None and isinstance(prompt, str):
1225
+ batch_size = 1
1226
+ elif prompt is not None and isinstance(prompt, list):
1227
+ batch_size = len(prompt)
1228
+ else:
1229
+ batch_size = prompt_embeds.shape[0]
1230
+
1231
+ device = self._execution_device
1232
+
1233
+ # 3. Encode input prompt
1234
+ text_encoder_lora_scale = (
1235
+ self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None
1236
+ )
1237
+ (
1238
+ prompt_embeds,
1239
+ negative_prompt_embeds,
1240
+ pooled_prompt_embeds,
1241
+ negative_pooled_prompt_embeds,
1242
+ ) = self.encode_prompt(
1243
+ prompt=prompt,
1244
+ prompt_2=prompt_2,
1245
+ device=device,
1246
+ num_images_per_prompt=num_images_per_prompt,
1247
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
1248
+ negative_prompt=negative_prompt,
1249
+ negative_prompt_2=negative_prompt_2,
1250
+ prompt_embeds=prompt_embeds,
1251
+ negative_prompt_embeds=negative_prompt_embeds,
1252
+ pooled_prompt_embeds=pooled_prompt_embeds,
1253
+ negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
1254
+ lora_scale=text_encoder_lora_scale,
1255
+ clip_skip=self.clip_skip,
1256
+ )
1257
+
1258
+ # 4. Preprocess image
1259
+ image = self.image_processor.preprocess(image)
1260
+
1261
+ # 5. Prepare timesteps
1262
+ def denoising_value_valid(dnv):
1263
+ return isinstance(dnv, float) and 0 < dnv < 1
1264
+
1265
+ if XLA_AVAILABLE:
1266
+ timestep_device = "cpu"
1267
+ else:
1268
+ timestep_device = device
1269
+ timesteps, num_inference_steps = retrieve_timesteps(
1270
+ self.scheduler, num_inference_steps, timestep_device, timesteps, sigmas
1271
+ )
1272
+ timesteps, num_inference_steps = self.get_timesteps(
1273
+ num_inference_steps,
1274
+ strength,
1275
+ device,
1276
+ denoising_start=self.denoising_start if denoising_value_valid(self.denoising_start) else None,
1277
+ )
1278
+ latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
1279
+
1280
+ add_noise = True if self.denoising_start is None else False
1281
+
1282
+ # 6. Prepare latent variables
1283
+ if latents is None:
1284
+ latents = self.prepare_latents(
1285
+ image,
1286
+ latent_timestep,
1287
+ batch_size,
1288
+ num_images_per_prompt,
1289
+ prompt_embeds.dtype,
1290
+ device,
1291
+ generator,
1292
+ add_noise,
1293
+ )
1294
+ # 7. Prepare extra step kwargs.
1295
+ extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
1296
+
1297
+ height, width = latents.shape[-2:]
1298
+ height = height * self.vae_scale_factor
1299
+ width = width * self.vae_scale_factor
1300
+
1301
+ original_size = original_size or (height, width)
1302
+ target_size = target_size or (height, width)
1303
+
1304
+ # 8. Prepare added time ids & embeddings
1305
+ if negative_original_size is None:
1306
+ negative_original_size = original_size
1307
+ if negative_target_size is None:
1308
+ negative_target_size = target_size
1309
+
1310
+ add_text_embeds = pooled_prompt_embeds
1311
+ if self.text_encoder_2 is None:
1312
+ text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1])
1313
+ else:
1314
+ text_encoder_projection_dim = self.text_encoder_2.config.projection_dim
1315
+
1316
+ add_time_ids, add_neg_time_ids = self._get_add_time_ids(
1317
+ original_size,
1318
+ crops_coords_top_left,
1319
+ target_size,
1320
+ aesthetic_score,
1321
+ negative_aesthetic_score,
1322
+ negative_original_size,
1323
+ negative_crops_coords_top_left,
1324
+ negative_target_size,
1325
+ dtype=prompt_embeds.dtype,
1326
+ text_encoder_projection_dim=text_encoder_projection_dim,
1327
+ )
1328
+ add_time_ids = add_time_ids.repeat(batch_size * num_images_per_prompt, 1)
1329
+
1330
+ if self.do_classifier_free_guidance:
1331
+ prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
1332
+ add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0)
1333
+ add_neg_time_ids = add_neg_time_ids.repeat(batch_size * num_images_per_prompt, 1)
1334
+ add_time_ids = torch.cat([add_neg_time_ids, add_time_ids], dim=0)
1335
+
1336
+ prompt_embeds = prompt_embeds.to(device)
1337
+ add_text_embeds = add_text_embeds.to(device)
1338
+ add_time_ids = add_time_ids.to(device)
1339
+
1340
+ if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
1341
+ image_embeds = self.prepare_ip_adapter_image_embeds(
1342
+ ip_adapter_image,
1343
+ ip_adapter_image_embeds,
1344
+ device,
1345
+ batch_size * num_images_per_prompt,
1346
+ self.do_classifier_free_guidance,
1347
+ )
1348
+
1349
+ # 9. Denoising loop
1350
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
1351
+
1352
+ # 9.1 Apply denoising_end
1353
+ if (
1354
+ self.denoising_end is not None
1355
+ and self.denoising_start is not None
1356
+ and denoising_value_valid(self.denoising_end)
1357
+ and denoising_value_valid(self.denoising_start)
1358
+ and self.denoising_start >= self.denoising_end
1359
+ ):
1360
+ raise ValueError(
1361
+ f"`denoising_start`: {self.denoising_start} cannot be larger than or equal to `denoising_end`: "
1362
+ + f" {self.denoising_end} when using type float."
1363
+ )
1364
+ elif self.denoising_end is not None and denoising_value_valid(self.denoising_end):
1365
+ discrete_timestep_cutoff = int(
1366
+ round(
1367
+ self.scheduler.config.num_train_timesteps
1368
+ - (self.denoising_end * self.scheduler.config.num_train_timesteps)
1369
+ )
1370
+ )
1371
+ num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps)))
1372
+ timesteps = timesteps[:num_inference_steps]
1373
+
1374
+ # 9.2 Optionally get Guidance Scale Embedding
1375
+ timestep_cond = None
1376
+ if self.unet.config.time_cond_proj_dim is not None:
1377
+ guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)
1378
+ timestep_cond = self.get_guidance_scale_embedding(
1379
+ guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim
1380
+ ).to(device=device, dtype=latents.dtype)
1381
+
1382
+ self._num_timesteps = len(timesteps)
1383
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
1384
+ for i, t in enumerate(timesteps):
1385
+ if self.interrupt:
1386
+ continue
1387
+
1388
+ # expand the latents if we are doing classifier free guidance
1389
+ latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
1390
+
1391
+ latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
1392
+
1393
+ # predict the noise residual
1394
+ added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
1395
+ if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
1396
+ added_cond_kwargs["image_embeds"] = image_embeds
1397
+ noise_pred = self.unet(
1398
+ latent_model_input,
1399
+ t,
1400
+ encoder_hidden_states=prompt_embeds,
1401
+ timestep_cond=timestep_cond,
1402
+ cross_attention_kwargs=self.cross_attention_kwargs,
1403
+ added_cond_kwargs=added_cond_kwargs,
1404
+ return_dict=False,
1405
+ )[0]
1406
+
1407
+ # perform guidance
1408
+ if self.do_classifier_free_guidance:
1409
+ noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
1410
+ noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)
1411
+
1412
+ if self.do_classifier_free_guidance and self.guidance_rescale > 0.0:
1413
+ # Based on 3.4. in https://huggingface.co/papers/2305.08891
1414
+ noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale)
1415
+
1416
+ # compute the previous noisy sample x_t -> x_t-1
1417
+ latents_dtype = latents.dtype
1418
+ latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
1419
+ if latents.dtype != latents_dtype:
1420
+ if torch.backends.mps.is_available():
1421
+ # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
1422
+ latents = latents.to(latents_dtype)
1423
+
1424
+ if callback_on_step_end is not None:
1425
+ callback_kwargs = {}
1426
+ for k in callback_on_step_end_tensor_inputs:
1427
+ callback_kwargs[k] = locals()[k]
1428
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
1429
+
1430
+ latents = callback_outputs.pop("latents", latents)
1431
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
1432
+ add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds)
1433
+ add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids)
1434
+
1435
+ # call the callback, if provided
1436
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
1437
+ progress_bar.update()
1438
+ if callback is not None and i % callback_steps == 0:
1439
+ step_idx = i // getattr(self.scheduler, "order", 1)
1440
+ callback(step_idx, t, latents)
1441
+
1442
+ if XLA_AVAILABLE:
1443
+ xm.mark_step()
1444
+
1445
+ if not output_type == "latent":
1446
+ # make sure the VAE is in float32 mode, as it overflows in float16
1447
+ needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast
1448
+
1449
+ if needs_upcasting:
1450
+ self.upcast_vae()
1451
+ latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)
1452
+ elif latents.dtype != self.vae.dtype:
1453
+ if torch.backends.mps.is_available():
1454
+ # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
1455
+ self.vae = self.vae.to(latents.dtype)
1456
+
1457
+ # unscale/denormalize the latents
1458
+ # denormalize with the mean and std if available and not None
1459
+ has_latents_mean = hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None
1460
+ has_latents_std = hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None
1461
+ if has_latents_mean and has_latents_std:
1462
+ latents_mean = (
1463
+ torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype)
1464
+ )
1465
+ latents_std = (
1466
+ torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype)
1467
+ )
1468
+ latents = latents * latents_std / self.vae.config.scaling_factor + latents_mean
1469
+ else:
1470
+ latents = latents / self.vae.config.scaling_factor
1471
+
1472
+ image = self.vae.decode(latents, return_dict=False)[0]
1473
+
1474
+ # cast back to fp16 if needed
1475
+ if needs_upcasting:
1476
+ self.vae.to(dtype=torch.float16)
1477
+ else:
1478
+ image = latents
1479
+
1480
+ # apply watermark if available
1481
+ if self.watermark is not None:
1482
+ image = self.watermark.apply_watermark(image)
1483
+
1484
+ image = self.image_processor.postprocess(image, output_type=output_type)
1485
+
1486
+ # Offload all models
1487
+ self.maybe_free_model_hooks()
1488
+
1489
+ if not return_dict:
1490
+ return (image,)
1491
+
1492
+ return StableDiffusionXLPipelineOutput(images=image)
western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py ADDED
@@ -0,0 +1,1734 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2026 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from typing import Any, Callable
17
+
18
+ import numpy as np
19
+ import PIL.Image
20
+ import torch
21
+ from transformers import (
22
+ CLIPImageProcessor,
23
+ CLIPTextModel,
24
+ CLIPTextModelWithProjection,
25
+ CLIPTokenizer,
26
+ CLIPVisionModelWithProjection,
27
+ )
28
+
29
+ from ...callbacks import MultiPipelineCallbacks, PipelineCallback
30
+ from ...image_processor import PipelineImageInput, VaeImageProcessor
31
+ from ...loaders import (
32
+ FromSingleFileMixin,
33
+ IPAdapterMixin,
34
+ StableDiffusionXLLoraLoaderMixin,
35
+ TextualInversionLoaderMixin,
36
+ )
37
+ from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel
38
+ from ...models.lora import adjust_lora_scale_text_encoder
39
+ from ...schedulers import KarrasDiffusionSchedulers
40
+ from ...utils import (
41
+ USE_PEFT_BACKEND,
42
+ deprecate,
43
+ is_invisible_watermark_available,
44
+ is_torch_xla_available,
45
+ logging,
46
+ replace_example_docstring,
47
+ scale_lora_layers,
48
+ unscale_lora_layers,
49
+ )
50
+ from ...utils.torch_utils import randn_tensor
51
+ from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin
52
+ from .pipeline_output import StableDiffusionXLPipelineOutput
53
+
54
+
55
+ if is_invisible_watermark_available():
56
+ from .watermark import StableDiffusionXLWatermarker
57
+
58
+ if is_torch_xla_available():
59
+ import torch_xla.core.xla_model as xm
60
+
61
+ XLA_AVAILABLE = True
62
+ else:
63
+ XLA_AVAILABLE = False
64
+
65
+
66
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
67
+
68
+
69
+ EXAMPLE_DOC_STRING = """
70
+ Examples:
71
+ ```py
72
+ >>> import torch
73
+ >>> from diffusers import StableDiffusionXLInpaintPipeline
74
+ >>> from diffusers.utils import load_image
75
+
76
+ >>> pipe = StableDiffusionXLInpaintPipeline.from_pretrained(
77
+ ... "stabilityai/stable-diffusion-xl-base-1.0",
78
+ ... torch_dtype=torch.float16,
79
+ ... variant="fp16",
80
+ ... use_safetensors=True,
81
+ ... )
82
+ >>> pipe.to("cuda")
83
+
84
+ >>> img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png"
85
+ >>> mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png"
86
+
87
+ >>> init_image = load_image(img_url).convert("RGB")
88
+ >>> mask_image = load_image(mask_url).convert("RGB")
89
+
90
+ >>> prompt = "A majestic tiger sitting on a bench"
91
+ >>> image = pipe(
92
+ ... prompt=prompt, image=init_image, mask_image=mask_image, num_inference_steps=50, strength=0.80
93
+ ... ).images[0]
94
+ ```
95
+ """
96
+
97
+
98
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg
99
+ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):
100
+ r"""
101
+ Rescales `noise_cfg` tensor based on `guidance_rescale` to improve image quality and fix overexposure. Based on
102
+ Section 3.4 from [Common Diffusion Noise Schedules and Sample Steps are
103
+ Flawed](https://huggingface.co/papers/2305.08891).
104
+
105
+ Args:
106
+ noise_cfg (`torch.Tensor`):
107
+ The predicted noise tensor for the guided diffusion process.
108
+ noise_pred_text (`torch.Tensor`):
109
+ The predicted noise tensor for the text-guided diffusion process.
110
+ guidance_rescale (`float`, *optional*, defaults to 0.0):
111
+ A rescale factor applied to the noise predictions.
112
+
113
+ Returns:
114
+ noise_cfg (`torch.Tensor`): The rescaled noise prediction tensor.
115
+ """
116
+ std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)
117
+ std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)
118
+ # rescale the results from guidance (fixes overexposure)
119
+ noise_pred_rescaled = noise_cfg * (std_text / std_cfg)
120
+ # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images
121
+ noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg
122
+ return noise_cfg
123
+
124
+
125
+ def mask_pil_to_torch(mask, height, width):
126
+ # preprocess mask
127
+ if isinstance(mask, (PIL.Image.Image, np.ndarray)):
128
+ mask = [mask]
129
+
130
+ if isinstance(mask, list) and isinstance(mask[0], PIL.Image.Image):
131
+ mask = [i.resize((width, height), resample=PIL.Image.LANCZOS) for i in mask]
132
+ mask = np.concatenate([np.array(m.convert("L"))[None, None, :] for m in mask], axis=0)
133
+ mask = mask.astype(np.float32) / 255.0
134
+ elif isinstance(mask, list) and isinstance(mask[0], np.ndarray):
135
+ mask = np.concatenate([m[None, None, :] for m in mask], axis=0)
136
+
137
+ mask = torch.from_numpy(mask)
138
+ return mask
139
+
140
+
141
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
142
+ def retrieve_latents(
143
+ encoder_output: torch.Tensor, generator: torch.Generator | None = None, sample_mode: str = "sample"
144
+ ):
145
+ if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
146
+ return encoder_output.latent_dist.sample(generator)
147
+ elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
148
+ return encoder_output.latent_dist.mode()
149
+ elif hasattr(encoder_output, "latents"):
150
+ return encoder_output.latents
151
+ else:
152
+ raise AttributeError("Could not access latents of provided encoder_output")
153
+
154
+
155
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
156
+ def retrieve_timesteps(
157
+ scheduler,
158
+ num_inference_steps: int | None = None,
159
+ device: str | torch.device | None = None,
160
+ timesteps: list[int] | None = None,
161
+ sigmas: list[float] | None = None,
162
+ **kwargs,
163
+ ):
164
+ r"""
165
+ Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
166
+ custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
167
+
168
+ Args:
169
+ scheduler (`SchedulerMixin`):
170
+ The scheduler to get timesteps from.
171
+ num_inference_steps (`int`):
172
+ The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
173
+ must be `None`.
174
+ device (`str` or `torch.device`, *optional*):
175
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
176
+ timesteps (`list[int]`, *optional*):
177
+ Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
178
+ `num_inference_steps` and `sigmas` must be `None`.
179
+ sigmas (`list[float]`, *optional*):
180
+ Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
181
+ `num_inference_steps` and `timesteps` must be `None`.
182
+
183
+ Returns:
184
+ `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
185
+ second element is the number of inference steps.
186
+ """
187
+ if timesteps is not None and sigmas is not None:
188
+ raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
189
+ if timesteps is not None:
190
+ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
191
+ if not accepts_timesteps:
192
+ raise ValueError(
193
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
194
+ f" timestep schedules. Please check whether you are using the correct scheduler."
195
+ )
196
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
197
+ timesteps = scheduler.timesteps
198
+ num_inference_steps = len(timesteps)
199
+ elif sigmas is not None:
200
+ accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
201
+ if not accept_sigmas:
202
+ raise ValueError(
203
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
204
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
205
+ )
206
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
207
+ timesteps = scheduler.timesteps
208
+ num_inference_steps = len(timesteps)
209
+ else:
210
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
211
+ timesteps = scheduler.timesteps
212
+ return timesteps, num_inference_steps
213
+
214
+
215
+ class StableDiffusionXLInpaintPipeline(
216
+ DiffusionPipeline,
217
+ StableDiffusionMixin,
218
+ TextualInversionLoaderMixin,
219
+ StableDiffusionXLLoraLoaderMixin,
220
+ FromSingleFileMixin,
221
+ IPAdapterMixin,
222
+ ):
223
+ r"""
224
+ Pipeline for text-to-image generation using Stable Diffusion XL.
225
+
226
+ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
227
+ library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
228
+
229
+ The pipeline also inherits the following loading methods:
230
+ - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings
231
+ - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files
232
+ - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights
233
+ - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights
234
+ - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters
235
+
236
+ Args:
237
+ vae ([`AutoencoderKL`]):
238
+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
239
+ text_encoder ([`CLIPTextModel`]):
240
+ Frozen text-encoder. Stable Diffusion XL uses the text portion of
241
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
242
+ the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
243
+ text_encoder_2 ([` CLIPTextModelWithProjection`]):
244
+ Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of
245
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection),
246
+ specifically the
247
+ [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k)
248
+ variant.
249
+ tokenizer (`CLIPTokenizer`):
250
+ Tokenizer of class
251
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
252
+ tokenizer_2 (`CLIPTokenizer`):
253
+ Second Tokenizer of class
254
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
255
+ unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.
256
+ scheduler ([`SchedulerMixin`]):
257
+ A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
258
+ [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
259
+ requires_aesthetics_score (`bool`, *optional*, defaults to `"False"`):
260
+ Whether the `unet` requires a aesthetic_score condition to be passed during inference. Also see the config
261
+ of `stabilityai/stable-diffusion-xl-refiner-1-0`.
262
+ force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`):
263
+ Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of
264
+ `stabilityai/stable-diffusion-xl-base-1-0`.
265
+ add_watermarker (`bool`, *optional*):
266
+ Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to
267
+ watermark output images. If not defined, it will default to True if the package is installed, otherwise no
268
+ watermarker will be used.
269
+ """
270
+
271
+ model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->unet->vae"
272
+
273
+ _optional_components = [
274
+ "tokenizer",
275
+ "tokenizer_2",
276
+ "text_encoder",
277
+ "text_encoder_2",
278
+ "image_encoder",
279
+ "feature_extractor",
280
+ ]
281
+ _callback_tensor_inputs = [
282
+ "latents",
283
+ "prompt_embeds",
284
+ "add_text_embeds",
285
+ "add_time_ids",
286
+ "mask",
287
+ "masked_image_latents",
288
+ ]
289
+
290
+ def __init__(
291
+ self,
292
+ vae: AutoencoderKL,
293
+ text_encoder: CLIPTextModel,
294
+ text_encoder_2: CLIPTextModelWithProjection,
295
+ tokenizer: CLIPTokenizer,
296
+ tokenizer_2: CLIPTokenizer,
297
+ unet: UNet2DConditionModel,
298
+ scheduler: KarrasDiffusionSchedulers,
299
+ image_encoder: CLIPVisionModelWithProjection = None,
300
+ feature_extractor: CLIPImageProcessor = None,
301
+ requires_aesthetics_score: bool = False,
302
+ force_zeros_for_empty_prompt: bool = True,
303
+ add_watermarker: bool | None = None,
304
+ ):
305
+ super().__init__()
306
+
307
+ self.register_modules(
308
+ vae=vae,
309
+ text_encoder=text_encoder,
310
+ text_encoder_2=text_encoder_2,
311
+ tokenizer=tokenizer,
312
+ tokenizer_2=tokenizer_2,
313
+ unet=unet,
314
+ image_encoder=image_encoder,
315
+ feature_extractor=feature_extractor,
316
+ scheduler=scheduler,
317
+ )
318
+ self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt)
319
+ self.register_to_config(requires_aesthetics_score=requires_aesthetics_score)
320
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
321
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
322
+ self.mask_processor = VaeImageProcessor(
323
+ vae_scale_factor=self.vae_scale_factor, do_normalize=False, do_binarize=True, do_convert_grayscale=True
324
+ )
325
+
326
+ add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available()
327
+
328
+ if add_watermarker:
329
+ self.watermark = StableDiffusionXLWatermarker()
330
+ else:
331
+ self.watermark = None
332
+
333
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image
334
+ def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None):
335
+ dtype = next(self.image_encoder.parameters()).dtype
336
+
337
+ if not isinstance(image, torch.Tensor):
338
+ image = self.feature_extractor(image, return_tensors="pt").pixel_values
339
+
340
+ image = image.to(device=device, dtype=dtype)
341
+ if output_hidden_states:
342
+ image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2]
343
+ image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0)
344
+ uncond_image_enc_hidden_states = self.image_encoder(
345
+ torch.zeros_like(image), output_hidden_states=True
346
+ ).hidden_states[-2]
347
+ uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave(
348
+ num_images_per_prompt, dim=0
349
+ )
350
+ return image_enc_hidden_states, uncond_image_enc_hidden_states
351
+ else:
352
+ image_embeds = self.image_encoder(image).image_embeds
353
+ image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0)
354
+ uncond_image_embeds = torch.zeros_like(image_embeds)
355
+
356
+ return image_embeds, uncond_image_embeds
357
+
358
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds
359
+ def prepare_ip_adapter_image_embeds(
360
+ self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance
361
+ ):
362
+ image_embeds = []
363
+ if do_classifier_free_guidance:
364
+ negative_image_embeds = []
365
+ if ip_adapter_image_embeds is None:
366
+ if not isinstance(ip_adapter_image, list):
367
+ ip_adapter_image = [ip_adapter_image]
368
+
369
+ if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers):
370
+ raise ValueError(
371
+ f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters."
372
+ )
373
+
374
+ for single_ip_adapter_image, image_proj_layer in zip(
375
+ ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers
376
+ ):
377
+ output_hidden_state = not isinstance(image_proj_layer, ImageProjection)
378
+ single_image_embeds, single_negative_image_embeds = self.encode_image(
379
+ single_ip_adapter_image, device, 1, output_hidden_state
380
+ )
381
+
382
+ image_embeds.append(single_image_embeds[None, :])
383
+ if do_classifier_free_guidance:
384
+ negative_image_embeds.append(single_negative_image_embeds[None, :])
385
+ else:
386
+ for single_image_embeds in ip_adapter_image_embeds:
387
+ if do_classifier_free_guidance:
388
+ single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2)
389
+ negative_image_embeds.append(single_negative_image_embeds)
390
+ image_embeds.append(single_image_embeds)
391
+
392
+ ip_adapter_image_embeds = []
393
+ for i, single_image_embeds in enumerate(image_embeds):
394
+ single_image_embeds = torch.cat([single_image_embeds] * num_images_per_prompt, dim=0)
395
+ if do_classifier_free_guidance:
396
+ single_negative_image_embeds = torch.cat([negative_image_embeds[i]] * num_images_per_prompt, dim=0)
397
+ single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds], dim=0)
398
+
399
+ single_image_embeds = single_image_embeds.to(device=device)
400
+ ip_adapter_image_embeds.append(single_image_embeds)
401
+
402
+ return ip_adapter_image_embeds
403
+
404
+ # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.encode_prompt
405
+ def encode_prompt(
406
+ self,
407
+ prompt: str,
408
+ prompt_2: str | None = None,
409
+ device: torch.device | None = None,
410
+ num_images_per_prompt: int = 1,
411
+ do_classifier_free_guidance: bool = True,
412
+ negative_prompt: str | None = None,
413
+ negative_prompt_2: str | None = None,
414
+ prompt_embeds: torch.Tensor | None = None,
415
+ negative_prompt_embeds: torch.Tensor | None = None,
416
+ pooled_prompt_embeds: torch.Tensor | None = None,
417
+ negative_pooled_prompt_embeds: torch.Tensor | None = None,
418
+ lora_scale: float | None = None,
419
+ clip_skip: int | None = None,
420
+ ):
421
+ r"""
422
+ Encodes the prompt into text encoder hidden states.
423
+
424
+ Args:
425
+ prompt (`str` or `list[str]`, *optional*):
426
+ prompt to be encoded
427
+ prompt_2 (`str` or `list[str]`, *optional*):
428
+ The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
429
+ used in both text-encoders
430
+ device: (`torch.device`):
431
+ torch device
432
+ num_images_per_prompt (`int`):
433
+ number of images that should be generated per prompt
434
+ do_classifier_free_guidance (`bool`):
435
+ whether to use classifier free guidance or not
436
+ negative_prompt (`str` or `list[str]`, *optional*):
437
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
438
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
439
+ less than `1`).
440
+ negative_prompt_2 (`str` or `list[str]`, *optional*):
441
+ The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
442
+ `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
443
+ prompt_embeds (`torch.Tensor`, *optional*):
444
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
445
+ provided, text embeddings will be generated from `prompt` input argument.
446
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
447
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
448
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
449
+ argument.
450
+ pooled_prompt_embeds (`torch.Tensor`, *optional*):
451
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
452
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
453
+ negative_pooled_prompt_embeds (`torch.Tensor`, *optional*):
454
+ Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
455
+ weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
456
+ input argument.
457
+ lora_scale (`float`, *optional*):
458
+ A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
459
+ clip_skip (`int`, *optional*):
460
+ Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
461
+ the output of the pre-final layer will be used for computing the prompt embeddings.
462
+ """
463
+ device = device or self._execution_device
464
+
465
+ # set lora scale so that monkey patched LoRA
466
+ # function of text encoder can correctly access it
467
+ if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin):
468
+ self._lora_scale = lora_scale
469
+
470
+ # dynamically adjust the LoRA scale
471
+ if self.text_encoder is not None:
472
+ if not USE_PEFT_BACKEND:
473
+ adjust_lora_scale_text_encoder(self.text_encoder, lora_scale)
474
+ else:
475
+ scale_lora_layers(self.text_encoder, lora_scale)
476
+
477
+ if self.text_encoder_2 is not None:
478
+ if not USE_PEFT_BACKEND:
479
+ adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale)
480
+ else:
481
+ scale_lora_layers(self.text_encoder_2, lora_scale)
482
+
483
+ prompt = [prompt] if isinstance(prompt, str) else prompt
484
+
485
+ if prompt is not None:
486
+ batch_size = len(prompt)
487
+ else:
488
+ batch_size = prompt_embeds.shape[0]
489
+
490
+ # Define tokenizers and text encoders
491
+ tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2]
492
+ text_encoders = (
493
+ [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2]
494
+ )
495
+
496
+ if prompt_embeds is None:
497
+ prompt_2 = prompt_2 or prompt
498
+ prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2
499
+
500
+ # textual inversion: process multi-vector tokens if necessary
501
+ prompt_embeds_list = []
502
+ prompts = [prompt, prompt_2]
503
+ for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders):
504
+ if isinstance(self, TextualInversionLoaderMixin):
505
+ prompt = self.maybe_convert_prompt(prompt, tokenizer)
506
+
507
+ text_inputs = tokenizer(
508
+ prompt,
509
+ padding="max_length",
510
+ max_length=tokenizer.model_max_length,
511
+ truncation=True,
512
+ return_tensors="pt",
513
+ )
514
+
515
+ text_input_ids = text_inputs.input_ids
516
+ untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
517
+
518
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(
519
+ text_input_ids, untruncated_ids
520
+ ):
521
+ removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1])
522
+ logger.warning(
523
+ "The following part of your input was truncated because CLIP can only handle sequences up to"
524
+ f" {tokenizer.model_max_length} tokens: {removed_text}"
525
+ )
526
+
527
+ prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True)
528
+
529
+ # We are only ALWAYS interested in the pooled output of the final text encoder
530
+ if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2:
531
+ pooled_prompt_embeds = prompt_embeds[0]
532
+
533
+ if clip_skip is None:
534
+ prompt_embeds = prompt_embeds.hidden_states[-2]
535
+ else:
536
+ # "2" because SDXL always indexes from the penultimate layer.
537
+ prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)]
538
+
539
+ prompt_embeds_list.append(prompt_embeds)
540
+
541
+ prompt_embeds = torch.concat(prompt_embeds_list, dim=-1)
542
+
543
+ # get unconditional embeddings for classifier free guidance
544
+ zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt
545
+ if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt:
546
+ negative_prompt_embeds = torch.zeros_like(prompt_embeds)
547
+ negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds)
548
+ elif do_classifier_free_guidance and negative_prompt_embeds is None:
549
+ negative_prompt = negative_prompt or ""
550
+ negative_prompt_2 = negative_prompt_2 or negative_prompt
551
+
552
+ # normalize str to list
553
+ negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
554
+ negative_prompt_2 = (
555
+ batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2
556
+ )
557
+
558
+ uncond_tokens: list[str]
559
+ if prompt is not None and type(prompt) is not type(negative_prompt):
560
+ raise TypeError(
561
+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
562
+ f" {type(prompt)}."
563
+ )
564
+ elif batch_size != len(negative_prompt):
565
+ raise ValueError(
566
+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
567
+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
568
+ " the batch size of `prompt`."
569
+ )
570
+ else:
571
+ uncond_tokens = [negative_prompt, negative_prompt_2]
572
+
573
+ negative_prompt_embeds_list = []
574
+ for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders):
575
+ if isinstance(self, TextualInversionLoaderMixin):
576
+ negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer)
577
+
578
+ max_length = prompt_embeds.shape[1]
579
+ uncond_input = tokenizer(
580
+ negative_prompt,
581
+ padding="max_length",
582
+ max_length=max_length,
583
+ truncation=True,
584
+ return_tensors="pt",
585
+ )
586
+
587
+ negative_prompt_embeds = text_encoder(
588
+ uncond_input.input_ids.to(device),
589
+ output_hidden_states=True,
590
+ )
591
+
592
+ # We are only ALWAYS interested in the pooled output of the final text encoder
593
+ if negative_pooled_prompt_embeds is None and negative_prompt_embeds[0].ndim == 2:
594
+ negative_pooled_prompt_embeds = negative_prompt_embeds[0]
595
+ negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2]
596
+
597
+ negative_prompt_embeds_list.append(negative_prompt_embeds)
598
+
599
+ negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1)
600
+
601
+ if self.text_encoder_2 is not None:
602
+ prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)
603
+ else:
604
+ prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device)
605
+
606
+ bs_embed, seq_len, _ = prompt_embeds.shape
607
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
608
+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
609
+ prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
610
+
611
+ if do_classifier_free_guidance:
612
+ # duplicate unconditional embeddings for each generation per prompt, using mps friendly method
613
+ seq_len = negative_prompt_embeds.shape[1]
614
+
615
+ if self.text_encoder_2 is not None:
616
+ negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)
617
+ else:
618
+ negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device)
619
+
620
+ negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
621
+ negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
622
+
623
+ pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
624
+ bs_embed * num_images_per_prompt, -1
625
+ )
626
+ if do_classifier_free_guidance:
627
+ negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
628
+ bs_embed * num_images_per_prompt, -1
629
+ )
630
+
631
+ if self.text_encoder is not None:
632
+ if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND:
633
+ # Retrieve the original scale by scaling back the LoRA layers
634
+ unscale_lora_layers(self.text_encoder, lora_scale)
635
+
636
+ if self.text_encoder_2 is not None:
637
+ if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND:
638
+ # Retrieve the original scale by scaling back the LoRA layers
639
+ unscale_lora_layers(self.text_encoder_2, lora_scale)
640
+
641
+ return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds
642
+
643
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs
644
+ def prepare_extra_step_kwargs(self, generator, eta):
645
+ # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
646
+ # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
647
+ # eta corresponds to η in DDIM paper: https://huggingface.co/papers/2010.02502
648
+ # and should be between [0, 1]
649
+
650
+ accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
651
+ extra_step_kwargs = {}
652
+ if accepts_eta:
653
+ extra_step_kwargs["eta"] = eta
654
+
655
+ # check if the scheduler accepts generator
656
+ accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
657
+ if accepts_generator:
658
+ extra_step_kwargs["generator"] = generator
659
+ return extra_step_kwargs
660
+
661
+ def check_inputs(
662
+ self,
663
+ prompt,
664
+ prompt_2,
665
+ image,
666
+ mask_image,
667
+ height,
668
+ width,
669
+ strength,
670
+ callback_steps,
671
+ output_type,
672
+ negative_prompt=None,
673
+ negative_prompt_2=None,
674
+ prompt_embeds=None,
675
+ negative_prompt_embeds=None,
676
+ ip_adapter_image=None,
677
+ ip_adapter_image_embeds=None,
678
+ callback_on_step_end_tensor_inputs=None,
679
+ padding_mask_crop=None,
680
+ ):
681
+ if strength < 0 or strength > 1:
682
+ raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}")
683
+
684
+ if height % 8 != 0 or width % 8 != 0:
685
+ raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
686
+
687
+ if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0):
688
+ raise ValueError(
689
+ f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
690
+ f" {type(callback_steps)}."
691
+ )
692
+
693
+ if callback_on_step_end_tensor_inputs is not None and not all(
694
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
695
+ ):
696
+ raise ValueError(
697
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
698
+ )
699
+
700
+ if prompt is not None and prompt_embeds is not None:
701
+ raise ValueError(
702
+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
703
+ " only forward one of the two."
704
+ )
705
+ elif prompt_2 is not None and prompt_embeds is not None:
706
+ raise ValueError(
707
+ f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
708
+ " only forward one of the two."
709
+ )
710
+ elif prompt is None and prompt_embeds is None:
711
+ raise ValueError(
712
+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
713
+ )
714
+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
715
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
716
+ elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)):
717
+ raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}")
718
+
719
+ if negative_prompt is not None and negative_prompt_embeds is not None:
720
+ raise ValueError(
721
+ f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
722
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
723
+ )
724
+ elif negative_prompt_2 is not None and negative_prompt_embeds is not None:
725
+ raise ValueError(
726
+ f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:"
727
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
728
+ )
729
+
730
+ if prompt_embeds is not None and negative_prompt_embeds is not None:
731
+ if prompt_embeds.shape != negative_prompt_embeds.shape:
732
+ raise ValueError(
733
+ "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"
734
+ f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"
735
+ f" {negative_prompt_embeds.shape}."
736
+ )
737
+ if padding_mask_crop is not None:
738
+ if not isinstance(image, PIL.Image.Image):
739
+ raise ValueError(
740
+ f"The image should be a PIL image when inpainting mask crop, but is of type {type(image)}."
741
+ )
742
+ if not isinstance(mask_image, PIL.Image.Image):
743
+ raise ValueError(
744
+ f"The mask image should be a PIL image when inpainting mask crop, but is of type"
745
+ f" {type(mask_image)}."
746
+ )
747
+ if output_type != "pil":
748
+ raise ValueError(f"The output type should be PIL when inpainting mask crop, but is {output_type}.")
749
+
750
+ if ip_adapter_image is not None and ip_adapter_image_embeds is not None:
751
+ raise ValueError(
752
+ "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined."
753
+ )
754
+
755
+ if ip_adapter_image_embeds is not None:
756
+ if not isinstance(ip_adapter_image_embeds, list):
757
+ raise ValueError(
758
+ f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}"
759
+ )
760
+ elif ip_adapter_image_embeds[0].ndim not in [3, 4]:
761
+ raise ValueError(
762
+ f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D"
763
+ )
764
+
765
+ def prepare_latents(
766
+ self,
767
+ batch_size,
768
+ num_channels_latents,
769
+ height,
770
+ width,
771
+ dtype,
772
+ device,
773
+ generator,
774
+ latents=None,
775
+ image=None,
776
+ timestep=None,
777
+ is_strength_max=True,
778
+ add_noise=True,
779
+ return_noise=False,
780
+ return_image_latents=False,
781
+ ):
782
+ shape = (
783
+ batch_size,
784
+ num_channels_latents,
785
+ int(height) // self.vae_scale_factor,
786
+ int(width) // self.vae_scale_factor,
787
+ )
788
+ if isinstance(generator, list) and len(generator) != batch_size:
789
+ raise ValueError(
790
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
791
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
792
+ )
793
+
794
+ if (image is None or timestep is None) and not is_strength_max:
795
+ raise ValueError(
796
+ "Since strength < 1. initial latents are to be initialised as a combination of Image + Noise."
797
+ "However, either the image or the noise timestep has not been provided."
798
+ )
799
+
800
+ if image.shape[1] == 4:
801
+ image_latents = image.to(device=device, dtype=dtype)
802
+ image_latents = image_latents.repeat(batch_size // image_latents.shape[0], 1, 1, 1)
803
+ elif return_image_latents or (latents is None and not is_strength_max):
804
+ image = image.to(device=device, dtype=dtype)
805
+ image_latents = self._encode_vae_image(image=image, generator=generator)
806
+ image_latents = image_latents.repeat(batch_size // image_latents.shape[0], 1, 1, 1)
807
+
808
+ if latents is None and add_noise:
809
+ noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
810
+ # if strength is 1. then initialise the latents to noise, else initial to image + noise
811
+ latents = noise if is_strength_max else self.scheduler.add_noise(image_latents, noise, timestep)
812
+ # if pure noise then scale the initial latents by the Scheduler's init sigma
813
+ latents = latents * self.scheduler.init_noise_sigma if is_strength_max else latents
814
+ elif add_noise:
815
+ noise = latents.to(device)
816
+ latents = noise * self.scheduler.init_noise_sigma
817
+ else:
818
+ noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
819
+ latents = image_latents.to(device)
820
+
821
+ outputs = (latents,)
822
+
823
+ if return_noise:
824
+ outputs += (noise,)
825
+
826
+ if return_image_latents:
827
+ outputs += (image_latents,)
828
+
829
+ return outputs
830
+
831
+ def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator):
832
+ dtype = image.dtype
833
+ if self.vae.config.force_upcast:
834
+ image = image.float()
835
+ self.vae.to(dtype=torch.float32)
836
+
837
+ if isinstance(generator, list):
838
+ image_latents = [
839
+ retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i])
840
+ for i in range(image.shape[0])
841
+ ]
842
+ image_latents = torch.cat(image_latents, dim=0)
843
+ else:
844
+ image_latents = retrieve_latents(self.vae.encode(image), generator=generator)
845
+
846
+ if self.vae.config.force_upcast:
847
+ self.vae.to(dtype)
848
+
849
+ image_latents = image_latents.to(dtype)
850
+ image_latents = self.vae.config.scaling_factor * image_latents
851
+
852
+ return image_latents
853
+
854
+ def prepare_mask_latents(
855
+ self, mask, masked_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance
856
+ ):
857
+ # resize the mask to latents shape as we concatenate the mask to the latents
858
+ # we do that before converting to dtype to avoid breaking in case we're using cpu_offload
859
+ # and half precision
860
+ mask = torch.nn.functional.interpolate(
861
+ mask, size=(height // self.vae_scale_factor, width // self.vae_scale_factor)
862
+ )
863
+ mask = mask.to(device=device, dtype=dtype)
864
+
865
+ # duplicate mask and masked_image_latents for each generation per prompt, using mps friendly method
866
+ if mask.shape[0] < batch_size:
867
+ if not batch_size % mask.shape[0] == 0:
868
+ raise ValueError(
869
+ "The passed mask and the required batch size don't match. Masks are supposed to be duplicated to"
870
+ f" a total batch size of {batch_size}, but {mask.shape[0]} masks were passed. Make sure the number"
871
+ " of masks that you pass is divisible by the total requested batch size."
872
+ )
873
+ mask = mask.repeat(batch_size // mask.shape[0], 1, 1, 1)
874
+
875
+ mask = torch.cat([mask] * 2) if do_classifier_free_guidance else mask
876
+
877
+ if masked_image is not None and masked_image.shape[1] == 4:
878
+ masked_image_latents = masked_image
879
+ else:
880
+ masked_image_latents = None
881
+
882
+ if masked_image is not None:
883
+ if masked_image_latents is None:
884
+ masked_image = masked_image.to(device=device, dtype=dtype)
885
+ masked_image_latents = self._encode_vae_image(masked_image, generator=generator)
886
+
887
+ if masked_image_latents.shape[0] < batch_size:
888
+ if not batch_size % masked_image_latents.shape[0] == 0:
889
+ raise ValueError(
890
+ "The passed images and the required batch size don't match. Images are supposed to be duplicated"
891
+ f" to a total batch size of {batch_size}, but {masked_image_latents.shape[0]} images were passed."
892
+ " Make sure the number of images that you pass is divisible by the total requested batch size."
893
+ )
894
+ masked_image_latents = masked_image_latents.repeat(
895
+ batch_size // masked_image_latents.shape[0], 1, 1, 1
896
+ )
897
+
898
+ masked_image_latents = (
899
+ torch.cat([masked_image_latents] * 2) if do_classifier_free_guidance else masked_image_latents
900
+ )
901
+
902
+ # aligning device to prevent device errors when concating it with the latent model input
903
+ masked_image_latents = masked_image_latents.to(device=device, dtype=dtype)
904
+
905
+ return mask, masked_image_latents
906
+
907
+ # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl_img2img.StableDiffusionXLImg2ImgPipeline.get_timesteps
908
+ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=None):
909
+ # get the original timestep using init_timestep
910
+ if denoising_start is None:
911
+ init_timestep = min(int(num_inference_steps * strength), num_inference_steps)
912
+ t_start = max(num_inference_steps - init_timestep, 0)
913
+
914
+ timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :]
915
+ if hasattr(self.scheduler, "set_begin_index"):
916
+ self.scheduler.set_begin_index(t_start * self.scheduler.order)
917
+
918
+ return timesteps, num_inference_steps - t_start
919
+
920
+ else:
921
+ # Strength is irrelevant if we directly request a timestep to start at;
922
+ # that is, strength is determined by the denoising_start instead.
923
+ discrete_timestep_cutoff = int(
924
+ round(
925
+ self.scheduler.config.num_train_timesteps
926
+ - (denoising_start * self.scheduler.config.num_train_timesteps)
927
+ )
928
+ )
929
+
930
+ num_inference_steps = (self.scheduler.timesteps < discrete_timestep_cutoff).sum().item()
931
+ if self.scheduler.order == 2 and num_inference_steps % 2 == 0:
932
+ # if the scheduler is a 2nd order scheduler we might have to do +1
933
+ # because `num_inference_steps` might be even given that every timestep
934
+ # (except the highest one) is duplicated. If `num_inference_steps` is even it would
935
+ # mean that we cut the timesteps in the middle of the denoising step
936
+ # (between 1st and 2nd derivative) which leads to incorrect results. By adding 1
937
+ # we ensure that the denoising process always ends after the 2nd derivate step of the scheduler
938
+ num_inference_steps = num_inference_steps + 1
939
+
940
+ # because t_n+1 >= t_n, we slice the timesteps starting from the end
941
+ t_start = len(self.scheduler.timesteps) - num_inference_steps
942
+ timesteps = self.scheduler.timesteps[t_start:]
943
+ if hasattr(self.scheduler, "set_begin_index"):
944
+ self.scheduler.set_begin_index(t_start)
945
+ return timesteps, num_inference_steps
946
+
947
+ # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl_img2img.StableDiffusionXLImg2ImgPipeline._get_add_time_ids
948
+ def _get_add_time_ids(
949
+ self,
950
+ original_size,
951
+ crops_coords_top_left,
952
+ target_size,
953
+ aesthetic_score,
954
+ negative_aesthetic_score,
955
+ negative_original_size,
956
+ negative_crops_coords_top_left,
957
+ negative_target_size,
958
+ dtype,
959
+ text_encoder_projection_dim=None,
960
+ ):
961
+ if self.config.requires_aesthetics_score:
962
+ add_time_ids = list(original_size + crops_coords_top_left + (aesthetic_score,))
963
+ add_neg_time_ids = list(
964
+ negative_original_size + negative_crops_coords_top_left + (negative_aesthetic_score,)
965
+ )
966
+ else:
967
+ add_time_ids = list(original_size + crops_coords_top_left + target_size)
968
+ add_neg_time_ids = list(negative_original_size + crops_coords_top_left + negative_target_size)
969
+
970
+ passed_add_embed_dim = (
971
+ self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim
972
+ )
973
+ expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features
974
+
975
+ if (
976
+ expected_add_embed_dim > passed_add_embed_dim
977
+ and (expected_add_embed_dim - passed_add_embed_dim) == self.unet.config.addition_time_embed_dim
978
+ ):
979
+ raise ValueError(
980
+ f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. Please make sure to enable `requires_aesthetics_score` with `pipe.register_to_config(requires_aesthetics_score=True)` to make sure `aesthetic_score` {aesthetic_score} and `negative_aesthetic_score` {negative_aesthetic_score} is correctly used by the model."
981
+ )
982
+ elif (
983
+ expected_add_embed_dim < passed_add_embed_dim
984
+ and (passed_add_embed_dim - expected_add_embed_dim) == self.unet.config.addition_time_embed_dim
985
+ ):
986
+ raise ValueError(
987
+ f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. Please make sure to disable `requires_aesthetics_score` with `pipe.register_to_config(requires_aesthetics_score=False)` to make sure `target_size` {target_size} is correctly used by the model."
988
+ )
989
+ elif expected_add_embed_dim != passed_add_embed_dim:
990
+ raise ValueError(
991
+ f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`."
992
+ )
993
+
994
+ add_time_ids = torch.tensor([add_time_ids], dtype=dtype)
995
+ add_neg_time_ids = torch.tensor([add_neg_time_ids], dtype=dtype)
996
+
997
+ return add_time_ids, add_neg_time_ids
998
+
999
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.StableDiffusionUpscalePipeline.upcast_vae
1000
+ def upcast_vae(self):
1001
+ deprecate(
1002
+ "upcast_vae",
1003
+ "1.0.0",
1004
+ "`upcast_vae` is deprecated. Please use `pipe.vae.to(torch.float32)`. For more details, please refer to: https://github.com/huggingface/diffusers/pull/12619#issue-3606633695.",
1005
+ )
1006
+ self.vae.to(dtype=torch.float32)
1007
+
1008
+ # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding
1009
+ def get_guidance_scale_embedding(
1010
+ self, w: torch.Tensor, embedding_dim: int = 512, dtype: torch.dtype = torch.float32
1011
+ ) -> torch.Tensor:
1012
+ """
1013
+ See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298
1014
+
1015
+ Args:
1016
+ w (`torch.Tensor`):
1017
+ Generate embedding vectors with a specified guidance scale to subsequently enrich timestep embeddings.
1018
+ embedding_dim (`int`, *optional*, defaults to 512):
1019
+ Dimension of the embeddings to generate.
1020
+ dtype (`torch.dtype`, *optional*, defaults to `torch.float32`):
1021
+ Data type of the generated embeddings.
1022
+
1023
+ Returns:
1024
+ `torch.Tensor`: Embedding vectors with shape `(len(w), embedding_dim)`.
1025
+ """
1026
+ assert len(w.shape) == 1
1027
+ w = w * 1000.0
1028
+
1029
+ half_dim = embedding_dim // 2
1030
+ emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1)
1031
+ emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb)
1032
+ emb = w.to(dtype)[:, None] * emb[None, :]
1033
+ emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)
1034
+ if embedding_dim % 2 == 1: # zero pad
1035
+ emb = torch.nn.functional.pad(emb, (0, 1))
1036
+ assert emb.shape == (w.shape[0], embedding_dim)
1037
+ return emb
1038
+
1039
+ @property
1040
+ def guidance_scale(self):
1041
+ return self._guidance_scale
1042
+
1043
+ @property
1044
+ def guidance_rescale(self):
1045
+ return self._guidance_rescale
1046
+
1047
+ @property
1048
+ def clip_skip(self):
1049
+ return self._clip_skip
1050
+
1051
+ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
1052
+ # of the Imagen paper: https://huggingface.co/papers/2205.11487 . `guidance_scale = 1`
1053
+ # corresponds to doing no classifier free guidance.
1054
+ @property
1055
+ def do_classifier_free_guidance(self):
1056
+ return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None
1057
+
1058
+ @property
1059
+ def cross_attention_kwargs(self):
1060
+ return self._cross_attention_kwargs
1061
+
1062
+ @property
1063
+ def denoising_end(self):
1064
+ return self._denoising_end
1065
+
1066
+ @property
1067
+ def denoising_start(self):
1068
+ return self._denoising_start
1069
+
1070
+ @property
1071
+ def num_timesteps(self):
1072
+ return self._num_timesteps
1073
+
1074
+ @property
1075
+ def interrupt(self):
1076
+ return self._interrupt
1077
+
1078
+ @torch.no_grad()
1079
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
1080
+ def __call__(
1081
+ self,
1082
+ prompt: str | list[str] = None,
1083
+ prompt_2: str | list[str] | None = None,
1084
+ image: PipelineImageInput = None,
1085
+ mask_image: PipelineImageInput = None,
1086
+ masked_image_latents: torch.Tensor = None,
1087
+ height: int | None = None,
1088
+ width: int | None = None,
1089
+ padding_mask_crop: int | None = None,
1090
+ strength: float = 0.9999,
1091
+ num_inference_steps: int = 50,
1092
+ timesteps: list[int] = None,
1093
+ sigmas: list[float] = None,
1094
+ denoising_start: float | None = None,
1095
+ denoising_end: float | None = None,
1096
+ guidance_scale: float = 7.5,
1097
+ negative_prompt: str | list[str] | None = None,
1098
+ negative_prompt_2: str | list[str] | None = None,
1099
+ num_images_per_prompt: int | None = 1,
1100
+ eta: float = 0.0,
1101
+ generator: torch.Generator | list[torch.Generator] | None = None,
1102
+ latents: torch.Tensor | None = None,
1103
+ prompt_embeds: torch.Tensor | None = None,
1104
+ negative_prompt_embeds: torch.Tensor | None = None,
1105
+ pooled_prompt_embeds: torch.Tensor | None = None,
1106
+ negative_pooled_prompt_embeds: torch.Tensor | None = None,
1107
+ ip_adapter_image: PipelineImageInput | None = None,
1108
+ ip_adapter_image_embeds: list[torch.Tensor] | None = None,
1109
+ output_type: str | None = "pil",
1110
+ return_dict: bool = True,
1111
+ cross_attention_kwargs: dict[str, Any] | None = None,
1112
+ guidance_rescale: float = 0.0,
1113
+ original_size: tuple[int, int] = None,
1114
+ crops_coords_top_left: tuple[int, int] = (0, 0),
1115
+ target_size: tuple[int, int] = None,
1116
+ negative_original_size: tuple[int, int] | None = None,
1117
+ negative_crops_coords_top_left: tuple[int, int] = (0, 0),
1118
+ negative_target_size: tuple[int, int] | None = None,
1119
+ aesthetic_score: float = 6.0,
1120
+ negative_aesthetic_score: float = 2.5,
1121
+ clip_skip: int | None = None,
1122
+ callback_on_step_end: Callable[[int, int], None] | PipelineCallback | MultiPipelineCallbacks | None = None,
1123
+ callback_on_step_end_tensor_inputs: list[str] = ["latents"],
1124
+ **kwargs,
1125
+ ):
1126
+ r"""
1127
+ Function invoked when calling the pipeline for generation.
1128
+
1129
+ Args:
1130
+ prompt (`str` or `list[str]`, *optional*):
1131
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
1132
+ instead.
1133
+ prompt_2 (`str` or `list[str]`, *optional*):
1134
+ The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
1135
+ used in both text-encoders
1136
+ image (`PIL.Image.Image`):
1137
+ `Image`, or tensor representing an image batch which will be inpainted, *i.e.* parts of the image will
1138
+ be masked out with `mask_image` and repainted according to `prompt`.
1139
+ mask_image (`PIL.Image.Image`):
1140
+ `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be
1141
+ repainted, while black pixels will be preserved. If `mask_image` is a PIL image, it will be converted
1142
+ to a single channel (luminance) before use. If it's a tensor, it should contain one color channel (L)
1143
+ instead of 3, so the expected shape would be `(B, H, W, 1)`.
1144
+ masked_image_latents (`torch.Tensor`, *optional*):
1145
+ Pre-encoded latent of the masked image (for inpainting).
1146
+ height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
1147
+ The height in pixels of the generated image. This is set to 1024 by default for the best results.
1148
+ Anything below 512 pixels won't work well for
1149
+ [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
1150
+ and checkpoints that are not specifically fine-tuned on low resolutions.
1151
+ width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
1152
+ The width in pixels of the generated image. This is set to 1024 by default for the best results.
1153
+ Anything below 512 pixels won't work well for
1154
+ [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
1155
+ and checkpoints that are not specifically fine-tuned on low resolutions.
1156
+ padding_mask_crop (`int`, *optional*, defaults to `None`):
1157
+ The size of margin in the crop to be applied to the image and masking. If `None`, no crop is applied to
1158
+ image and mask_image. If `padding_mask_crop` is not `None`, it will first find a rectangular region
1159
+ with the same aspect ration of the image and contains all masked area, and then expand that area based
1160
+ on `padding_mask_crop`. The image and mask_image will then be cropped based on the expanded area before
1161
+ resizing to the original image size for inpainting. This is useful when the masked area is small while
1162
+ the image is large and contain information irrelevant for inpainting, such as background.
1163
+ strength (`float`, *optional*, defaults to 0.9999):
1164
+ Conceptually, indicates how much to transform the masked portion of the reference `image`. Must be
1165
+ between 0 and 1. `image` will be used as a starting point, adding more noise to it the larger the
1166
+ `strength`. The number of denoising steps depends on the amount of noise initially added. When
1167
+ `strength` is 1, added noise will be maximum and the denoising process will run for the full number of
1168
+ iterations specified in `num_inference_steps`. A value of 1, therefore, essentially ignores the masked
1169
+ portion of the reference `image`. Note that in the case of `denoising_start` being declared as an
1170
+ integer, the value of `strength` will be ignored.
1171
+ num_inference_steps (`int`, *optional*, defaults to 50):
1172
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
1173
+ expense of slower inference.
1174
+ timesteps (`list[int]`, *optional*):
1175
+ Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument
1176
+ in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is
1177
+ passed will be used. Must be in descending order.
1178
+ sigmas (`list[float]`, *optional*):
1179
+ Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
1180
+ their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
1181
+ will be used.
1182
+ denoising_start (`float`, *optional*):
1183
+ When specified, indicates the fraction (between 0.0 and 1.0) of the total denoising process to be
1184
+ bypassed before it is initiated. Consequently, the initial part of the denoising process is skipped and
1185
+ it is assumed that the passed `image` is a partly denoised image. Note that when this is specified,
1186
+ strength will be ignored. The `denoising_start` parameter is particularly beneficial when this pipeline
1187
+ is integrated into a "Mixture of Denoisers" multi-pipeline setup, as detailed in [**Refining the Image
1188
+ Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output).
1189
+ denoising_end (`float`, *optional*):
1190
+ When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be
1191
+ completed before it is intentionally prematurely terminated. As a result, the returned sample will
1192
+ still retain a substantial amount of noise (ca. final 20% of timesteps still needed) and should be
1193
+ denoised by a successor pipeline that has `denoising_start` set to 0.8 so that it only denoises the
1194
+ final 20% of the scheduler. The denoising_end parameter should ideally be utilized when this pipeline
1195
+ forms a part of a "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image
1196
+ Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output).
1197
+ guidance_scale (`float`, *optional*, defaults to 7.5):
1198
+ Guidance scale as defined in [Classifier-Free Diffusion
1199
+ Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2.
1200
+ of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
1201
+ `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to
1202
+ the text `prompt`, usually at the expense of lower image quality.
1203
+ negative_prompt (`str` or `list[str]`, *optional*):
1204
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
1205
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
1206
+ less than `1`).
1207
+ negative_prompt_2 (`str` or `list[str]`, *optional*):
1208
+ The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
1209
+ `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
1210
+ prompt_embeds (`torch.Tensor`, *optional*):
1211
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
1212
+ provided, text embeddings will be generated from `prompt` input argument.
1213
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
1214
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
1215
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
1216
+ argument.
1217
+ pooled_prompt_embeds (`torch.Tensor`, *optional*):
1218
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
1219
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
1220
+ negative_pooled_prompt_embeds (`torch.Tensor`, *optional*):
1221
+ Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
1222
+ weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
1223
+ input argument.
1224
+ ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.
1225
+ ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*):
1226
+ Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of
1227
+ IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should
1228
+ contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not
1229
+ provided, embeddings are computed from the `ip_adapter_image` input argument.
1230
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
1231
+ The number of images to generate per prompt.
1232
+ eta (`float`, *optional*, defaults to 0.0):
1233
+ Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only
1234
+ applies to [`schedulers.DDIMScheduler`], will be ignored for others.
1235
+ generator (`torch.Generator`, *optional*):
1236
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
1237
+ to make generation deterministic.
1238
+ latents (`torch.Tensor`, *optional*):
1239
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
1240
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
1241
+ tensor will be generated by sampling using the supplied random `generator`.
1242
+ output_type (`str`, *optional*, defaults to `"pil"`):
1243
+ The output format of the generate image. Choose between
1244
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
1245
+ return_dict (`bool`, *optional*, defaults to `True`):
1246
+ Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
1247
+ plain tuple.
1248
+ cross_attention_kwargs (`dict`, *optional*):
1249
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
1250
+ `self.processor` in
1251
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
1252
+ original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
1253
+ If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled.
1254
+ `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as
1255
+ explained in section 2.2 of
1256
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
1257
+ crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)):
1258
+ `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position
1259
+ `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting
1260
+ `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of
1261
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
1262
+ target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
1263
+ For most cases, `target_size` should be set to the desired height and width of the generated image. If
1264
+ not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in
1265
+ section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
1266
+ negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
1267
+ To negatively condition the generation process based on a specific image resolution. Part of SDXL's
1268
+ micro-conditioning as explained in section 2.2 of
1269
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
1270
+ information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
1271
+ negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)):
1272
+ To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's
1273
+ micro-conditioning as explained in section 2.2 of
1274
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
1275
+ information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
1276
+ negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
1277
+ To negatively condition the generation process based on a target image resolution. It should be as same
1278
+ as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of
1279
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
1280
+ information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
1281
+ aesthetic_score (`float`, *optional*, defaults to 6.0):
1282
+ Used to simulate an aesthetic score of the generated image by influencing the positive text condition.
1283
+ Part of SDXL's micro-conditioning as explained in section 2.2 of
1284
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
1285
+ negative_aesthetic_score (`float`, *optional*, defaults to 2.5):
1286
+ Part of SDXL's micro-conditioning as explained in section 2.2 of
1287
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). Can be used to
1288
+ simulate an aesthetic score of the generated image by influencing the negative text condition.
1289
+ guidance_rescale (`float`, *optional*, defaults to 0.0):
1290
+ Guidance rescale factor from [Common Diffusion Noise Schedules and Sample Steps are
1291
+ Flawed](https://arxiv.org/pdf/2305.08891.pdf).
1292
+ clip_skip (`int`, *optional*):
1293
+ Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
1294
+ the output of the pre-final layer will be used for computing the prompt embeddings.
1295
+ callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*):
1296
+ A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of
1297
+ each denoising step during the inference. with the following arguments: `callback_on_step_end(self:
1298
+ DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a
1299
+ list of all tensors as specified by `callback_on_step_end_tensor_inputs`.
1300
+ callback_on_step_end_tensor_inputs (`list`, *optional*):
1301
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
1302
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
1303
+ `._callback_tensor_inputs` attribute of your pipeline class.
1304
+
1305
+ Examples:
1306
+
1307
+ Returns:
1308
+ [`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] or `tuple`:
1309
+ [`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a
1310
+ `tuple. `tuple. When returning a tuple, the first element is a list with the generated images.
1311
+ """
1312
+
1313
+ callback = kwargs.pop("callback", None)
1314
+ callback_steps = kwargs.pop("callback_steps", None)
1315
+
1316
+ if callback is not None:
1317
+ deprecate(
1318
+ "callback",
1319
+ "1.0.0",
1320
+ "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
1321
+ )
1322
+ if callback_steps is not None:
1323
+ deprecate(
1324
+ "callback_steps",
1325
+ "1.0.0",
1326
+ "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
1327
+ )
1328
+
1329
+ if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
1330
+ callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
1331
+
1332
+ # 0. Default height and width to unet
1333
+ height = height or self.unet.config.sample_size * self.vae_scale_factor
1334
+ width = width or self.unet.config.sample_size * self.vae_scale_factor
1335
+
1336
+ # 1. Check inputs
1337
+ self.check_inputs(
1338
+ prompt,
1339
+ prompt_2,
1340
+ image,
1341
+ mask_image,
1342
+ height,
1343
+ width,
1344
+ strength,
1345
+ callback_steps,
1346
+ output_type,
1347
+ negative_prompt,
1348
+ negative_prompt_2,
1349
+ prompt_embeds,
1350
+ negative_prompt_embeds,
1351
+ ip_adapter_image,
1352
+ ip_adapter_image_embeds,
1353
+ callback_on_step_end_tensor_inputs,
1354
+ padding_mask_crop,
1355
+ )
1356
+
1357
+ self._guidance_scale = guidance_scale
1358
+ self._guidance_rescale = guidance_rescale
1359
+ self._clip_skip = clip_skip
1360
+ self._cross_attention_kwargs = cross_attention_kwargs
1361
+ self._denoising_end = denoising_end
1362
+ self._denoising_start = denoising_start
1363
+ self._interrupt = False
1364
+
1365
+ # 2. Define call parameters
1366
+ if prompt is not None and isinstance(prompt, str):
1367
+ batch_size = 1
1368
+ elif prompt is not None and isinstance(prompt, list):
1369
+ batch_size = len(prompt)
1370
+ else:
1371
+ batch_size = prompt_embeds.shape[0]
1372
+
1373
+ device = self._execution_device
1374
+
1375
+ # 3. Encode input prompt
1376
+ text_encoder_lora_scale = (
1377
+ self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None
1378
+ )
1379
+
1380
+ (
1381
+ prompt_embeds,
1382
+ negative_prompt_embeds,
1383
+ pooled_prompt_embeds,
1384
+ negative_pooled_prompt_embeds,
1385
+ ) = self.encode_prompt(
1386
+ prompt=prompt,
1387
+ prompt_2=prompt_2,
1388
+ device=device,
1389
+ num_images_per_prompt=num_images_per_prompt,
1390
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
1391
+ negative_prompt=negative_prompt,
1392
+ negative_prompt_2=negative_prompt_2,
1393
+ prompt_embeds=prompt_embeds,
1394
+ negative_prompt_embeds=negative_prompt_embeds,
1395
+ pooled_prompt_embeds=pooled_prompt_embeds,
1396
+ negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
1397
+ lora_scale=text_encoder_lora_scale,
1398
+ clip_skip=self.clip_skip,
1399
+ )
1400
+
1401
+ # 4. set timesteps
1402
+ def denoising_value_valid(dnv):
1403
+ return isinstance(dnv, float) and 0 < dnv < 1
1404
+
1405
+ if XLA_AVAILABLE:
1406
+ timestep_device = "cpu"
1407
+ else:
1408
+ timestep_device = device
1409
+ timesteps, num_inference_steps = retrieve_timesteps(
1410
+ self.scheduler, num_inference_steps, timestep_device, timesteps, sigmas
1411
+ )
1412
+ timesteps, num_inference_steps = self.get_timesteps(
1413
+ num_inference_steps,
1414
+ strength,
1415
+ device,
1416
+ denoising_start=self.denoising_start if denoising_value_valid(self.denoising_start) else None,
1417
+ )
1418
+ # check that number of inference steps is not < 1 - as this doesn't make sense
1419
+ if num_inference_steps < 1:
1420
+ raise ValueError(
1421
+ f"After adjusting the num_inference_steps by strength parameter: {strength}, the number of pipeline"
1422
+ f"steps is {num_inference_steps} which is < 1 and not appropriate for this pipeline."
1423
+ )
1424
+ # at which timestep to set the initial noise (n.b. 50% if strength is 0.5)
1425
+ latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
1426
+ # create a boolean to check if the strength is set to 1. if so then initialise the latents with pure noise
1427
+ is_strength_max = strength == 1.0
1428
+
1429
+ # 5. Preprocess mask and image
1430
+ if padding_mask_crop is not None:
1431
+ crops_coords = self.mask_processor.get_crop_region(mask_image, width, height, pad=padding_mask_crop)
1432
+ resize_mode = "fill"
1433
+ else:
1434
+ crops_coords = None
1435
+ resize_mode = "default"
1436
+
1437
+ original_image = image
1438
+ init_image = self.image_processor.preprocess(
1439
+ image, height=height, width=width, crops_coords=crops_coords, resize_mode=resize_mode
1440
+ )
1441
+ init_image = init_image.to(dtype=torch.float32)
1442
+
1443
+ mask = self.mask_processor.preprocess(
1444
+ mask_image, height=height, width=width, resize_mode=resize_mode, crops_coords=crops_coords
1445
+ )
1446
+
1447
+ if masked_image_latents is not None:
1448
+ masked_image = masked_image_latents
1449
+ elif init_image.shape[1] == 4:
1450
+ # if images are in latent space, we can't mask it
1451
+ masked_image = None
1452
+ else:
1453
+ masked_image = init_image * (mask < 0.5)
1454
+
1455
+ # 6. Prepare latent variables
1456
+ num_channels_latents = self.vae.config.latent_channels
1457
+ num_channels_unet = self.unet.config.in_channels
1458
+ return_image_latents = num_channels_unet == 4
1459
+
1460
+ add_noise = True if self.denoising_start is None else False
1461
+ latents_outputs = self.prepare_latents(
1462
+ batch_size * num_images_per_prompt,
1463
+ num_channels_latents,
1464
+ height,
1465
+ width,
1466
+ prompt_embeds.dtype,
1467
+ device,
1468
+ generator,
1469
+ latents,
1470
+ image=init_image,
1471
+ timestep=latent_timestep,
1472
+ is_strength_max=is_strength_max,
1473
+ add_noise=add_noise,
1474
+ return_noise=True,
1475
+ return_image_latents=return_image_latents,
1476
+ )
1477
+
1478
+ if return_image_latents:
1479
+ latents, noise, image_latents = latents_outputs
1480
+ else:
1481
+ latents, noise = latents_outputs
1482
+
1483
+ # 7. Prepare mask latent variables
1484
+ mask, masked_image_latents = self.prepare_mask_latents(
1485
+ mask,
1486
+ masked_image,
1487
+ batch_size * num_images_per_prompt,
1488
+ height,
1489
+ width,
1490
+ prompt_embeds.dtype,
1491
+ device,
1492
+ generator,
1493
+ self.do_classifier_free_guidance,
1494
+ )
1495
+
1496
+ # 8. Check that sizes of mask, masked image and latents match
1497
+ if num_channels_unet == 9:
1498
+ # default case for stable-diffusion-v1-5/stable-diffusion-inpainting
1499
+ num_channels_mask = mask.shape[1]
1500
+ num_channels_masked_image = masked_image_latents.shape[1]
1501
+ if num_channels_latents + num_channels_mask + num_channels_masked_image != self.unet.config.in_channels:
1502
+ raise ValueError(
1503
+ f"Incorrect configuration settings! The config of `pipeline.unet`: {self.unet.config} expects"
1504
+ f" {self.unet.config.in_channels} but received `num_channels_latents`: {num_channels_latents} +"
1505
+ f" `num_channels_mask`: {num_channels_mask} + `num_channels_masked_image`: {num_channels_masked_image}"
1506
+ f" = {num_channels_latents + num_channels_masked_image + num_channels_mask}. Please verify the config of"
1507
+ " `pipeline.unet` or your `mask_image` or `image` input."
1508
+ )
1509
+ elif num_channels_unet != 4:
1510
+ raise ValueError(
1511
+ f"The unet {self.unet.__class__} should have either 4 or 9 input channels, not {self.unet.config.in_channels}."
1512
+ )
1513
+ # 8.1 Prepare extra step kwargs.
1514
+ extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
1515
+
1516
+ # 9. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
1517
+ height, width = latents.shape[-2:]
1518
+ height = height * self.vae_scale_factor
1519
+ width = width * self.vae_scale_factor
1520
+
1521
+ original_size = original_size or (height, width)
1522
+ target_size = target_size or (height, width)
1523
+
1524
+ # 10. Prepare added time ids & embeddings
1525
+ if negative_original_size is None:
1526
+ negative_original_size = original_size
1527
+ if negative_target_size is None:
1528
+ negative_target_size = target_size
1529
+
1530
+ add_text_embeds = pooled_prompt_embeds
1531
+ if self.text_encoder_2 is None:
1532
+ text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1])
1533
+ else:
1534
+ text_encoder_projection_dim = self.text_encoder_2.config.projection_dim
1535
+
1536
+ add_time_ids, add_neg_time_ids = self._get_add_time_ids(
1537
+ original_size,
1538
+ crops_coords_top_left,
1539
+ target_size,
1540
+ aesthetic_score,
1541
+ negative_aesthetic_score,
1542
+ negative_original_size,
1543
+ negative_crops_coords_top_left,
1544
+ negative_target_size,
1545
+ dtype=prompt_embeds.dtype,
1546
+ text_encoder_projection_dim=text_encoder_projection_dim,
1547
+ )
1548
+ add_time_ids = add_time_ids.repeat(batch_size * num_images_per_prompt, 1)
1549
+
1550
+ if self.do_classifier_free_guidance:
1551
+ prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
1552
+ add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0)
1553
+ add_neg_time_ids = add_neg_time_ids.repeat(batch_size * num_images_per_prompt, 1)
1554
+ add_time_ids = torch.cat([add_neg_time_ids, add_time_ids], dim=0)
1555
+
1556
+ prompt_embeds = prompt_embeds.to(device)
1557
+ add_text_embeds = add_text_embeds.to(device)
1558
+ add_time_ids = add_time_ids.to(device)
1559
+
1560
+ if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
1561
+ image_embeds = self.prepare_ip_adapter_image_embeds(
1562
+ ip_adapter_image,
1563
+ ip_adapter_image_embeds,
1564
+ device,
1565
+ batch_size * num_images_per_prompt,
1566
+ self.do_classifier_free_guidance,
1567
+ )
1568
+
1569
+ # 11. Denoising loop
1570
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
1571
+
1572
+ if (
1573
+ self.denoising_end is not None
1574
+ and self.denoising_start is not None
1575
+ and denoising_value_valid(self.denoising_end)
1576
+ and denoising_value_valid(self.denoising_start)
1577
+ and self.denoising_start >= self.denoising_end
1578
+ ):
1579
+ raise ValueError(
1580
+ f"`denoising_start`: {self.denoising_start} cannot be larger than or equal to `denoising_end`: "
1581
+ + f" {self.denoising_end} when using type float."
1582
+ )
1583
+ elif self.denoising_end is not None and denoising_value_valid(self.denoising_end):
1584
+ discrete_timestep_cutoff = int(
1585
+ round(
1586
+ self.scheduler.config.num_train_timesteps
1587
+ - (self.denoising_end * self.scheduler.config.num_train_timesteps)
1588
+ )
1589
+ )
1590
+ num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps)))
1591
+ timesteps = timesteps[:num_inference_steps]
1592
+
1593
+ # 11.1 Optionally get Guidance Scale Embedding
1594
+ timestep_cond = None
1595
+ if self.unet.config.time_cond_proj_dim is not None:
1596
+ guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)
1597
+ timestep_cond = self.get_guidance_scale_embedding(
1598
+ guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim
1599
+ ).to(device=device, dtype=latents.dtype)
1600
+
1601
+ self._num_timesteps = len(timesteps)
1602
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
1603
+ for i, t in enumerate(timesteps):
1604
+ if self.interrupt:
1605
+ continue
1606
+ # expand the latents if we are doing classifier free guidance
1607
+ latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
1608
+
1609
+ # concat latents, mask, masked_image_latents in the channel dimension
1610
+ latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
1611
+
1612
+ if num_channels_unet == 9:
1613
+ latent_model_input = torch.cat([latent_model_input, mask, masked_image_latents], dim=1)
1614
+
1615
+ # predict the noise residual
1616
+ added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
1617
+ if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
1618
+ added_cond_kwargs["image_embeds"] = image_embeds
1619
+ noise_pred = self.unet(
1620
+ latent_model_input,
1621
+ t,
1622
+ encoder_hidden_states=prompt_embeds,
1623
+ timestep_cond=timestep_cond,
1624
+ cross_attention_kwargs=self.cross_attention_kwargs,
1625
+ added_cond_kwargs=added_cond_kwargs,
1626
+ return_dict=False,
1627
+ )[0]
1628
+
1629
+ # perform guidance
1630
+ if self.do_classifier_free_guidance:
1631
+ noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
1632
+ noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)
1633
+
1634
+ if self.do_classifier_free_guidance and self.guidance_rescale > 0.0:
1635
+ # Based on 3.4. in https://huggingface.co/papers/2305.08891
1636
+ noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale)
1637
+
1638
+ # compute the previous noisy sample x_t -> x_t-1
1639
+ latents_dtype = latents.dtype
1640
+ latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
1641
+ if latents.dtype != latents_dtype:
1642
+ if torch.backends.mps.is_available():
1643
+ # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
1644
+ latents = latents.to(latents_dtype)
1645
+
1646
+ if num_channels_unet == 4:
1647
+ init_latents_proper = image_latents
1648
+ if self.do_classifier_free_guidance:
1649
+ init_mask, _ = mask.chunk(2)
1650
+ else:
1651
+ init_mask = mask
1652
+
1653
+ if i < len(timesteps) - 1:
1654
+ noise_timestep = timesteps[i + 1]
1655
+ init_latents_proper = self.scheduler.add_noise(
1656
+ init_latents_proper, noise, torch.tensor([noise_timestep])
1657
+ )
1658
+
1659
+ latents = (1 - init_mask) * init_latents_proper + init_mask * latents
1660
+
1661
+ if callback_on_step_end is not None:
1662
+ callback_kwargs = {}
1663
+ for k in callback_on_step_end_tensor_inputs:
1664
+ callback_kwargs[k] = locals()[k]
1665
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
1666
+
1667
+ latents = callback_outputs.pop("latents", latents)
1668
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
1669
+ add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds)
1670
+ add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids)
1671
+ mask = callback_outputs.pop("mask", mask)
1672
+ masked_image_latents = callback_outputs.pop("masked_image_latents", masked_image_latents)
1673
+
1674
+ # call the callback, if provided
1675
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
1676
+ progress_bar.update()
1677
+ if callback is not None and i % callback_steps == 0:
1678
+ step_idx = i // getattr(self.scheduler, "order", 1)
1679
+ callback(step_idx, t, latents)
1680
+
1681
+ if XLA_AVAILABLE:
1682
+ xm.mark_step()
1683
+
1684
+ if not output_type == "latent":
1685
+ # make sure the VAE is in float32 mode, as it overflows in float16
1686
+ needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast
1687
+
1688
+ if needs_upcasting:
1689
+ self.upcast_vae()
1690
+ latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)
1691
+ elif latents.dtype != self.vae.dtype:
1692
+ if torch.backends.mps.is_available():
1693
+ # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
1694
+ self.vae = self.vae.to(latents.dtype)
1695
+
1696
+ # unscale/denormalize the latents
1697
+ # denormalize with the mean and std if available and not None
1698
+ has_latents_mean = hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None
1699
+ has_latents_std = hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None
1700
+ if has_latents_mean and has_latents_std:
1701
+ latents_mean = (
1702
+ torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype)
1703
+ )
1704
+ latents_std = (
1705
+ torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype)
1706
+ )
1707
+ latents = latents * latents_std / self.vae.config.scaling_factor + latents_mean
1708
+ else:
1709
+ latents = latents / self.vae.config.scaling_factor
1710
+
1711
+ image = self.vae.decode(latents, return_dict=False)[0]
1712
+
1713
+ # cast back to fp16 if needed
1714
+ if needs_upcasting:
1715
+ self.vae.to(dtype=torch.float16)
1716
+ else:
1717
+ return StableDiffusionXLPipelineOutput(images=latents)
1718
+
1719
+ # apply watermark if available
1720
+ if self.watermark is not None:
1721
+ image = self.watermark.apply_watermark(image)
1722
+
1723
+ image = self.image_processor.postprocess(image, output_type=output_type)
1724
+
1725
+ if padding_mask_crop is not None:
1726
+ image = [self.image_processor.apply_overlay(mask_image, original_image, i, crops_coords) for i in image]
1727
+
1728
+ # Offload all models
1729
+ self.maybe_free_model_hooks()
1730
+
1731
+ if not return_dict:
1732
+ return (image,)
1733
+
1734
+ return StableDiffusionXLPipelineOutput(images=image)
western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py ADDED
@@ -0,0 +1,977 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 Harutatsu Akiyama and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from typing import Any, Callable
17
+
18
+ import PIL.Image
19
+ import torch
20
+ from transformers import CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer
21
+
22
+ from ...image_processor import PipelineImageInput, VaeImageProcessor
23
+ from ...loaders import FromSingleFileMixin, StableDiffusionXLLoraLoaderMixin, TextualInversionLoaderMixin
24
+ from ...models import AutoencoderKL, UNet2DConditionModel
25
+ from ...models.lora import adjust_lora_scale_text_encoder
26
+ from ...schedulers import KarrasDiffusionSchedulers
27
+ from ...utils import (
28
+ USE_PEFT_BACKEND,
29
+ deprecate,
30
+ is_invisible_watermark_available,
31
+ is_torch_xla_available,
32
+ logging,
33
+ replace_example_docstring,
34
+ scale_lora_layers,
35
+ )
36
+ from ...utils.torch_utils import randn_tensor
37
+ from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin
38
+ from .pipeline_output import StableDiffusionXLPipelineOutput
39
+
40
+
41
+ if is_invisible_watermark_available():
42
+ from .watermark import StableDiffusionXLWatermarker
43
+
44
+ if is_torch_xla_available():
45
+ import torch_xla.core.xla_model as xm
46
+
47
+ XLA_AVAILABLE = True
48
+ else:
49
+ XLA_AVAILABLE = False
50
+
51
+
52
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
53
+
54
+ EXAMPLE_DOC_STRING = """
55
+ Examples:
56
+ ```py
57
+ >>> import torch
58
+ >>> from diffusers import StableDiffusionXLInstructPix2PixPipeline
59
+ >>> from diffusers.utils import load_image
60
+
61
+ >>> resolution = 768
62
+ >>> image = load_image(
63
+ ... "https://hf.co/datasets/diffusers/diffusers-images-docs/resolve/main/mountain.png"
64
+ ... ).resize((resolution, resolution))
65
+ >>> edit_instruction = "Turn sky into a cloudy one"
66
+
67
+ >>> pipe = StableDiffusionXLInstructPix2PixPipeline.from_pretrained(
68
+ ... "diffusers/sdxl-instructpix2pix-768", torch_dtype=torch.float16
69
+ ... ).to("cuda")
70
+
71
+ >>> edited_image = pipe(
72
+ ... prompt=edit_instruction,
73
+ ... image=image,
74
+ ... height=resolution,
75
+ ... width=resolution,
76
+ ... guidance_scale=3.0,
77
+ ... image_guidance_scale=1.5,
78
+ ... num_inference_steps=30,
79
+ ... ).images[0]
80
+ >>> edited_image
81
+ ```
82
+ """
83
+
84
+
85
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
86
+ def retrieve_latents(
87
+ encoder_output: torch.Tensor, generator: torch.Generator | None = None, sample_mode: str = "sample"
88
+ ):
89
+ if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
90
+ return encoder_output.latent_dist.sample(generator)
91
+ elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
92
+ return encoder_output.latent_dist.mode()
93
+ elif hasattr(encoder_output, "latents"):
94
+ return encoder_output.latents
95
+ else:
96
+ raise AttributeError("Could not access latents of provided encoder_output")
97
+
98
+
99
+ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):
100
+ """
101
+ Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and
102
+ Sample Steps are Flawed](https://huggingface.co/papers/2305.08891). See Section 3.4
103
+ """
104
+ std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)
105
+ std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)
106
+ # rescale the results from guidance (fixes overexposure)
107
+ noise_pred_rescaled = noise_cfg * (std_text / std_cfg)
108
+ # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images
109
+ noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg
110
+ return noise_cfg
111
+
112
+
113
+ class StableDiffusionXLInstructPix2PixPipeline(
114
+ DiffusionPipeline,
115
+ StableDiffusionMixin,
116
+ TextualInversionLoaderMixin,
117
+ FromSingleFileMixin,
118
+ StableDiffusionXLLoraLoaderMixin,
119
+ ):
120
+ r"""
121
+ Pipeline for pixel-level image editing by following text instructions. Based on Stable Diffusion XL.
122
+
123
+ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
124
+ library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
125
+
126
+ The pipeline also inherits the following loading methods:
127
+ - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings
128
+ - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files
129
+ - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights
130
+ - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights
131
+
132
+ Args:
133
+ vae ([`AutoencoderKL`]):
134
+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
135
+ text_encoder ([`CLIPTextModel`]):
136
+ Frozen text-encoder. Stable Diffusion XL uses the text portion of
137
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
138
+ the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
139
+ text_encoder_2 ([` CLIPTextModelWithProjection`]):
140
+ Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of
141
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection),
142
+ specifically the
143
+ [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k)
144
+ variant.
145
+ tokenizer (`CLIPTokenizer`):
146
+ Tokenizer of class
147
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
148
+ tokenizer_2 (`CLIPTokenizer`):
149
+ Second Tokenizer of class
150
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
151
+ unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.
152
+ scheduler ([`SchedulerMixin`]):
153
+ A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
154
+ [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
155
+ requires_aesthetics_score (`bool`, *optional*, defaults to `"False"`):
156
+ Whether the `unet` requires a aesthetic_score condition to be passed during inference. Also see the config
157
+ of `stabilityai/stable-diffusion-xl-refiner-1-0`.
158
+ force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`):
159
+ Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of
160
+ `stabilityai/stable-diffusion-xl-base-1-0`.
161
+ add_watermarker (`bool`, *optional*):
162
+ Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to
163
+ watermark output images. If not defined, it will default to True if the package is installed, otherwise no
164
+ watermarker will be used.
165
+ is_cosxl_edit (`bool`, *optional*):
166
+ When set the image latents are scaled.
167
+ """
168
+
169
+ model_cpu_offload_seq = "text_encoder->text_encoder_2->unet->vae"
170
+ _optional_components = ["tokenizer", "tokenizer_2", "text_encoder", "text_encoder_2"]
171
+
172
+ def __init__(
173
+ self,
174
+ vae: AutoencoderKL,
175
+ text_encoder: CLIPTextModel,
176
+ text_encoder_2: CLIPTextModelWithProjection,
177
+ tokenizer: CLIPTokenizer,
178
+ tokenizer_2: CLIPTokenizer,
179
+ unet: UNet2DConditionModel,
180
+ scheduler: KarrasDiffusionSchedulers,
181
+ force_zeros_for_empty_prompt: bool = True,
182
+ add_watermarker: bool | None = None,
183
+ is_cosxl_edit: bool | None = False,
184
+ ):
185
+ super().__init__()
186
+
187
+ self.register_modules(
188
+ vae=vae,
189
+ text_encoder=text_encoder,
190
+ text_encoder_2=text_encoder_2,
191
+ tokenizer=tokenizer,
192
+ tokenizer_2=tokenizer_2,
193
+ unet=unet,
194
+ scheduler=scheduler,
195
+ )
196
+ self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt)
197
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
198
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
199
+ self.default_sample_size = (
200
+ self.unet.config.sample_size
201
+ if hasattr(self, "unet") and self.unet is not None and hasattr(self.unet.config, "sample_size")
202
+ else 128
203
+ )
204
+ self.is_cosxl_edit = is_cosxl_edit
205
+
206
+ add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available()
207
+
208
+ if add_watermarker:
209
+ self.watermark = StableDiffusionXLWatermarker()
210
+ else:
211
+ self.watermark = None
212
+
213
+ def encode_prompt(
214
+ self,
215
+ prompt: str,
216
+ prompt_2: str | None = None,
217
+ device: torch.device | None = None,
218
+ num_images_per_prompt: int = 1,
219
+ do_classifier_free_guidance: bool = True,
220
+ negative_prompt: str | None = None,
221
+ negative_prompt_2: str | None = None,
222
+ prompt_embeds: torch.Tensor | None = None,
223
+ negative_prompt_embeds: torch.Tensor | None = None,
224
+ pooled_prompt_embeds: torch.Tensor | None = None,
225
+ negative_pooled_prompt_embeds: torch.Tensor | None = None,
226
+ lora_scale: float | None = None,
227
+ ):
228
+ r"""
229
+ Encodes the prompt into text encoder hidden states.
230
+
231
+ Args:
232
+ prompt (`str` or `list[str]`, *optional*):
233
+ prompt to be encoded
234
+ prompt_2 (`str` or `list[str]`, *optional*):
235
+ The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
236
+ used in both text-encoders
237
+ device: (`torch.device`):
238
+ torch device
239
+ num_images_per_prompt (`int`):
240
+ number of images that should be generated per prompt
241
+ do_classifier_free_guidance (`bool`):
242
+ whether to use classifier free guidance or not
243
+ negative_prompt (`str` or `list[str]`, *optional*):
244
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
245
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
246
+ less than `1`).
247
+ negative_prompt_2 (`str` or `list[str]`, *optional*):
248
+ The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
249
+ `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
250
+ prompt_embeds (`torch.Tensor`, *optional*):
251
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
252
+ provided, text embeddings will be generated from `prompt` input argument.
253
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
254
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
255
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
256
+ argument.
257
+ pooled_prompt_embeds (`torch.Tensor`, *optional*):
258
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
259
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
260
+ negative_pooled_prompt_embeds (`torch.Tensor`, *optional*):
261
+ Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
262
+ weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
263
+ input argument.
264
+ lora_scale (`float`, *optional*):
265
+ A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
266
+ """
267
+ device = device or self._execution_device
268
+
269
+ # set lora scale so that monkey patched LoRA
270
+ # function of text encoder can correctly access it
271
+ if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin):
272
+ self._lora_scale = lora_scale
273
+
274
+ # dynamically adjust the LoRA scale
275
+ if self.text_encoder is not None:
276
+ if not USE_PEFT_BACKEND:
277
+ adjust_lora_scale_text_encoder(self.text_encoder, lora_scale)
278
+ else:
279
+ scale_lora_layers(self.text_encoder, lora_scale)
280
+
281
+ if self.text_encoder_2 is not None:
282
+ if not USE_PEFT_BACKEND:
283
+ adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale)
284
+ else:
285
+ scale_lora_layers(self.text_encoder_2, lora_scale)
286
+
287
+ if prompt is not None and isinstance(prompt, str):
288
+ batch_size = 1
289
+ elif prompt is not None and isinstance(prompt, list):
290
+ batch_size = len(prompt)
291
+ else:
292
+ batch_size = prompt_embeds.shape[0]
293
+
294
+ # Define tokenizers and text encoders
295
+ tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2]
296
+ text_encoders = (
297
+ [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2]
298
+ )
299
+
300
+ if prompt_embeds is None:
301
+ prompt_2 = prompt_2 or prompt
302
+ # textual inversion: process multi-vector tokens if necessary
303
+ prompt_embeds_list = []
304
+ prompts = [prompt, prompt_2]
305
+ for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders):
306
+ if isinstance(self, TextualInversionLoaderMixin):
307
+ prompt = self.maybe_convert_prompt(prompt, tokenizer)
308
+
309
+ text_inputs = tokenizer(
310
+ prompt,
311
+ padding="max_length",
312
+ max_length=tokenizer.model_max_length,
313
+ truncation=True,
314
+ return_tensors="pt",
315
+ )
316
+
317
+ text_input_ids = text_inputs.input_ids
318
+ untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
319
+
320
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(
321
+ text_input_ids, untruncated_ids
322
+ ):
323
+ removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1])
324
+ logger.warning(
325
+ "The following part of your input was truncated because CLIP can only handle sequences up to"
326
+ f" {tokenizer.model_max_length} tokens: {removed_text}"
327
+ )
328
+
329
+ prompt_embeds = text_encoder(
330
+ text_input_ids.to(device),
331
+ output_hidden_states=True,
332
+ )
333
+
334
+ # We are only ALWAYS interested in the pooled output of the final text encoder
335
+ if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2:
336
+ pooled_prompt_embeds = prompt_embeds[0]
337
+
338
+ prompt_embeds = prompt_embeds.hidden_states[-2]
339
+
340
+ prompt_embeds_list.append(prompt_embeds)
341
+
342
+ prompt_embeds = torch.concat(prompt_embeds_list, dim=-1)
343
+
344
+ # get unconditional embeddings for classifier free guidance
345
+ zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt
346
+ if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt:
347
+ negative_prompt_embeds = torch.zeros_like(prompt_embeds)
348
+ negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds)
349
+ elif do_classifier_free_guidance and negative_prompt_embeds is None:
350
+ negative_prompt = negative_prompt or ""
351
+ negative_prompt_2 = negative_prompt_2 or negative_prompt
352
+
353
+ uncond_tokens: list[str]
354
+ if prompt is not None and type(prompt) is not type(negative_prompt):
355
+ raise TypeError(
356
+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
357
+ f" {type(prompt)}."
358
+ )
359
+ elif isinstance(negative_prompt, str):
360
+ uncond_tokens = [negative_prompt, negative_prompt_2]
361
+ elif batch_size != len(negative_prompt):
362
+ raise ValueError(
363
+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
364
+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
365
+ " the batch size of `prompt`."
366
+ )
367
+ else:
368
+ uncond_tokens = [negative_prompt, negative_prompt_2]
369
+
370
+ negative_prompt_embeds_list = []
371
+ for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders):
372
+ if isinstance(self, TextualInversionLoaderMixin):
373
+ negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer)
374
+
375
+ max_length = prompt_embeds.shape[1]
376
+ uncond_input = tokenizer(
377
+ negative_prompt,
378
+ padding="max_length",
379
+ max_length=max_length,
380
+ truncation=True,
381
+ return_tensors="pt",
382
+ )
383
+
384
+ negative_prompt_embeds = text_encoder(
385
+ uncond_input.input_ids.to(device),
386
+ output_hidden_states=True,
387
+ )
388
+ # We are only ALWAYS interested in the pooled output of the final text encoder
389
+ if negative_pooled_prompt_embeds is None and negative_prompt_embeds[0].ndim == 2:
390
+ negative_pooled_prompt_embeds = negative_prompt_embeds[0]
391
+ negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2]
392
+
393
+ negative_prompt_embeds_list.append(negative_prompt_embeds)
394
+
395
+ negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1)
396
+
397
+ prompt_embeds_dtype = self.text_encoder_2.dtype if self.text_encoder_2 is not None else self.unet.dtype
398
+ prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
399
+ bs_embed, seq_len, _ = prompt_embeds.shape
400
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
401
+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
402
+ prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
403
+
404
+ if do_classifier_free_guidance:
405
+ # duplicate unconditional embeddings for each generation per prompt, using mps friendly method
406
+ seq_len = negative_prompt_embeds.shape[1]
407
+ negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
408
+ negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
409
+ negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
410
+
411
+ pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
412
+ bs_embed * num_images_per_prompt, -1
413
+ )
414
+ if do_classifier_free_guidance:
415
+ negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
416
+ bs_embed * num_images_per_prompt, -1
417
+ )
418
+
419
+ return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds
420
+
421
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs
422
+ def prepare_extra_step_kwargs(self, generator, eta):
423
+ # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
424
+ # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
425
+ # eta corresponds to η in DDIM paper: https://huggingface.co/papers/2010.02502
426
+ # and should be between [0, 1]
427
+
428
+ accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
429
+ extra_step_kwargs = {}
430
+ if accepts_eta:
431
+ extra_step_kwargs["eta"] = eta
432
+
433
+ # check if the scheduler accepts generator
434
+ accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
435
+ if accepts_generator:
436
+ extra_step_kwargs["generator"] = generator
437
+ return extra_step_kwargs
438
+
439
+ def check_inputs(
440
+ self,
441
+ prompt,
442
+ callback_steps,
443
+ negative_prompt=None,
444
+ prompt_embeds=None,
445
+ negative_prompt_embeds=None,
446
+ callback_on_step_end_tensor_inputs=None,
447
+ ):
448
+ if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0):
449
+ raise ValueError(
450
+ f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
451
+ f" {type(callback_steps)}."
452
+ )
453
+
454
+ if callback_on_step_end_tensor_inputs is not None and not all(
455
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
456
+ ):
457
+ raise ValueError(
458
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
459
+ )
460
+
461
+ if prompt is not None and prompt_embeds is not None:
462
+ raise ValueError(
463
+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
464
+ " only forward one of the two."
465
+ )
466
+ elif prompt is None and prompt_embeds is None:
467
+ raise ValueError(
468
+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
469
+ )
470
+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
471
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
472
+
473
+ if negative_prompt is not None and negative_prompt_embeds is not None:
474
+ raise ValueError(
475
+ f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
476
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
477
+ )
478
+
479
+ if prompt_embeds is not None and negative_prompt_embeds is not None:
480
+ if prompt_embeds.shape != negative_prompt_embeds.shape:
481
+ raise ValueError(
482
+ "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"
483
+ f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"
484
+ f" {negative_prompt_embeds.shape}."
485
+ )
486
+
487
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents
488
+ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):
489
+ shape = (
490
+ batch_size,
491
+ num_channels_latents,
492
+ int(height) // self.vae_scale_factor,
493
+ int(width) // self.vae_scale_factor,
494
+ )
495
+ if isinstance(generator, list) and len(generator) != batch_size:
496
+ raise ValueError(
497
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
498
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
499
+ )
500
+
501
+ if latents is None:
502
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
503
+ else:
504
+ latents = latents.to(device)
505
+
506
+ # scale the initial noise by the standard deviation required by the scheduler
507
+ latents = latents * self.scheduler.init_noise_sigma
508
+ return latents
509
+
510
+ def prepare_image_latents(
511
+ self, image, batch_size, num_images_per_prompt, dtype, device, do_classifier_free_guidance, generator=None
512
+ ):
513
+ if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)):
514
+ raise ValueError(
515
+ f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}"
516
+ )
517
+
518
+ image = image.to(device=device, dtype=dtype)
519
+
520
+ batch_size = batch_size * num_images_per_prompt
521
+
522
+ if image.shape[1] == 4:
523
+ image_latents = image
524
+ else:
525
+ # make sure the VAE is in float32 mode, as it overflows in float16
526
+ needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast
527
+ if needs_upcasting:
528
+ image = image.float()
529
+ self.upcast_vae()
530
+
531
+ image_latents = retrieve_latents(self.vae.encode(image), sample_mode="argmax")
532
+
533
+ # cast back to fp16 if needed
534
+ if needs_upcasting:
535
+ self.vae.to(dtype=torch.float16)
536
+
537
+ if batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] == 0:
538
+ # expand image_latents for batch_size
539
+ deprecation_message = (
540
+ f"You have passed {batch_size} text prompts (`prompt`), but only {image_latents.shape[0]} initial"
541
+ " images (`image`). Initial images are now duplicating to match the number of text prompts. Note"
542
+ " that this behavior is deprecated and will be removed in a version 1.0.0. Please make sure to update"
543
+ " your script to pass as many initial images as text prompts to suppress this warning."
544
+ )
545
+ deprecate("len(prompt) != len(image)", "1.0.0", deprecation_message, standard_warn=False)
546
+ additional_image_per_prompt = batch_size // image_latents.shape[0]
547
+ image_latents = torch.cat([image_latents] * additional_image_per_prompt, dim=0)
548
+ elif batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] != 0:
549
+ raise ValueError(
550
+ f"Cannot duplicate `image` of batch size {image_latents.shape[0]} to {batch_size} text prompts."
551
+ )
552
+ else:
553
+ image_latents = torch.cat([image_latents], dim=0)
554
+
555
+ if do_classifier_free_guidance:
556
+ uncond_image_latents = torch.zeros_like(image_latents)
557
+ image_latents = torch.cat([image_latents, image_latents, uncond_image_latents], dim=0)
558
+
559
+ if image_latents.dtype != self.vae.dtype:
560
+ image_latents = image_latents.to(dtype=self.vae.dtype)
561
+
562
+ if self.is_cosxl_edit:
563
+ image_latents = image_latents * self.vae.config.scaling_factor
564
+
565
+ return image_latents
566
+
567
+ # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline._get_add_time_ids
568
+ def _get_add_time_ids(
569
+ self, original_size, crops_coords_top_left, target_size, dtype, text_encoder_projection_dim=None
570
+ ):
571
+ add_time_ids = list(original_size + crops_coords_top_left + target_size)
572
+
573
+ passed_add_embed_dim = (
574
+ self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim
575
+ )
576
+ expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features
577
+
578
+ if expected_add_embed_dim != passed_add_embed_dim:
579
+ raise ValueError(
580
+ f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`."
581
+ )
582
+
583
+ add_time_ids = torch.tensor([add_time_ids], dtype=dtype)
584
+ return add_time_ids
585
+
586
+ # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.upcast_vae
587
+ def upcast_vae(self):
588
+ deprecate(
589
+ "upcast_vae",
590
+ "1.0.0",
591
+ "`upcast_vae` is deprecated. Please use `pipe.vae.to(torch.float32)`. For more details, please refer to: https://github.com/huggingface/diffusers/pull/12619#issue-3606633695.",
592
+ )
593
+ self.vae.to(dtype=torch.float32)
594
+
595
+ @torch.no_grad()
596
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
597
+ def __call__(
598
+ self,
599
+ prompt: str | list[str] = None,
600
+ prompt_2: str | list[str] | None = None,
601
+ image: PipelineImageInput = None,
602
+ height: int | None = None,
603
+ width: int | None = None,
604
+ num_inference_steps: int = 100,
605
+ denoising_end: float | None = None,
606
+ guidance_scale: float = 5.0,
607
+ image_guidance_scale: float = 1.5,
608
+ negative_prompt: str | list[str] | None = None,
609
+ negative_prompt_2: str | list[str] | None = None,
610
+ num_images_per_prompt: int | None = 1,
611
+ eta: float = 0.0,
612
+ generator: torch.Generator | list[torch.Generator] | None = None,
613
+ latents: torch.Tensor | None = None,
614
+ prompt_embeds: torch.Tensor | None = None,
615
+ negative_prompt_embeds: torch.Tensor | None = None,
616
+ pooled_prompt_embeds: torch.Tensor | None = None,
617
+ negative_pooled_prompt_embeds: torch.Tensor | None = None,
618
+ output_type: str | None = "pil",
619
+ return_dict: bool = True,
620
+ callback: Callable[[int, int, torch.Tensor], None] | None = None,
621
+ callback_steps: int = 1,
622
+ cross_attention_kwargs: dict[str, Any] | None = None,
623
+ guidance_rescale: float = 0.0,
624
+ original_size: tuple[int, int] = None,
625
+ crops_coords_top_left: tuple[int, int] = (0, 0),
626
+ target_size: tuple[int, int] = None,
627
+ ):
628
+ r"""
629
+ Function invoked when calling the pipeline for generation.
630
+
631
+ Args:
632
+ prompt (`str` or `list[str]`, *optional*):
633
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
634
+ instead.
635
+ prompt_2 (`str` or `list[str]`, *optional*):
636
+ The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
637
+ used in both text-encoders
638
+ image (`torch.Tensor` or `PIL.Image.Image` or `np.ndarray` or `list[torch.Tensor]` or `list[PIL.Image.Image]` or `list[np.ndarray]`):
639
+ The image(s) to modify with the pipeline.
640
+ height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
641
+ The height in pixels of the generated image.
642
+ width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
643
+ The width in pixels of the generated image.
644
+ num_inference_steps (`int`, *optional*, defaults to 50):
645
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
646
+ expense of slower inference.
647
+ denoising_end (`float`, *optional*):
648
+ When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be
649
+ completed before it is intentionally prematurely terminated. As a result, the returned sample will
650
+ still retain a substantial amount of noise as determined by the discrete timesteps selected by the
651
+ scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a
652
+ "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image
653
+ Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output)
654
+ guidance_scale (`float`, *optional*, defaults to 5.0):
655
+ Guidance scale as defined in [Classifier-Free Diffusion
656
+ Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2.
657
+ of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
658
+ `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to
659
+ the text `prompt`, usually at the expense of lower image quality.
660
+ image_guidance_scale (`float`, *optional*, defaults to 1.5):
661
+ Image guidance scale is to push the generated image towards the initial image `image`. Image guidance
662
+ scale is enabled by setting `image_guidance_scale > 1`. Higher image guidance scale encourages to
663
+ generate images that are closely linked to the source image `image`, usually at the expense of lower
664
+ image quality. This pipeline requires a value of at least `1`.
665
+ negative_prompt (`str` or `list[str]`, *optional*):
666
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
667
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
668
+ less than `1`).
669
+ negative_prompt_2 (`str` or `list[str]`, *optional*):
670
+ The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
671
+ `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders.
672
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
673
+ The number of images to generate per prompt.
674
+ eta (`float`, *optional*, defaults to 0.0):
675
+ Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only
676
+ applies to [`schedulers.DDIMScheduler`], will be ignored for others.
677
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
678
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
679
+ to make generation deterministic.
680
+ latents (`torch.Tensor`, *optional*):
681
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
682
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
683
+ tensor will be generated by sampling using the supplied random `generator`.
684
+ prompt_embeds (`torch.Tensor`, *optional*):
685
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
686
+ provided, text embeddings will be generated from `prompt` input argument.
687
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
688
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
689
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
690
+ argument.
691
+ pooled_prompt_embeds (`torch.Tensor`, *optional*):
692
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
693
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
694
+ negative_pooled_prompt_embeds (`torch.Tensor`, *optional*):
695
+ Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
696
+ weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
697
+ input argument.
698
+ output_type (`str`, *optional*, defaults to `"pil"`):
699
+ The output format of the generate image. Choose between
700
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
701
+ return_dict (`bool`, *optional*, defaults to `True`):
702
+ Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] instead of a
703
+ plain tuple.
704
+ callback (`Callable`, *optional*):
705
+ A function that will be called every `callback_steps` steps during inference. The function will be
706
+ called with the following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`.
707
+ callback_steps (`int`, *optional*, defaults to 1):
708
+ The frequency at which the `callback` function will be called. If not specified, the callback will be
709
+ called at every step.
710
+ cross_attention_kwargs (`dict`, *optional*):
711
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
712
+ `self.processor` in
713
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
714
+ guidance_rescale (`float`, *optional*, defaults to 0.0):
715
+ Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are
716
+ Flawed](https://huggingface.co/papers/2305.08891) `guidance_scale` is defined as `φ` in equation 16. of
717
+ [Common Diffusion Noise Schedules and Sample Steps are
718
+ Flawed](https://huggingface.co/papers/2305.08891). Guidance rescale factor should fix overexposure when
719
+ using zero terminal SNR.
720
+ original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
721
+ If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled.
722
+ `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as
723
+ explained in section 2.2 of
724
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
725
+ crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)):
726
+ `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position
727
+ `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting
728
+ `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of
729
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
730
+ target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
731
+ For most cases, `target_size` should be set to the desired height and width of the generated image. If
732
+ not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in
733
+ section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
734
+
735
+ Examples:
736
+
737
+ Returns:
738
+ [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`:
739
+ [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a
740
+ `tuple`. When returning a tuple, the first element is a list with the generated images.
741
+ """
742
+ # 0. Default height and width to unet
743
+ height = height or self.default_sample_size * self.vae_scale_factor
744
+ width = width or self.default_sample_size * self.vae_scale_factor
745
+
746
+ original_size = original_size or (height, width)
747
+ target_size = target_size or (height, width)
748
+
749
+ # 1. Check inputs. Raise error if not correct
750
+ self.check_inputs(prompt, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds)
751
+
752
+ if image is None:
753
+ raise ValueError("`image` input cannot be undefined.")
754
+
755
+ # 2. Define call parameters
756
+ if prompt is not None and isinstance(prompt, str):
757
+ batch_size = 1
758
+ elif prompt is not None and isinstance(prompt, list):
759
+ batch_size = len(prompt)
760
+ else:
761
+ batch_size = prompt_embeds.shape[0]
762
+
763
+ device = self._execution_device
764
+
765
+ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
766
+ # of the Imagen paper: https://huggingface.co/papers/2205.11487 . `guidance_scale = 1`
767
+ # corresponds to doing no classifier free guidance.
768
+ do_classifier_free_guidance = guidance_scale > 1.0 and image_guidance_scale >= 1.0
769
+
770
+ # 3. Encode input prompt
771
+ text_encoder_lora_scale = (
772
+ cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None
773
+ )
774
+ (
775
+ prompt_embeds,
776
+ negative_prompt_embeds,
777
+ pooled_prompt_embeds,
778
+ negative_pooled_prompt_embeds,
779
+ ) = self.encode_prompt(
780
+ prompt=prompt,
781
+ prompt_2=prompt_2,
782
+ device=device,
783
+ num_images_per_prompt=num_images_per_prompt,
784
+ do_classifier_free_guidance=do_classifier_free_guidance,
785
+ negative_prompt=negative_prompt,
786
+ negative_prompt_2=negative_prompt_2,
787
+ prompt_embeds=prompt_embeds,
788
+ negative_prompt_embeds=negative_prompt_embeds,
789
+ pooled_prompt_embeds=pooled_prompt_embeds,
790
+ negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
791
+ lora_scale=text_encoder_lora_scale,
792
+ )
793
+
794
+ # 4. Preprocess image
795
+ image = self.image_processor.preprocess(image, height=height, width=width).to(device)
796
+
797
+ # 5. Prepare timesteps
798
+ self.scheduler.set_timesteps(num_inference_steps, device=device)
799
+ timesteps = self.scheduler.timesteps
800
+
801
+ # 6. Prepare Image latents
802
+ image_latents = self.prepare_image_latents(
803
+ image,
804
+ batch_size,
805
+ num_images_per_prompt,
806
+ prompt_embeds.dtype,
807
+ device,
808
+ do_classifier_free_guidance,
809
+ )
810
+
811
+ # 7. Prepare latent variables
812
+ num_channels_latents = self.vae.config.latent_channels
813
+ latents = self.prepare_latents(
814
+ batch_size * num_images_per_prompt,
815
+ num_channels_latents,
816
+ height,
817
+ width,
818
+ prompt_embeds.dtype,
819
+ device,
820
+ generator,
821
+ latents,
822
+ )
823
+
824
+ # 8. Check that shapes of latents and image match the UNet channels
825
+ num_channels_image = image_latents.shape[1]
826
+ if num_channels_latents + num_channels_image != self.unet.config.in_channels:
827
+ raise ValueError(
828
+ f"Incorrect configuration settings! The config of `pipeline.unet`: {self.unet.config} expects"
829
+ f" {self.unet.config.in_channels} but received `num_channels_latents`: {num_channels_latents} +"
830
+ f" `num_channels_image`: {num_channels_image} "
831
+ f" = {num_channels_latents + num_channels_image}. Please verify the config of"
832
+ " `pipeline.unet` or your `image` input."
833
+ )
834
+
835
+ # 9. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
836
+ extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
837
+
838
+ # 10. Prepare added time ids & embeddings
839
+ add_text_embeds = pooled_prompt_embeds
840
+ if self.text_encoder_2 is None:
841
+ text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1])
842
+ else:
843
+ text_encoder_projection_dim = self.text_encoder_2.config.projection_dim
844
+
845
+ add_time_ids = self._get_add_time_ids(
846
+ original_size,
847
+ crops_coords_top_left,
848
+ target_size,
849
+ dtype=prompt_embeds.dtype,
850
+ text_encoder_projection_dim=text_encoder_projection_dim,
851
+ )
852
+
853
+ if do_classifier_free_guidance:
854
+ # The extra concat similar to how it's done in SD InstructPix2Pix.
855
+ prompt_embeds = torch.cat([prompt_embeds, negative_prompt_embeds, negative_prompt_embeds], dim=0)
856
+ add_text_embeds = torch.cat(
857
+ [add_text_embeds, negative_pooled_prompt_embeds, negative_pooled_prompt_embeds], dim=0
858
+ )
859
+ add_time_ids = torch.cat([add_time_ids, add_time_ids, add_time_ids], dim=0)
860
+
861
+ prompt_embeds = prompt_embeds.to(device)
862
+ add_text_embeds = add_text_embeds.to(device)
863
+ add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1)
864
+
865
+ # 11. Denoising loop
866
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
867
+ if denoising_end is not None and isinstance(denoising_end, float) and denoising_end > 0 and denoising_end < 1:
868
+ discrete_timestep_cutoff = int(
869
+ round(
870
+ self.scheduler.config.num_train_timesteps
871
+ - (denoising_end * self.scheduler.config.num_train_timesteps)
872
+ )
873
+ )
874
+ num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps)))
875
+ timesteps = timesteps[:num_inference_steps]
876
+
877
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
878
+ for i, t in enumerate(timesteps):
879
+ # Expand the latents if we are doing classifier free guidance.
880
+ # The latents are expanded 3 times because for pix2pix the guidance
881
+ # is applied for both the text and the input image.
882
+ latent_model_input = torch.cat([latents] * 3) if do_classifier_free_guidance else latents
883
+
884
+ # concat latents, image_latents in the channel dimension
885
+ scaled_latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
886
+ scaled_latent_model_input = torch.cat([scaled_latent_model_input, image_latents], dim=1)
887
+
888
+ # predict the noise residual
889
+ added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
890
+ noise_pred = self.unet(
891
+ scaled_latent_model_input,
892
+ t,
893
+ encoder_hidden_states=prompt_embeds,
894
+ cross_attention_kwargs=cross_attention_kwargs,
895
+ added_cond_kwargs=added_cond_kwargs,
896
+ return_dict=False,
897
+ )[0]
898
+
899
+ # perform guidance
900
+ if do_classifier_free_guidance:
901
+ noise_pred_text, noise_pred_image, noise_pred_uncond = noise_pred.chunk(3)
902
+ noise_pred = (
903
+ noise_pred_uncond
904
+ + guidance_scale * (noise_pred_text - noise_pred_image)
905
+ + image_guidance_scale * (noise_pred_image - noise_pred_uncond)
906
+ )
907
+
908
+ if do_classifier_free_guidance and guidance_rescale > 0.0:
909
+ # Based on 3.4. in https://huggingface.co/papers/2305.08891
910
+ noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale)
911
+
912
+ # compute the previous noisy sample x_t -> x_t-1
913
+ latents_dtype = latents.dtype
914
+ latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
915
+ if latents.dtype != latents_dtype:
916
+ if torch.backends.mps.is_available():
917
+ # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
918
+ latents = latents.to(latents_dtype)
919
+
920
+ # call the callback, if provided
921
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
922
+ progress_bar.update()
923
+ if callback is not None and i % callback_steps == 0:
924
+ step_idx = i // getattr(self.scheduler, "order", 1)
925
+ callback(step_idx, t, latents)
926
+
927
+ if XLA_AVAILABLE:
928
+ xm.mark_step()
929
+
930
+ if not output_type == "latent":
931
+ # make sure the VAE is in float32 mode, as it overflows in float16
932
+ needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast
933
+
934
+ if needs_upcasting:
935
+ self.upcast_vae()
936
+ latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)
937
+ elif latents.dtype != self.vae.dtype:
938
+ if torch.backends.mps.is_available():
939
+ # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
940
+ self.vae = self.vae.to(latents.dtype)
941
+
942
+ # unscale/denormalize the latents
943
+ # denormalize with the mean and std if available and not None
944
+ has_latents_mean = hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None
945
+ has_latents_std = hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None
946
+ if has_latents_mean and has_latents_std:
947
+ latents_mean = (
948
+ torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype)
949
+ )
950
+ latents_std = (
951
+ torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype)
952
+ )
953
+ latents = latents * latents_std / self.vae.config.scaling_factor + latents_mean
954
+ else:
955
+ latents = latents / self.vae.config.scaling_factor
956
+
957
+ image = self.vae.decode(latents, return_dict=False)[0]
958
+
959
+ # cast back to fp16 if needed
960
+ if needs_upcasting:
961
+ self.vae.to(dtype=torch.float16)
962
+ else:
963
+ return StableDiffusionXLPipelineOutput(images=latents)
964
+
965
+ # apply watermark if available
966
+ if self.watermark is not None:
967
+ image = self.watermark.apply_watermark(image)
968
+
969
+ image = self.image_processor.postprocess(image, output_type=output_type)
970
+
971
+ # Offload all models
972
+ self.maybe_free_model_hooks()
973
+
974
+ if not return_dict:
975
+ return (image,)
976
+
977
+ return StableDiffusionXLPipelineOutput(images=image)
western_ai/europe/diffusers/src/diffusers/pipelines/stable_diffusion_xl/watermark.py ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ import torch
3
+
4
+ from ...utils import is_invisible_watermark_available
5
+
6
+
7
+ if is_invisible_watermark_available():
8
+ from imwatermark import WatermarkEncoder
9
+
10
+
11
+ # Copied from https://github.com/Stability-AI/generative-models/blob/613af104c6b85184091d42d374fef420eddb356d/scripts/demo/streamlit_helpers.py#L66
12
+ WATERMARK_MESSAGE = 0b101100111110110010010000011110111011000110011110
13
+ # bin(x)[2:] gives bits of x as str, use int to convert them to 0/1
14
+ WATERMARK_BITS = [int(bit) for bit in bin(WATERMARK_MESSAGE)[2:]]
15
+
16
+
17
+ class StableDiffusionXLWatermarker:
18
+ def __init__(self):
19
+ self.watermark = WATERMARK_BITS
20
+ self.encoder = WatermarkEncoder()
21
+
22
+ self.encoder.set_watermark("bits", self.watermark)
23
+
24
+ def apply_watermark(self, images: torch.Tensor):
25
+ # can't encode images that are smaller than 256
26
+ if images.shape[-1] < 256:
27
+ return images
28
+
29
+ images = (255 * (images / 2 + 0.5)).cpu().permute(0, 2, 3, 1).float().numpy()
30
+
31
+ # Convert RGB to BGR, which is the channel order expected by the watermark encoder.
32
+ images = images[:, :, :, ::-1]
33
+
34
+ # Add watermark and convert BGR back to RGB
35
+ images = [self.encoder.encode(image, "dwtDct")[:, :, ::-1] for image in images]
36
+
37
+ images = np.array(images)
38
+
39
+ images = torch.from_numpy(images).permute(0, 3, 1, 2)
40
+
41
+ images = torch.clamp(2 * (images / 255 - 0.5), min=-1.0, max=1.0)
42
+ return images
western_ai/europe/diffusers/src/diffusers/pipelines/stable_video_diffusion/__init__.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import TYPE_CHECKING
2
+
3
+ from ...utils import (
4
+ DIFFUSERS_SLOW_IMPORT,
5
+ BaseOutput,
6
+ OptionalDependencyNotAvailable,
7
+ _LazyModule,
8
+ get_objects_from_module,
9
+ is_torch_available,
10
+ is_transformers_available,
11
+ )
12
+
13
+
14
+ _dummy_objects = {}
15
+ _import_structure = {}
16
+
17
+ try:
18
+ if not (is_transformers_available() and is_torch_available()):
19
+ raise OptionalDependencyNotAvailable()
20
+ except OptionalDependencyNotAvailable:
21
+ from ...utils import dummy_torch_and_transformers_objects
22
+
23
+ _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects))
24
+ else:
25
+ _import_structure.update(
26
+ {
27
+ "pipeline_stable_video_diffusion": [
28
+ "StableVideoDiffusionPipeline",
29
+ "StableVideoDiffusionPipelineOutput",
30
+ ],
31
+ }
32
+ )
33
+
34
+
35
+ if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT:
36
+ try:
37
+ if not (is_transformers_available() and is_torch_available()):
38
+ raise OptionalDependencyNotAvailable()
39
+ except OptionalDependencyNotAvailable:
40
+ from ...utils.dummy_torch_and_transformers_objects import *
41
+ else:
42
+ from .pipeline_stable_video_diffusion import (
43
+ StableVideoDiffusionPipeline,
44
+ StableVideoDiffusionPipelineOutput,
45
+ )
46
+
47
+ else:
48
+ import sys
49
+
50
+ sys.modules[__name__] = _LazyModule(
51
+ __name__,
52
+ globals()["__file__"],
53
+ _import_structure,
54
+ module_spec=__spec__,
55
+ )
56
+
57
+ for name, value in _dummy_objects.items():
58
+ setattr(sys.modules[__name__], name, value)
western_ai/europe/diffusers/src/diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py ADDED
@@ -0,0 +1,743 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2026 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from dataclasses import dataclass
17
+ from typing import Callable
18
+
19
+ import numpy as np
20
+ import PIL.Image
21
+ import torch
22
+ from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection
23
+
24
+ from ...image_processor import PipelineImageInput
25
+ from ...models import AutoencoderKLTemporalDecoder, UNetSpatioTemporalConditionModel
26
+ from ...schedulers import EulerDiscreteScheduler
27
+ from ...utils import BaseOutput, is_torch_xla_available, logging, replace_example_docstring
28
+ from ...utils.torch_utils import is_compiled_module, randn_tensor
29
+ from ...video_processor import VideoProcessor
30
+ from ..pipeline_utils import DiffusionPipeline
31
+
32
+
33
+ if is_torch_xla_available():
34
+ import torch_xla.core.xla_model as xm
35
+
36
+ XLA_AVAILABLE = True
37
+ else:
38
+ XLA_AVAILABLE = False
39
+
40
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
41
+
42
+
43
+ EXAMPLE_DOC_STRING = """
44
+ Examples:
45
+ ```py
46
+ >>> from diffusers import StableVideoDiffusionPipeline
47
+ >>> from diffusers.utils import load_image, export_to_video
48
+
49
+ >>> pipe = StableVideoDiffusionPipeline.from_pretrained(
50
+ ... "stabilityai/stable-video-diffusion-img2vid-xt", torch_dtype=torch.float16, variant="fp16"
51
+ ... )
52
+ >>> pipe.to("cuda")
53
+
54
+ >>> image = load_image(
55
+ ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/svd-docstring-example.jpeg"
56
+ ... )
57
+ >>> image = image.resize((1024, 576))
58
+
59
+ >>> frames = pipe(image, num_frames=25, decode_chunk_size=8).frames[0]
60
+ >>> export_to_video(frames, "generated.mp4", fps=7)
61
+ ```
62
+ """
63
+
64
+
65
+ def _append_dims(x, target_dims):
66
+ """Appends dimensions to the end of a tensor until it has target_dims dimensions."""
67
+ dims_to_append = target_dims - x.ndim
68
+ if dims_to_append < 0:
69
+ raise ValueError(f"input has {x.ndim} dims but target_dims is {target_dims}, which is less")
70
+ return x[(...,) + (None,) * dims_to_append]
71
+
72
+
73
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
74
+ def retrieve_timesteps(
75
+ scheduler,
76
+ num_inference_steps: int | None = None,
77
+ device: str | torch.device | None = None,
78
+ timesteps: list[int] | None = None,
79
+ sigmas: list[float] | None = None,
80
+ **kwargs,
81
+ ):
82
+ r"""
83
+ Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
84
+ custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
85
+
86
+ Args:
87
+ scheduler (`SchedulerMixin`):
88
+ The scheduler to get timesteps from.
89
+ num_inference_steps (`int`):
90
+ The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
91
+ must be `None`.
92
+ device (`str` or `torch.device`, *optional*):
93
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
94
+ timesteps (`list[int]`, *optional*):
95
+ Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
96
+ `num_inference_steps` and `sigmas` must be `None`.
97
+ sigmas (`list[float]`, *optional*):
98
+ Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
99
+ `num_inference_steps` and `timesteps` must be `None`.
100
+
101
+ Returns:
102
+ `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
103
+ second element is the number of inference steps.
104
+ """
105
+ if timesteps is not None and sigmas is not None:
106
+ raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
107
+ if timesteps is not None:
108
+ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
109
+ if not accepts_timesteps:
110
+ raise ValueError(
111
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
112
+ f" timestep schedules. Please check whether you are using the correct scheduler."
113
+ )
114
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
115
+ timesteps = scheduler.timesteps
116
+ num_inference_steps = len(timesteps)
117
+ elif sigmas is not None:
118
+ accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
119
+ if not accept_sigmas:
120
+ raise ValueError(
121
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
122
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
123
+ )
124
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
125
+ timesteps = scheduler.timesteps
126
+ num_inference_steps = len(timesteps)
127
+ else:
128
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
129
+ timesteps = scheduler.timesteps
130
+ return timesteps, num_inference_steps
131
+
132
+
133
+ @dataclass
134
+ class StableVideoDiffusionPipelineOutput(BaseOutput):
135
+ r"""
136
+ Output class for Stable Video Diffusion pipeline.
137
+
138
+ Args:
139
+ frames (`[list[list[PIL.Image.Image]]`, `np.ndarray`, `torch.Tensor`]):
140
+ list of denoised PIL images of length `batch_size` or numpy array or torch tensor of shape `(batch_size,
141
+ num_frames, height, width, num_channels)`.
142
+ """
143
+
144
+ frames: list[list[PIL.Image.Image]] | np.ndarray | torch.Tensor
145
+
146
+
147
+ class StableVideoDiffusionPipeline(DiffusionPipeline):
148
+ r"""
149
+ Pipeline to generate video from an input image using Stable Video Diffusion.
150
+
151
+ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods
152
+ implemented for all pipelines (downloading, saving, running on a particular device, etc.).
153
+
154
+ Args:
155
+ vae ([`AutoencoderKLTemporalDecoder`]):
156
+ Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations.
157
+ image_encoder ([`~transformers.CLIPVisionModelWithProjection`]):
158
+ Frozen CLIP image-encoder
159
+ ([laion/CLIP-ViT-H-14-laion2B-s32B-b79K](https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K)).
160
+ unet ([`UNetSpatioTemporalConditionModel`]):
161
+ A `UNetSpatioTemporalConditionModel` to denoise the encoded image latents.
162
+ scheduler ([`EulerDiscreteScheduler`]):
163
+ A scheduler to be used in combination with `unet` to denoise the encoded image latents.
164
+ feature_extractor ([`~transformers.CLIPImageProcessor`]):
165
+ A `CLIPImageProcessor` to extract features from generated images.
166
+ """
167
+
168
+ model_cpu_offload_seq = "image_encoder->unet->vae"
169
+ _callback_tensor_inputs = ["latents"]
170
+
171
+ def __init__(
172
+ self,
173
+ vae: AutoencoderKLTemporalDecoder,
174
+ image_encoder: CLIPVisionModelWithProjection,
175
+ unet: UNetSpatioTemporalConditionModel,
176
+ scheduler: EulerDiscreteScheduler,
177
+ feature_extractor: CLIPImageProcessor,
178
+ ):
179
+ super().__init__()
180
+
181
+ self.register_modules(
182
+ vae=vae,
183
+ image_encoder=image_encoder,
184
+ unet=unet,
185
+ scheduler=scheduler,
186
+ feature_extractor=feature_extractor,
187
+ )
188
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
189
+ self.video_processor = VideoProcessor(do_resize=True, vae_scale_factor=self.vae_scale_factor)
190
+
191
+ def _encode_image(
192
+ self,
193
+ image: PipelineImageInput,
194
+ device: str | torch.device,
195
+ num_videos_per_prompt: int,
196
+ do_classifier_free_guidance: bool,
197
+ ) -> torch.Tensor:
198
+ dtype = next(self.image_encoder.parameters()).dtype
199
+
200
+ if not isinstance(image, torch.Tensor):
201
+ image = self.video_processor.pil_to_numpy(image)
202
+ image = self.video_processor.numpy_to_pt(image)
203
+
204
+ # We normalize the image before resizing to match with the original implementation.
205
+ # Then we unnormalize it after resizing.
206
+ image = image * 2.0 - 1.0
207
+ image = _resize_with_antialiasing(image, (224, 224))
208
+ image = (image + 1.0) / 2.0
209
+
210
+ # Normalize the image with for CLIP input
211
+ image = self.feature_extractor(
212
+ images=image,
213
+ do_normalize=True,
214
+ do_center_crop=False,
215
+ do_resize=False,
216
+ do_rescale=False,
217
+ return_tensors="pt",
218
+ ).pixel_values
219
+
220
+ image = image.to(device=device, dtype=dtype)
221
+ image_embeddings = self.image_encoder(image).image_embeds
222
+ image_embeddings = image_embeddings.unsqueeze(1)
223
+
224
+ # duplicate image embeddings for each generation per prompt, using mps friendly method
225
+ bs_embed, seq_len, _ = image_embeddings.shape
226
+ image_embeddings = image_embeddings.repeat(1, num_videos_per_prompt, 1)
227
+ image_embeddings = image_embeddings.view(bs_embed * num_videos_per_prompt, seq_len, -1)
228
+
229
+ if do_classifier_free_guidance:
230
+ negative_image_embeddings = torch.zeros_like(image_embeddings)
231
+
232
+ # For classifier free guidance, we need to do two forward passes.
233
+ # Here we concatenate the unconditional and text embeddings into a single batch
234
+ # to avoid doing two forward passes
235
+ image_embeddings = torch.cat([negative_image_embeddings, image_embeddings])
236
+
237
+ return image_embeddings
238
+
239
+ def _encode_vae_image(
240
+ self,
241
+ image: torch.Tensor,
242
+ device: str | torch.device,
243
+ num_videos_per_prompt: int,
244
+ do_classifier_free_guidance: bool,
245
+ ):
246
+ image = image.to(device=device)
247
+ image_latents = self.vae.encode(image).latent_dist.mode()
248
+
249
+ # duplicate image_latents for each generation per prompt, using mps friendly method
250
+ image_latents = image_latents.repeat(num_videos_per_prompt, 1, 1, 1)
251
+
252
+ if do_classifier_free_guidance:
253
+ negative_image_latents = torch.zeros_like(image_latents)
254
+
255
+ # For classifier free guidance, we need to do two forward passes.
256
+ # Here we concatenate the unconditional and text embeddings into a single batch
257
+ # to avoid doing two forward passes
258
+ image_latents = torch.cat([negative_image_latents, image_latents])
259
+
260
+ return image_latents
261
+
262
+ def _get_add_time_ids(
263
+ self,
264
+ fps: int,
265
+ motion_bucket_id: int,
266
+ noise_aug_strength: float,
267
+ dtype: torch.dtype,
268
+ batch_size: int,
269
+ num_videos_per_prompt: int,
270
+ do_classifier_free_guidance: bool,
271
+ ):
272
+ add_time_ids = [fps, motion_bucket_id, noise_aug_strength]
273
+
274
+ passed_add_embed_dim = self.unet.config.addition_time_embed_dim * len(add_time_ids)
275
+ expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features
276
+
277
+ if expected_add_embed_dim != passed_add_embed_dim:
278
+ raise ValueError(
279
+ f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`."
280
+ )
281
+
282
+ add_time_ids = torch.tensor([add_time_ids], dtype=dtype)
283
+ add_time_ids = add_time_ids.repeat(batch_size * num_videos_per_prompt, 1)
284
+
285
+ if do_classifier_free_guidance:
286
+ add_time_ids = torch.cat([add_time_ids, add_time_ids])
287
+
288
+ return add_time_ids
289
+
290
+ def decode_latents(self, latents: torch.Tensor, num_frames: int, decode_chunk_size: int = 14):
291
+ # [batch, frames, channels, height, width] -> [batch*frames, channels, height, width]
292
+ latents = latents.flatten(0, 1)
293
+
294
+ latents = 1 / self.vae.config.scaling_factor * latents
295
+
296
+ forward_vae_fn = self.vae._orig_mod.forward if is_compiled_module(self.vae) else self.vae.forward
297
+ accepts_num_frames = "num_frames" in set(inspect.signature(forward_vae_fn).parameters.keys())
298
+
299
+ # decode decode_chunk_size frames at a time to avoid OOM
300
+ frames = []
301
+ for i in range(0, latents.shape[0], decode_chunk_size):
302
+ num_frames_in = latents[i : i + decode_chunk_size].shape[0]
303
+ decode_kwargs = {}
304
+ if accepts_num_frames:
305
+ # we only pass num_frames_in if it's expected
306
+ decode_kwargs["num_frames"] = num_frames_in
307
+
308
+ frame = self.vae.decode(latents[i : i + decode_chunk_size], **decode_kwargs).sample
309
+ frames.append(frame)
310
+ frames = torch.cat(frames, dim=0)
311
+
312
+ # [batch*frames, channels, height, width] -> [batch, channels, frames, height, width]
313
+ frames = frames.reshape(-1, num_frames, *frames.shape[1:]).permute(0, 2, 1, 3, 4)
314
+
315
+ # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16
316
+ frames = frames.float()
317
+ return frames
318
+
319
+ def check_inputs(self, image, height, width):
320
+ if (
321
+ not isinstance(image, torch.Tensor)
322
+ and not isinstance(image, PIL.Image.Image)
323
+ and not isinstance(image, list)
324
+ ):
325
+ raise ValueError(
326
+ "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `list[PIL.Image.Image]` but is"
327
+ f" {type(image)}"
328
+ )
329
+
330
+ if height % 8 != 0 or width % 8 != 0:
331
+ raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
332
+
333
+ def prepare_latents(
334
+ self,
335
+ batch_size: int,
336
+ num_frames: int,
337
+ num_channels_latents: int,
338
+ height: int,
339
+ width: int,
340
+ dtype: torch.dtype,
341
+ device: str | torch.device,
342
+ generator: torch.Generator,
343
+ latents: torch.Tensor | None = None,
344
+ ):
345
+ shape = (
346
+ batch_size,
347
+ num_frames,
348
+ num_channels_latents // 2,
349
+ height // self.vae_scale_factor,
350
+ width // self.vae_scale_factor,
351
+ )
352
+ if isinstance(generator, list) and len(generator) != batch_size:
353
+ raise ValueError(
354
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
355
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
356
+ )
357
+
358
+ if latents is None:
359
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
360
+ else:
361
+ latents = latents.to(device)
362
+
363
+ # scale the initial noise by the standard deviation required by the scheduler
364
+ latents = latents * self.scheduler.init_noise_sigma
365
+ return latents
366
+
367
+ @property
368
+ def guidance_scale(self):
369
+ return self._guidance_scale
370
+
371
+ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
372
+ # of the Imagen paper: https://huggingface.co/papers/2205.11487 . `guidance_scale = 1`
373
+ # corresponds to doing no classifier free guidance.
374
+ @property
375
+ def do_classifier_free_guidance(self):
376
+ if isinstance(self.guidance_scale, (int, float)):
377
+ return self.guidance_scale > 1
378
+ return self.guidance_scale.max() > 1
379
+
380
+ @property
381
+ def num_timesteps(self):
382
+ return self._num_timesteps
383
+
384
+ @torch.no_grad()
385
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
386
+ def __call__(
387
+ self,
388
+ image: PIL.Image.Image | list[PIL.Image.Image] | torch.Tensor,
389
+ height: int = 576,
390
+ width: int = 1024,
391
+ num_frames: int | None = None,
392
+ num_inference_steps: int = 25,
393
+ sigmas: list[float] | None = None,
394
+ min_guidance_scale: float = 1.0,
395
+ max_guidance_scale: float = 3.0,
396
+ fps: int = 7,
397
+ motion_bucket_id: int = 127,
398
+ noise_aug_strength: float = 0.02,
399
+ decode_chunk_size: int | None = None,
400
+ num_videos_per_prompt: int | None = 1,
401
+ generator: torch.Generator | list[torch.Generator] | None = None,
402
+ latents: torch.Tensor | None = None,
403
+ output_type: str | None = "pil",
404
+ callback_on_step_end: Callable[[int, int], None] | None = None,
405
+ callback_on_step_end_tensor_inputs: list[str] = ["latents"],
406
+ return_dict: bool = True,
407
+ ):
408
+ r"""
409
+ The call function to the pipeline for generation.
410
+
411
+ Args:
412
+ image (`PIL.Image.Image` or `list[PIL.Image.Image]` or `torch.Tensor`):
413
+ Image(s) to guide image generation. If you provide a tensor, the expected value range is between `[0,
414
+ 1]`.
415
+ height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):
416
+ The height in pixels of the generated image.
417
+ width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):
418
+ The width in pixels of the generated image.
419
+ num_frames (`int`, *optional*):
420
+ The number of video frames to generate. Defaults to `self.unet.config.num_frames` (14 for
421
+ `stable-video-diffusion-img2vid` and to 25 for `stable-video-diffusion-img2vid-xt`).
422
+ num_inference_steps (`int`, *optional*, defaults to 25):
423
+ The number of denoising steps. More denoising steps usually lead to a higher quality video at the
424
+ expense of slower inference. This parameter is modulated by `strength`.
425
+ sigmas (`list[float]`, *optional*):
426
+ Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
427
+ their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
428
+ will be used.
429
+ min_guidance_scale (`float`, *optional*, defaults to 1.0):
430
+ The minimum guidance scale. Used for the classifier free guidance with first frame.
431
+ max_guidance_scale (`float`, *optional*, defaults to 3.0):
432
+ The maximum guidance scale. Used for the classifier free guidance with last frame.
433
+ fps (`int`, *optional*, defaults to 7):
434
+ Frames per second. The rate at which the generated images shall be exported to a video after
435
+ generation. Note that Stable Diffusion Video's UNet was micro-conditioned on fps-1 during training.
436
+ motion_bucket_id (`int`, *optional*, defaults to 127):
437
+ Used for conditioning the amount of motion for the generation. The higher the number the more motion
438
+ will be in the video.
439
+ noise_aug_strength (`float`, *optional*, defaults to 0.02):
440
+ The amount of noise added to the init image, the higher it is the less the video will look like the
441
+ init image. Increase it for more motion.
442
+ decode_chunk_size (`int`, *optional*):
443
+ The number of frames to decode at a time. Higher chunk size leads to better temporal consistency at the
444
+ expense of more memory usage. By default, the decoder decodes all frames at once for maximal quality.
445
+ For lower memory usage, reduce `decode_chunk_size`.
446
+ num_videos_per_prompt (`int`, *optional*, defaults to 1):
447
+ The number of videos to generate per prompt.
448
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
449
+ A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make
450
+ generation deterministic.
451
+ latents (`torch.Tensor`, *optional*):
452
+ Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for video
453
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
454
+ tensor is generated by sampling using the supplied random `generator`.
455
+ output_type (`str`, *optional*, defaults to `"pil"`):
456
+ The output format of the generated image. Choose between `pil`, `np` or `pt`.
457
+ callback_on_step_end (`Callable`, *optional*):
458
+ A function that is called at the end of each denoising step during inference. The function is called
459
+ with the following arguments:
460
+ `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`.
461
+ `callback_kwargs` will include a list of all tensors as specified by
462
+ `callback_on_step_end_tensor_inputs`.
463
+ callback_on_step_end_tensor_inputs (`list`, *optional*):
464
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
465
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
466
+ `._callback_tensor_inputs` attribute of your pipeline class.
467
+ return_dict (`bool`, *optional*, defaults to `True`):
468
+ Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
469
+ plain tuple.
470
+
471
+ Examples:
472
+
473
+ Returns:
474
+ [`~pipelines.stable_diffusion.StableVideoDiffusionPipelineOutput`] or `tuple`:
475
+ If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableVideoDiffusionPipelineOutput`] is
476
+ returned, otherwise a `tuple` of (`list[list[PIL.Image.Image]]` or `np.ndarray` or `torch.Tensor`) is
477
+ returned.
478
+ """
479
+ # 0. Default height and width to unet
480
+ height = height or self.unet.config.sample_size * self.vae_scale_factor
481
+ width = width or self.unet.config.sample_size * self.vae_scale_factor
482
+
483
+ num_frames = num_frames if num_frames is not None else self.unet.config.num_frames
484
+ decode_chunk_size = decode_chunk_size if decode_chunk_size is not None else num_frames
485
+
486
+ # 1. Check inputs. Raise error if not correct
487
+ self.check_inputs(image, height, width)
488
+
489
+ # 2. Define call parameters
490
+ if isinstance(image, PIL.Image.Image):
491
+ batch_size = 1
492
+ elif isinstance(image, list):
493
+ batch_size = len(image)
494
+ else:
495
+ batch_size = image.shape[0]
496
+ device = self._execution_device
497
+ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
498
+ # of the Imagen paper: https://huggingface.co/papers/2205.11487 . `guidance_scale = 1`
499
+ # corresponds to doing no classifier free guidance.
500
+ self._guidance_scale = max_guidance_scale
501
+
502
+ # 3. Encode input image
503
+ image_embeddings = self._encode_image(image, device, num_videos_per_prompt, self.do_classifier_free_guidance)
504
+
505
+ # NOTE: Stable Video Diffusion was conditioned on fps - 1, which is why it is reduced here.
506
+ # See: https://github.com/Stability-AI/generative-models/blob/ed0997173f98eaf8f4edf7ba5fe8f15c6b877fd3/scripts/sampling/simple_video_sample.py#L188
507
+ fps = fps - 1
508
+
509
+ # 4. Encode input image using VAE
510
+ image = self.video_processor.preprocess(image, height=height, width=width).to(device)
511
+ noise = randn_tensor(image.shape, generator=generator, device=device, dtype=image.dtype)
512
+ image = image + noise_aug_strength * noise
513
+
514
+ needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast
515
+ if needs_upcasting:
516
+ self.vae.to(dtype=torch.float32)
517
+
518
+ image_latents = self._encode_vae_image(
519
+ image,
520
+ device=device,
521
+ num_videos_per_prompt=num_videos_per_prompt,
522
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
523
+ )
524
+ image_latents = image_latents.to(image_embeddings.dtype)
525
+
526
+ # cast back to fp16 if needed
527
+ if needs_upcasting:
528
+ self.vae.to(dtype=torch.float16)
529
+
530
+ # Repeat the image latents for each frame so we can concatenate them with the noise
531
+ # image_latents [batch, channels, height, width] ->[batch, num_frames, channels, height, width]
532
+ image_latents = image_latents.unsqueeze(1).repeat(1, num_frames, 1, 1, 1)
533
+
534
+ # 5. Get Added Time IDs
535
+ added_time_ids = self._get_add_time_ids(
536
+ fps,
537
+ motion_bucket_id,
538
+ noise_aug_strength,
539
+ image_embeddings.dtype,
540
+ batch_size,
541
+ num_videos_per_prompt,
542
+ self.do_classifier_free_guidance,
543
+ )
544
+ added_time_ids = added_time_ids.to(device)
545
+
546
+ # 6. Prepare timesteps
547
+ if XLA_AVAILABLE:
548
+ timestep_device = "cpu"
549
+ else:
550
+ timestep_device = device
551
+ timesteps, num_inference_steps = retrieve_timesteps(
552
+ self.scheduler, num_inference_steps, timestep_device, None, sigmas
553
+ )
554
+
555
+ # 7. Prepare latent variables
556
+ num_channels_latents = self.unet.config.in_channels
557
+ latents = self.prepare_latents(
558
+ batch_size * num_videos_per_prompt,
559
+ num_frames,
560
+ num_channels_latents,
561
+ height,
562
+ width,
563
+ image_embeddings.dtype,
564
+ device,
565
+ generator,
566
+ latents,
567
+ )
568
+
569
+ # 8. Prepare guidance scale
570
+ guidance_scale = torch.linspace(min_guidance_scale, max_guidance_scale, num_frames).unsqueeze(0)
571
+ guidance_scale = guidance_scale.to(device, latents.dtype)
572
+ guidance_scale = guidance_scale.repeat(batch_size * num_videos_per_prompt, 1)
573
+ guidance_scale = _append_dims(guidance_scale, latents.ndim)
574
+
575
+ self._guidance_scale = guidance_scale
576
+
577
+ # 9. Denoising loop
578
+ num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
579
+ self._num_timesteps = len(timesteps)
580
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
581
+ for i, t in enumerate(timesteps):
582
+ # expand the latents if we are doing classifier free guidance
583
+ latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
584
+ latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
585
+
586
+ # Concatenate image_latents over channels dimension
587
+ latent_model_input = torch.cat([latent_model_input, image_latents], dim=2)
588
+
589
+ # predict the noise residual
590
+ noise_pred = self.unet(
591
+ latent_model_input,
592
+ t,
593
+ encoder_hidden_states=image_embeddings,
594
+ added_time_ids=added_time_ids,
595
+ return_dict=False,
596
+ )[0]
597
+
598
+ # perform guidance
599
+ if self.do_classifier_free_guidance:
600
+ noise_pred_uncond, noise_pred_cond = noise_pred.chunk(2)
601
+ noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_cond - noise_pred_uncond)
602
+
603
+ # compute the previous noisy sample x_t -> x_t-1
604
+ latents = self.scheduler.step(noise_pred, t, latents).prev_sample
605
+
606
+ if callback_on_step_end is not None:
607
+ callback_kwargs = {}
608
+ for k in callback_on_step_end_tensor_inputs:
609
+ callback_kwargs[k] = locals()[k]
610
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
611
+
612
+ latents = callback_outputs.pop("latents", latents)
613
+
614
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
615
+ progress_bar.update()
616
+
617
+ if XLA_AVAILABLE:
618
+ xm.mark_step()
619
+
620
+ if not output_type == "latent":
621
+ # cast back to fp16 if needed
622
+ if needs_upcasting:
623
+ self.vae.to(dtype=torch.float16)
624
+ frames = self.decode_latents(latents, num_frames, decode_chunk_size)
625
+ frames = self.video_processor.postprocess_video(video=frames, output_type=output_type)
626
+ else:
627
+ frames = latents
628
+
629
+ self.maybe_free_model_hooks()
630
+
631
+ if not return_dict:
632
+ return frames
633
+
634
+ return StableVideoDiffusionPipelineOutput(frames=frames)
635
+
636
+
637
+ # resizing utils
638
+ # TODO: clean up later
639
+ def _resize_with_antialiasing(input, size, interpolation="bicubic", align_corners=True):
640
+ h, w = input.shape[-2:]
641
+ factors = (h / size[0], w / size[1])
642
+
643
+ # First, we have to determine sigma
644
+ # Taken from skimage: https://github.com/scikit-image/scikit-image/blob/v0.19.2/skimage/transform/_warps.py#L171
645
+ sigmas = (
646
+ max((factors[0] - 1.0) / 2.0, 0.001),
647
+ max((factors[1] - 1.0) / 2.0, 0.001),
648
+ )
649
+
650
+ # Now kernel size. Good results are for 3 sigma, but that is kind of slow. Pillow uses 1 sigma
651
+ # https://github.com/python-pillow/Pillow/blob/master/src/libImaging/Resample.c#L206
652
+ # But they do it in the 2 passes, which gives better results. Let's try 2 sigmas for now
653
+ ks = int(max(2.0 * 2 * sigmas[0], 3)), int(max(2.0 * 2 * sigmas[1], 3))
654
+
655
+ # Make sure it is odd
656
+ if (ks[0] % 2) == 0:
657
+ ks = ks[0] + 1, ks[1]
658
+
659
+ if (ks[1] % 2) == 0:
660
+ ks = ks[0], ks[1] + 1
661
+
662
+ input = _gaussian_blur2d(input, ks, sigmas)
663
+
664
+ output = torch.nn.functional.interpolate(input, size=size, mode=interpolation, align_corners=align_corners)
665
+ return output
666
+
667
+
668
+ def _compute_padding(kernel_size):
669
+ """Compute padding tuple."""
670
+ # 4 or 6 ints: (padding_left, padding_right,padding_top,padding_bottom)
671
+ # https://pytorch.org/docs/stable/nn.html#torch.nn.functional.pad
672
+ if len(kernel_size) < 2:
673
+ raise AssertionError(kernel_size)
674
+ computed = [k - 1 for k in kernel_size]
675
+
676
+ # for even kernels we need to do asymmetric padding :(
677
+ out_padding = 2 * len(kernel_size) * [0]
678
+
679
+ for i in range(len(kernel_size)):
680
+ computed_tmp = computed[-(i + 1)]
681
+
682
+ pad_front = computed_tmp // 2
683
+ pad_rear = computed_tmp - pad_front
684
+
685
+ out_padding[2 * i + 0] = pad_front
686
+ out_padding[2 * i + 1] = pad_rear
687
+
688
+ return out_padding
689
+
690
+
691
+ def _filter2d(input, kernel):
692
+ # prepare kernel
693
+ b, c, h, w = input.shape
694
+ tmp_kernel = kernel[:, None, ...].to(device=input.device, dtype=input.dtype)
695
+
696
+ tmp_kernel = tmp_kernel.expand(-1, c, -1, -1)
697
+
698
+ height, width = tmp_kernel.shape[-2:]
699
+
700
+ padding_shape: list[int] = _compute_padding([height, width])
701
+ input = torch.nn.functional.pad(input, padding_shape, mode="reflect")
702
+
703
+ # kernel and input tensor reshape to align element-wise or batch-wise params
704
+ tmp_kernel = tmp_kernel.reshape(-1, 1, height, width)
705
+ input = input.view(-1, tmp_kernel.size(0), input.size(-2), input.size(-1))
706
+
707
+ # convolve the tensor with the kernel.
708
+ output = torch.nn.functional.conv2d(input, tmp_kernel, groups=tmp_kernel.size(0), padding=0, stride=1)
709
+
710
+ out = output.view(b, c, h, w)
711
+ return out
712
+
713
+
714
+ def _gaussian(window_size: int, sigma):
715
+ if isinstance(sigma, float):
716
+ sigma = torch.tensor([[sigma]])
717
+
718
+ batch_size = sigma.shape[0]
719
+
720
+ x = (torch.arange(window_size, device=sigma.device, dtype=sigma.dtype) - window_size // 2).expand(batch_size, -1)
721
+
722
+ if window_size % 2 == 0:
723
+ x = x + 0.5
724
+
725
+ gauss = torch.exp(-x.pow(2.0) / (2 * sigma.pow(2.0)))
726
+
727
+ return gauss / gauss.sum(-1, keepdim=True)
728
+
729
+
730
+ def _gaussian_blur2d(input, kernel_size, sigma):
731
+ if isinstance(sigma, tuple):
732
+ sigma = torch.tensor([sigma], dtype=input.dtype)
733
+ else:
734
+ sigma = sigma.to(dtype=input.dtype)
735
+
736
+ ky, kx = int(kernel_size[0]), int(kernel_size[1])
737
+ bs = sigma.shape[0]
738
+ kernel_x = _gaussian(kx, sigma[:, 1].view(bs, 1))
739
+ kernel_y = _gaussian(ky, sigma[:, 0].view(bs, 1))
740
+ out_x = _filter2d(input, kernel_x[..., None, :])
741
+ out = _filter2d(out_x, kernel_y[..., None])
742
+
743
+ return out
western_ai/europe/diffusers/src/diffusers/pipelines/t2i_adapter/__init__.py ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import TYPE_CHECKING
2
+
3
+ from ...utils import (
4
+ DIFFUSERS_SLOW_IMPORT,
5
+ OptionalDependencyNotAvailable,
6
+ _LazyModule,
7
+ get_objects_from_module,
8
+ is_torch_available,
9
+ is_transformers_available,
10
+ )
11
+
12
+
13
+ _dummy_objects = {}
14
+ _import_structure = {}
15
+
16
+ try:
17
+ if not (is_transformers_available() and is_torch_available()):
18
+ raise OptionalDependencyNotAvailable()
19
+ except OptionalDependencyNotAvailable:
20
+ from ...utils import dummy_torch_and_transformers_objects # noqa F403
21
+
22
+ _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects))
23
+ else:
24
+ _import_structure["pipeline_stable_diffusion_adapter"] = ["StableDiffusionAdapterPipeline"]
25
+ _import_structure["pipeline_stable_diffusion_xl_adapter"] = ["StableDiffusionXLAdapterPipeline"]
26
+
27
+
28
+ if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT:
29
+ try:
30
+ if not (is_transformers_available() and is_torch_available()):
31
+ raise OptionalDependencyNotAvailable()
32
+ except OptionalDependencyNotAvailable:
33
+ from ...utils.dummy_torch_and_transformers_objects import * # noqa F403
34
+ else:
35
+ from .pipeline_stable_diffusion_adapter import StableDiffusionAdapterPipeline
36
+ from .pipeline_stable_diffusion_xl_adapter import StableDiffusionXLAdapterPipeline
37
+ else:
38
+ import sys
39
+
40
+ sys.modules[__name__] = _LazyModule(
41
+ __name__,
42
+ globals()["__file__"],
43
+ _import_structure,
44
+ module_spec=__spec__,
45
+ )
46
+ for name, value in _dummy_objects.items():
47
+ setattr(sys.modules[__name__], name, value)
western_ai/europe/diffusers/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py ADDED
@@ -0,0 +1,960 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 TencentARC and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from dataclasses import dataclass
17
+ from typing import Any, Callable
18
+
19
+ import numpy as np
20
+ import PIL.Image
21
+ import torch
22
+ from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer
23
+
24
+ from ...image_processor import VaeImageProcessor
25
+ from ...loaders import FromSingleFileMixin, StableDiffusionLoraLoaderMixin, TextualInversionLoaderMixin
26
+ from ...models import AutoencoderKL, MultiAdapter, T2IAdapter, UNet2DConditionModel
27
+ from ...models.lora import adjust_lora_scale_text_encoder
28
+ from ...schedulers import KarrasDiffusionSchedulers
29
+ from ...utils import (
30
+ PIL_INTERPOLATION,
31
+ USE_PEFT_BACKEND,
32
+ BaseOutput,
33
+ deprecate,
34
+ is_torch_xla_available,
35
+ logging,
36
+ replace_example_docstring,
37
+ scale_lora_layers,
38
+ unscale_lora_layers,
39
+ )
40
+ from ...utils.torch_utils import randn_tensor
41
+ from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin
42
+ from ..stable_diffusion.safety_checker import StableDiffusionSafetyChecker
43
+
44
+
45
+ if is_torch_xla_available():
46
+ import torch_xla.core.xla_model as xm
47
+
48
+ XLA_AVAILABLE = True
49
+ else:
50
+ XLA_AVAILABLE = False
51
+
52
+
53
+ @dataclass
54
+ class StableDiffusionAdapterPipelineOutput(BaseOutput):
55
+ """
56
+ Args:
57
+ images (`list[PIL.Image.Image]` or `np.ndarray`)
58
+ list of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width,
59
+ num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline.
60
+ nsfw_content_detected (`list[bool]`)
61
+ list of flags denoting whether the corresponding generated image likely represents "not-safe-for-work"
62
+ (nsfw) content, or `None` if safety checking could not be performed.
63
+ """
64
+
65
+ images: list[PIL.Image.Image] | np.ndarray
66
+ nsfw_content_detected: list[bool] | None
67
+
68
+
69
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
70
+
71
+
72
+ EXAMPLE_DOC_STRING = """
73
+ Examples:
74
+ ```py
75
+ >>> from PIL import Image
76
+ >>> from diffusers.utils import load_image
77
+ >>> import torch
78
+ >>> from diffusers import StableDiffusionAdapterPipeline, T2IAdapter
79
+
80
+ >>> image = load_image(
81
+ ... "https://huggingface.co/datasets/diffusers/docs-images/resolve/main/t2i-adapter/color_ref.png"
82
+ ... )
83
+
84
+ >>> color_palette = image.resize((8, 8))
85
+ >>> color_palette = color_palette.resize((512, 512), resample=Image.Resampling.NEAREST)
86
+
87
+ >>> adapter = T2IAdapter.from_pretrained("TencentARC/t2iadapter_color_sd14v1", torch_dtype=torch.float16)
88
+ >>> pipe = StableDiffusionAdapterPipeline.from_pretrained(
89
+ ... "CompVis/stable-diffusion-v1-4",
90
+ ... adapter=adapter,
91
+ ... torch_dtype=torch.float16,
92
+ ... )
93
+
94
+ >>> pipe.to("cuda")
95
+
96
+ >>> out_image = pipe(
97
+ ... "At night, glowing cubes in front of the beach",
98
+ ... image=color_palette,
99
+ ... ).images[0]
100
+ ```
101
+ """
102
+
103
+
104
+ def _preprocess_adapter_image(image, height, width):
105
+ if isinstance(image, torch.Tensor):
106
+ return image
107
+ elif isinstance(image, PIL.Image.Image):
108
+ image = [image]
109
+
110
+ if isinstance(image[0], PIL.Image.Image):
111
+ image = [np.array(i.resize((width, height), resample=PIL_INTERPOLATION["lanczos"])) for i in image]
112
+ image = [
113
+ i[None, ..., None] if i.ndim == 2 else i[None, ...] for i in image
114
+ ] # expand [h, w] or [h, w, c] to [b, h, w, c]
115
+ image = np.concatenate(image, axis=0)
116
+ image = np.array(image).astype(np.float32) / 255.0
117
+ image = image.transpose(0, 3, 1, 2)
118
+ image = torch.from_numpy(image)
119
+ elif isinstance(image[0], torch.Tensor):
120
+ if image[0].ndim == 3:
121
+ image = torch.stack(image, dim=0)
122
+ elif image[0].ndim == 4:
123
+ image = torch.cat(image, dim=0)
124
+ else:
125
+ raise ValueError(
126
+ f"Invalid image tensor! Expecting image tensor with 3 or 4 dimension, but receive: {image[0].ndim}"
127
+ )
128
+ return image
129
+
130
+
131
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
132
+ def retrieve_timesteps(
133
+ scheduler,
134
+ num_inference_steps: int | None = None,
135
+ device: str | torch.device | None = None,
136
+ timesteps: list[int] | None = None,
137
+ sigmas: list[float] | None = None,
138
+ **kwargs,
139
+ ):
140
+ r"""
141
+ Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
142
+ custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
143
+
144
+ Args:
145
+ scheduler (`SchedulerMixin`):
146
+ The scheduler to get timesteps from.
147
+ num_inference_steps (`int`):
148
+ The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
149
+ must be `None`.
150
+ device (`str` or `torch.device`, *optional*):
151
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
152
+ timesteps (`list[int]`, *optional*):
153
+ Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
154
+ `num_inference_steps` and `sigmas` must be `None`.
155
+ sigmas (`list[float]`, *optional*):
156
+ Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
157
+ `num_inference_steps` and `timesteps` must be `None`.
158
+
159
+ Returns:
160
+ `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
161
+ second element is the number of inference steps.
162
+ """
163
+ if timesteps is not None and sigmas is not None:
164
+ raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
165
+ if timesteps is not None:
166
+ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
167
+ if not accepts_timesteps:
168
+ raise ValueError(
169
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
170
+ f" timestep schedules. Please check whether you are using the correct scheduler."
171
+ )
172
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
173
+ timesteps = scheduler.timesteps
174
+ num_inference_steps = len(timesteps)
175
+ elif sigmas is not None:
176
+ accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
177
+ if not accept_sigmas:
178
+ raise ValueError(
179
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
180
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
181
+ )
182
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
183
+ timesteps = scheduler.timesteps
184
+ num_inference_steps = len(timesteps)
185
+ else:
186
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
187
+ timesteps = scheduler.timesteps
188
+ return timesteps, num_inference_steps
189
+
190
+
191
+ class StableDiffusionAdapterPipeline(DiffusionPipeline, StableDiffusionMixin, FromSingleFileMixin):
192
+ r"""
193
+ Pipeline for text-to-image generation using Stable Diffusion augmented with T2I-Adapter
194
+ https://huggingface.co/papers/2302.08453
195
+
196
+ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
197
+ library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
198
+
199
+ Args:
200
+ adapter ([`T2IAdapter`] or [`MultiAdapter`] or `list[T2IAdapter]`):
201
+ Provides additional conditioning to the unet during the denoising process. If you set multiple Adapter as a
202
+ list, the outputs from each Adapter are added together to create one combined additional conditioning.
203
+ adapter_weights (`list[float]`, *optional*, defaults to None):
204
+ list of floats representing the weight which will be multiply to each adapter's output before adding them
205
+ together.
206
+ vae ([`AutoencoderKL`]):
207
+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
208
+ text_encoder ([`CLIPTextModel`]):
209
+ Frozen text-encoder. Stable Diffusion uses the text portion of
210
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
211
+ the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
212
+ tokenizer (`CLIPTokenizer`):
213
+ Tokenizer of class
214
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
215
+ unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.
216
+ scheduler ([`SchedulerMixin`]):
217
+ A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
218
+ [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
219
+ safety_checker ([`StableDiffusionSafetyChecker`]):
220
+ Classification module that estimates whether generated images could be considered offensive or harmful.
221
+ Please, refer to the [model card](https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5) for
222
+ details.
223
+ feature_extractor ([`CLIPImageProcessor`]):
224
+ Model that extracts features from generated images to be used as inputs for the `safety_checker`.
225
+ """
226
+
227
+ model_cpu_offload_seq = "text_encoder->adapter->unet->vae"
228
+ _optional_components = ["safety_checker", "feature_extractor"]
229
+
230
+ def __init__(
231
+ self,
232
+ vae: AutoencoderKL,
233
+ text_encoder: CLIPTextModel,
234
+ tokenizer: CLIPTokenizer,
235
+ unet: UNet2DConditionModel,
236
+ adapter: T2IAdapter | MultiAdapter | list[T2IAdapter],
237
+ scheduler: KarrasDiffusionSchedulers,
238
+ safety_checker: StableDiffusionSafetyChecker,
239
+ feature_extractor: CLIPImageProcessor,
240
+ requires_safety_checker: bool = True,
241
+ ):
242
+ super().__init__()
243
+
244
+ if safety_checker is None and requires_safety_checker:
245
+ logger.warning(
246
+ f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"
247
+ " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"
248
+ " results in services or applications open to the public. Both the diffusers team and Hugging Face"
249
+ " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"
250
+ " it only for use-cases that involve analyzing network behavior or auditing its results. For more"
251
+ " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."
252
+ )
253
+
254
+ if safety_checker is not None and feature_extractor is None:
255
+ raise ValueError(
256
+ "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"
257
+ " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."
258
+ )
259
+
260
+ if isinstance(adapter, (list, tuple)):
261
+ adapter = MultiAdapter(adapter)
262
+
263
+ self.register_modules(
264
+ vae=vae,
265
+ text_encoder=text_encoder,
266
+ tokenizer=tokenizer,
267
+ unet=unet,
268
+ adapter=adapter,
269
+ scheduler=scheduler,
270
+ safety_checker=safety_checker,
271
+ feature_extractor=feature_extractor,
272
+ )
273
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
274
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
275
+ self.register_to_config(requires_safety_checker=requires_safety_checker)
276
+
277
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt
278
+ def _encode_prompt(
279
+ self,
280
+ prompt,
281
+ device,
282
+ num_images_per_prompt,
283
+ do_classifier_free_guidance,
284
+ negative_prompt=None,
285
+ prompt_embeds: torch.Tensor | None = None,
286
+ negative_prompt_embeds: torch.Tensor | None = None,
287
+ lora_scale: float | None = None,
288
+ **kwargs,
289
+ ):
290
+ deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple."
291
+ deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False)
292
+
293
+ prompt_embeds_tuple = self.encode_prompt(
294
+ prompt=prompt,
295
+ device=device,
296
+ num_images_per_prompt=num_images_per_prompt,
297
+ do_classifier_free_guidance=do_classifier_free_guidance,
298
+ negative_prompt=negative_prompt,
299
+ prompt_embeds=prompt_embeds,
300
+ negative_prompt_embeds=negative_prompt_embeds,
301
+ lora_scale=lora_scale,
302
+ **kwargs,
303
+ )
304
+
305
+ # concatenate for backwards comp
306
+ prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]])
307
+
308
+ return prompt_embeds
309
+
310
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt
311
+ def encode_prompt(
312
+ self,
313
+ prompt,
314
+ device,
315
+ num_images_per_prompt,
316
+ do_classifier_free_guidance,
317
+ negative_prompt=None,
318
+ prompt_embeds: torch.Tensor | None = None,
319
+ negative_prompt_embeds: torch.Tensor | None = None,
320
+ lora_scale: float | None = None,
321
+ clip_skip: int | None = None,
322
+ ):
323
+ r"""
324
+ Encodes the prompt into text encoder hidden states.
325
+
326
+ Args:
327
+ prompt (`str` or `list[str]`, *optional*):
328
+ prompt to be encoded
329
+ device: (`torch.device`):
330
+ torch device
331
+ num_images_per_prompt (`int`):
332
+ number of images that should be generated per prompt
333
+ do_classifier_free_guidance (`bool`):
334
+ whether to use classifier free guidance or not
335
+ negative_prompt (`str` or `list[str]`, *optional*):
336
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
337
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
338
+ less than `1`).
339
+ prompt_embeds (`torch.Tensor`, *optional*):
340
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
341
+ provided, text embeddings will be generated from `prompt` input argument.
342
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
343
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
344
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
345
+ argument.
346
+ lora_scale (`float`, *optional*):
347
+ A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
348
+ clip_skip (`int`, *optional*):
349
+ Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
350
+ the output of the pre-final layer will be used for computing the prompt embeddings.
351
+ """
352
+ # set lora scale so that monkey patched LoRA
353
+ # function of text encoder can correctly access it
354
+ if lora_scale is not None and isinstance(self, StableDiffusionLoraLoaderMixin):
355
+ self._lora_scale = lora_scale
356
+
357
+ # dynamically adjust the LoRA scale
358
+ if not USE_PEFT_BACKEND:
359
+ adjust_lora_scale_text_encoder(self.text_encoder, lora_scale)
360
+ else:
361
+ scale_lora_layers(self.text_encoder, lora_scale)
362
+
363
+ if prompt is not None and isinstance(prompt, str):
364
+ batch_size = 1
365
+ elif prompt is not None and isinstance(prompt, list):
366
+ batch_size = len(prompt)
367
+ else:
368
+ batch_size = prompt_embeds.shape[0]
369
+
370
+ if prompt_embeds is None:
371
+ # textual inversion: process multi-vector tokens if necessary
372
+ if isinstance(self, TextualInversionLoaderMixin):
373
+ prompt = self.maybe_convert_prompt(prompt, self.tokenizer)
374
+
375
+ text_inputs = self.tokenizer(
376
+ prompt,
377
+ padding="max_length",
378
+ max_length=self.tokenizer.model_max_length,
379
+ truncation=True,
380
+ return_tensors="pt",
381
+ )
382
+ text_input_ids = text_inputs.input_ids
383
+ untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
384
+
385
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(
386
+ text_input_ids, untruncated_ids
387
+ ):
388
+ removed_text = self.tokenizer.batch_decode(
389
+ untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]
390
+ )
391
+ logger.warning(
392
+ "The following part of your input was truncated because CLIP can only handle sequences up to"
393
+ f" {self.tokenizer.model_max_length} tokens: {removed_text}"
394
+ )
395
+
396
+ if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:
397
+ attention_mask = text_inputs.attention_mask.to(device)
398
+ else:
399
+ attention_mask = None
400
+
401
+ if clip_skip is None:
402
+ prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask)
403
+ prompt_embeds = prompt_embeds[0]
404
+ else:
405
+ prompt_embeds = self.text_encoder(
406
+ text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True
407
+ )
408
+ # Access the `hidden_states` first, that contains a tuple of
409
+ # all the hidden states from the encoder layers. Then index into
410
+ # the tuple to access the hidden states from the desired layer.
411
+ prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)]
412
+ # We also need to apply the final LayerNorm here to not mess with the
413
+ # representations. The `last_hidden_states` that we typically use for
414
+ # obtaining the final prompt representations passes through the LayerNorm
415
+ # layer.
416
+ prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds)
417
+
418
+ if self.text_encoder is not None:
419
+ prompt_embeds_dtype = self.text_encoder.dtype
420
+ elif self.unet is not None:
421
+ prompt_embeds_dtype = self.unet.dtype
422
+ else:
423
+ prompt_embeds_dtype = prompt_embeds.dtype
424
+
425
+ prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
426
+
427
+ bs_embed, seq_len, _ = prompt_embeds.shape
428
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
429
+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
430
+ prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
431
+
432
+ # get unconditional embeddings for classifier free guidance
433
+ if do_classifier_free_guidance and negative_prompt_embeds is None:
434
+ uncond_tokens: list[str]
435
+ if negative_prompt is None:
436
+ uncond_tokens = [""] * batch_size
437
+ elif prompt is not None and type(prompt) is not type(negative_prompt):
438
+ raise TypeError(
439
+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
440
+ f" {type(prompt)}."
441
+ )
442
+ elif isinstance(negative_prompt, str):
443
+ uncond_tokens = [negative_prompt]
444
+ elif batch_size != len(negative_prompt):
445
+ raise ValueError(
446
+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
447
+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
448
+ " the batch size of `prompt`."
449
+ )
450
+ else:
451
+ uncond_tokens = negative_prompt
452
+
453
+ # textual inversion: process multi-vector tokens if necessary
454
+ if isinstance(self, TextualInversionLoaderMixin):
455
+ uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer)
456
+
457
+ max_length = prompt_embeds.shape[1]
458
+ uncond_input = self.tokenizer(
459
+ uncond_tokens,
460
+ padding="max_length",
461
+ max_length=max_length,
462
+ truncation=True,
463
+ return_tensors="pt",
464
+ )
465
+
466
+ if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:
467
+ attention_mask = uncond_input.attention_mask.to(device)
468
+ else:
469
+ attention_mask = None
470
+
471
+ negative_prompt_embeds = self.text_encoder(
472
+ uncond_input.input_ids.to(device),
473
+ attention_mask=attention_mask,
474
+ )
475
+ negative_prompt_embeds = negative_prompt_embeds[0]
476
+
477
+ if do_classifier_free_guidance:
478
+ # duplicate unconditional embeddings for each generation per prompt, using mps friendly method
479
+ seq_len = negative_prompt_embeds.shape[1]
480
+
481
+ negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
482
+
483
+ negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
484
+ negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
485
+
486
+ if self.text_encoder is not None:
487
+ if isinstance(self, StableDiffusionLoraLoaderMixin) and USE_PEFT_BACKEND:
488
+ # Retrieve the original scale by scaling back the LoRA layers
489
+ unscale_lora_layers(self.text_encoder, lora_scale)
490
+
491
+ return prompt_embeds, negative_prompt_embeds
492
+
493
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker
494
+ def run_safety_checker(self, image, device, dtype):
495
+ if self.safety_checker is None:
496
+ has_nsfw_concept = None
497
+ else:
498
+ if torch.is_tensor(image):
499
+ feature_extractor_input = self.image_processor.postprocess(image, output_type="pil")
500
+ else:
501
+ feature_extractor_input = self.image_processor.numpy_to_pil(image)
502
+ safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device)
503
+ image, has_nsfw_concept = self.safety_checker(
504
+ images=image, clip_input=safety_checker_input.pixel_values.to(dtype)
505
+ )
506
+ return image, has_nsfw_concept
507
+
508
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents
509
+ def decode_latents(self, latents):
510
+ deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead"
511
+ deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False)
512
+
513
+ latents = 1 / self.vae.config.scaling_factor * latents
514
+ image = self.vae.decode(latents, return_dict=False)[0]
515
+ image = (image / 2 + 0.5).clamp(0, 1)
516
+ # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16
517
+ image = image.cpu().permute(0, 2, 3, 1).float().numpy()
518
+ return image
519
+
520
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs
521
+ def prepare_extra_step_kwargs(self, generator, eta):
522
+ # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
523
+ # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
524
+ # eta corresponds to η in DDIM paper: https://huggingface.co/papers/2010.02502
525
+ # and should be between [0, 1]
526
+
527
+ accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
528
+ extra_step_kwargs = {}
529
+ if accepts_eta:
530
+ extra_step_kwargs["eta"] = eta
531
+
532
+ # check if the scheduler accepts generator
533
+ accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
534
+ if accepts_generator:
535
+ extra_step_kwargs["generator"] = generator
536
+ return extra_step_kwargs
537
+
538
+ def check_inputs(
539
+ self,
540
+ prompt,
541
+ height,
542
+ width,
543
+ callback_steps,
544
+ image,
545
+ negative_prompt=None,
546
+ prompt_embeds=None,
547
+ negative_prompt_embeds=None,
548
+ ):
549
+ if height % 8 != 0 or width % 8 != 0:
550
+ raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
551
+
552
+ if (callback_steps is None) or (
553
+ callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)
554
+ ):
555
+ raise ValueError(
556
+ f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
557
+ f" {type(callback_steps)}."
558
+ )
559
+
560
+ if prompt is not None and prompt_embeds is not None:
561
+ raise ValueError(
562
+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
563
+ " only forward one of the two."
564
+ )
565
+ elif prompt is None and prompt_embeds is None:
566
+ raise ValueError(
567
+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
568
+ )
569
+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
570
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
571
+
572
+ if negative_prompt is not None and negative_prompt_embeds is not None:
573
+ raise ValueError(
574
+ f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
575
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
576
+ )
577
+
578
+ if prompt_embeds is not None and negative_prompt_embeds is not None:
579
+ if prompt_embeds.shape != negative_prompt_embeds.shape:
580
+ raise ValueError(
581
+ "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"
582
+ f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"
583
+ f" {negative_prompt_embeds.shape}."
584
+ )
585
+
586
+ if isinstance(self.adapter, MultiAdapter):
587
+ if not isinstance(image, list):
588
+ raise ValueError(
589
+ "MultiAdapter is enabled, but `image` is not a list. Please pass a list of images to `image`."
590
+ )
591
+
592
+ if len(image) != len(self.adapter.adapters):
593
+ raise ValueError(
594
+ f"MultiAdapter requires passing the same number of images as adapters. Given {len(image)} images and {len(self.adapter.adapters)} adapters."
595
+ )
596
+
597
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents
598
+ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):
599
+ shape = (
600
+ batch_size,
601
+ num_channels_latents,
602
+ int(height) // self.vae_scale_factor,
603
+ int(width) // self.vae_scale_factor,
604
+ )
605
+ if isinstance(generator, list) and len(generator) != batch_size:
606
+ raise ValueError(
607
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
608
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
609
+ )
610
+
611
+ if latents is None:
612
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
613
+ else:
614
+ latents = latents.to(device)
615
+
616
+ # scale the initial noise by the standard deviation required by the scheduler
617
+ latents = latents * self.scheduler.init_noise_sigma
618
+ return latents
619
+
620
+ def _default_height_width(self, height, width, image):
621
+ # NOTE: It is possible that a list of images have different
622
+ # dimensions for each image, so just checking the first image
623
+ # is not _exactly_ correct, but it is simple.
624
+ while isinstance(image, list):
625
+ image = image[0]
626
+
627
+ if height is None:
628
+ if isinstance(image, PIL.Image.Image):
629
+ height = image.height
630
+ elif isinstance(image, torch.Tensor):
631
+ height = image.shape[-2]
632
+
633
+ # round down to nearest multiple of `self.adapter.downscale_factor`
634
+ height = (height // self.adapter.downscale_factor) * self.adapter.downscale_factor
635
+
636
+ if width is None:
637
+ if isinstance(image, PIL.Image.Image):
638
+ width = image.width
639
+ elif isinstance(image, torch.Tensor):
640
+ width = image.shape[-1]
641
+
642
+ # round down to nearest multiple of `self.adapter.downscale_factor`
643
+ width = (width // self.adapter.downscale_factor) * self.adapter.downscale_factor
644
+
645
+ return height, width
646
+
647
+ # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding
648
+ def get_guidance_scale_embedding(
649
+ self, w: torch.Tensor, embedding_dim: int = 512, dtype: torch.dtype = torch.float32
650
+ ) -> torch.Tensor:
651
+ """
652
+ See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298
653
+
654
+ Args:
655
+ w (`torch.Tensor`):
656
+ Generate embedding vectors with a specified guidance scale to subsequently enrich timestep embeddings.
657
+ embedding_dim (`int`, *optional*, defaults to 512):
658
+ Dimension of the embeddings to generate.
659
+ dtype (`torch.dtype`, *optional*, defaults to `torch.float32`):
660
+ Data type of the generated embeddings.
661
+
662
+ Returns:
663
+ `torch.Tensor`: Embedding vectors with shape `(len(w), embedding_dim)`.
664
+ """
665
+ assert len(w.shape) == 1
666
+ w = w * 1000.0
667
+
668
+ half_dim = embedding_dim // 2
669
+ emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1)
670
+ emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb)
671
+ emb = w.to(dtype)[:, None] * emb[None, :]
672
+ emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)
673
+ if embedding_dim % 2 == 1: # zero pad
674
+ emb = torch.nn.functional.pad(emb, (0, 1))
675
+ assert emb.shape == (w.shape[0], embedding_dim)
676
+ return emb
677
+
678
+ @property
679
+ def guidance_scale(self):
680
+ return self._guidance_scale
681
+
682
+ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
683
+ # of the Imagen paper: https://huggingface.co/papers/2205.11487 . `guidance_scale = 1`
684
+ # corresponds to doing no classifier free guidance.
685
+ @property
686
+ def do_classifier_free_guidance(self):
687
+ return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None
688
+
689
+ @torch.no_grad()
690
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
691
+ def __call__(
692
+ self,
693
+ prompt: str | list[str] = None,
694
+ image: torch.Tensor | PIL.Image.Image | list[PIL.Image.Image] = None,
695
+ height: int | None = None,
696
+ width: int | None = None,
697
+ num_inference_steps: int = 50,
698
+ timesteps: list[int] = None,
699
+ sigmas: list[float] = None,
700
+ guidance_scale: float = 7.5,
701
+ negative_prompt: str | list[str] | None = None,
702
+ num_images_per_prompt: int | None = 1,
703
+ eta: float = 0.0,
704
+ generator: torch.Generator | list[torch.Generator] | None = None,
705
+ latents: torch.Tensor | None = None,
706
+ prompt_embeds: torch.Tensor | None = None,
707
+ negative_prompt_embeds: torch.Tensor | None = None,
708
+ output_type: str | None = "pil",
709
+ return_dict: bool = True,
710
+ callback: Callable[[int, int, torch.Tensor], None] | None = None,
711
+ callback_steps: int = 1,
712
+ cross_attention_kwargs: dict[str, Any] | None = None,
713
+ adapter_conditioning_scale: float | list[float] = 1.0,
714
+ clip_skip: int | None = None,
715
+ ):
716
+ r"""
717
+ Function invoked when calling the pipeline for generation.
718
+
719
+ Args:
720
+ prompt (`str` or `list[str]`, *optional*):
721
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
722
+ instead.
723
+ image (`torch.Tensor`, `PIL.Image.Image`, `list[torch.Tensor]` or `list[PIL.Image.Image]` or `list[list[PIL.Image.Image]]`):
724
+ The Adapter input condition. Adapter uses this input condition to generate guidance to Unet. If the
725
+ type is specified as `torch.Tensor`, it is passed to Adapter as is. PIL.Image.Image` can also be
726
+ accepted as an image. The control image is automatically resized to fit the output image.
727
+ height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
728
+ The height in pixels of the generated image.
729
+ width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
730
+ The width in pixels of the generated image.
731
+ num_inference_steps (`int`, *optional*, defaults to 50):
732
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
733
+ expense of slower inference.
734
+ timesteps (`list[int]`, *optional*):
735
+ Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument
736
+ in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is
737
+ passed will be used. Must be in descending order.
738
+ sigmas (`list[float]`, *optional*):
739
+ Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
740
+ their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
741
+ will be used.
742
+ guidance_scale (`float`, *optional*, defaults to 7.5):
743
+ Guidance scale as defined in [Classifier-Free Diffusion
744
+ Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2.
745
+ of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
746
+ `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to
747
+ the text `prompt`, usually at the expense of lower image quality.
748
+ negative_prompt (`str` or `list[str]`, *optional*):
749
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
750
+ `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead.
751
+ Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`).
752
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
753
+ The number of images to generate per prompt.
754
+ eta (`float`, *optional*, defaults to 0.0):
755
+ Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only
756
+ applies to [`schedulers.DDIMScheduler`], will be ignored for others.
757
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
758
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
759
+ to make generation deterministic.
760
+ latents (`torch.Tensor`, *optional*):
761
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
762
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
763
+ tensor will be generated by sampling using the supplied random `generator`.
764
+ prompt_embeds (`torch.Tensor`, *optional*):
765
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
766
+ provided, text embeddings will be generated from `prompt` input argument.
767
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
768
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
769
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
770
+ argument.
771
+ output_type (`str`, *optional*, defaults to `"pil"`):
772
+ The output format of the generate image. Choose between
773
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
774
+ return_dict (`bool`, *optional*, defaults to `True`):
775
+ Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionAdapterPipelineOutput`] instead
776
+ of a plain tuple.
777
+ callback (`Callable`, *optional*):
778
+ A function that will be called every `callback_steps` steps during inference. The function will be
779
+ called with the following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`.
780
+ callback_steps (`int`, *optional*, defaults to 1):
781
+ The frequency at which the `callback` function will be called. If not specified, the callback will be
782
+ called at every step.
783
+ cross_attention_kwargs (`dict`, *optional*):
784
+ A kwargs dictionary that if specified is passed along to the `AttnProcessor` as defined under
785
+ `self.processor` in
786
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
787
+ adapter_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0):
788
+ The outputs of the adapter are multiplied by `adapter_conditioning_scale` before they are added to the
789
+ residual in the original unet. If multiple adapters are specified in init, you can set the
790
+ corresponding scale as a list.
791
+ clip_skip (`int`, *optional*):
792
+ Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
793
+ the output of the pre-final layer will be used for computing the prompt embeddings.
794
+ Examples:
795
+
796
+ Returns:
797
+ [`~pipelines.stable_diffusion.StableDiffusionAdapterPipelineOutput`] or `tuple`:
798
+ [`~pipelines.stable_diffusion.StableDiffusionAdapterPipelineOutput`] if `return_dict` is True, otherwise a
799
+ `tuple. When returning a tuple, the first element is a list with the generated images, and the second
800
+ element is a list of `bool`s denoting whether the corresponding generated image likely represents
801
+ "not-safe-for-work" (nsfw) content, according to the `safety_checker`.
802
+ """
803
+ # 0. Default height and width to unet
804
+ height, width = self._default_height_width(height, width, image)
805
+ device = self._execution_device
806
+
807
+ # 1. Check inputs. Raise error if not correct
808
+ self.check_inputs(
809
+ prompt, height, width, callback_steps, image, negative_prompt, prompt_embeds, negative_prompt_embeds
810
+ )
811
+
812
+ self._guidance_scale = guidance_scale
813
+
814
+ if isinstance(self.adapter, MultiAdapter):
815
+ adapter_input = []
816
+
817
+ for one_image in image:
818
+ one_image = _preprocess_adapter_image(one_image, height, width)
819
+ one_image = one_image.to(device=device, dtype=self.adapter.dtype)
820
+ adapter_input.append(one_image)
821
+ else:
822
+ adapter_input = _preprocess_adapter_image(image, height, width)
823
+ adapter_input = adapter_input.to(device=device, dtype=self.adapter.dtype)
824
+
825
+ # 2. Define call parameters
826
+ if prompt is not None and isinstance(prompt, str):
827
+ batch_size = 1
828
+ elif prompt is not None and isinstance(prompt, list):
829
+ batch_size = len(prompt)
830
+ else:
831
+ batch_size = prompt_embeds.shape[0]
832
+
833
+ # 3. Encode input prompt
834
+ prompt_embeds, negative_prompt_embeds = self.encode_prompt(
835
+ prompt,
836
+ device,
837
+ num_images_per_prompt,
838
+ self.do_classifier_free_guidance,
839
+ negative_prompt,
840
+ prompt_embeds=prompt_embeds,
841
+ negative_prompt_embeds=negative_prompt_embeds,
842
+ clip_skip=clip_skip,
843
+ )
844
+ # For classifier free guidance, we need to do two forward passes.
845
+ # Here we concatenate the unconditional and text embeddings into a single batch
846
+ # to avoid doing two forward passes
847
+ if self.do_classifier_free_guidance:
848
+ prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])
849
+
850
+ # 4. Prepare timesteps
851
+ if XLA_AVAILABLE:
852
+ timestep_device = "cpu"
853
+ else:
854
+ timestep_device = device
855
+ timesteps, num_inference_steps = retrieve_timesteps(
856
+ self.scheduler, num_inference_steps, timestep_device, timesteps, sigmas
857
+ )
858
+
859
+ # 5. Prepare latent variables
860
+ num_channels_latents = self.unet.config.in_channels
861
+ latents = self.prepare_latents(
862
+ batch_size * num_images_per_prompt,
863
+ num_channels_latents,
864
+ height,
865
+ width,
866
+ prompt_embeds.dtype,
867
+ device,
868
+ generator,
869
+ latents,
870
+ )
871
+
872
+ # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
873
+ extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
874
+
875
+ # 6.5 Optionally get Guidance Scale Embedding
876
+ timestep_cond = None
877
+ if self.unet.config.time_cond_proj_dim is not None:
878
+ guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)
879
+ timestep_cond = self.get_guidance_scale_embedding(
880
+ guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim
881
+ ).to(device=device, dtype=latents.dtype)
882
+
883
+ # 7. Denoising loop
884
+ if isinstance(self.adapter, MultiAdapter):
885
+ adapter_state = self.adapter(adapter_input, adapter_conditioning_scale)
886
+ for k, v in enumerate(adapter_state):
887
+ adapter_state[k] = v
888
+ else:
889
+ adapter_state = self.adapter(adapter_input)
890
+ for k, v in enumerate(adapter_state):
891
+ adapter_state[k] = v * adapter_conditioning_scale
892
+ if num_images_per_prompt > 1:
893
+ for k, v in enumerate(adapter_state):
894
+ adapter_state[k] = v.repeat(num_images_per_prompt, 1, 1, 1)
895
+ if self.do_classifier_free_guidance:
896
+ for k, v in enumerate(adapter_state):
897
+ adapter_state[k] = torch.cat([v] * 2, dim=0)
898
+
899
+ num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
900
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
901
+ for i, t in enumerate(timesteps):
902
+ # expand the latents if we are doing classifier free guidance
903
+ latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
904
+ latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
905
+
906
+ # predict the noise residual
907
+ noise_pred = self.unet(
908
+ latent_model_input,
909
+ t,
910
+ encoder_hidden_states=prompt_embeds,
911
+ timestep_cond=timestep_cond,
912
+ cross_attention_kwargs=cross_attention_kwargs,
913
+ down_intrablock_additional_residuals=[state.clone() for state in adapter_state],
914
+ return_dict=False,
915
+ )[0]
916
+
917
+ # perform guidance
918
+ if self.do_classifier_free_guidance:
919
+ noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
920
+ noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
921
+
922
+ # compute the previous noisy sample x_t -> x_t-1
923
+ latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample
924
+
925
+ # call the callback, if provided
926
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
927
+ progress_bar.update()
928
+ if callback is not None and i % callback_steps == 0:
929
+ step_idx = i // getattr(self.scheduler, "order", 1)
930
+ callback(step_idx, t, latents)
931
+
932
+ if XLA_AVAILABLE:
933
+ xm.mark_step()
934
+
935
+ if output_type == "latent":
936
+ image = latents
937
+ has_nsfw_concept = None
938
+ elif output_type == "pil":
939
+ # 8. Post-processing
940
+ image = self.decode_latents(latents)
941
+
942
+ # 9. Run safety checker
943
+ image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)
944
+
945
+ # 10. Convert to PIL
946
+ image = self.numpy_to_pil(image)
947
+ else:
948
+ # 8. Post-processing
949
+ image = self.decode_latents(latents)
950
+
951
+ # 9. Run safety checker
952
+ image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)
953
+
954
+ # Offload all models
955
+ self.maybe_free_model_hooks()
956
+
957
+ if not return_dict:
958
+ return (image, has_nsfw_concept)
959
+
960
+ return StableDiffusionAdapterPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)
western_ai/europe/diffusers/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py ADDED
@@ -0,0 +1,1303 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 TencentARC and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from typing import Any, Callable
17
+
18
+ import numpy as np
19
+ import PIL.Image
20
+ import torch
21
+ from transformers import (
22
+ CLIPImageProcessor,
23
+ CLIPTextModel,
24
+ CLIPTextModelWithProjection,
25
+ CLIPTokenizer,
26
+ CLIPVisionModelWithProjection,
27
+ )
28
+
29
+ from ...image_processor import PipelineImageInput, VaeImageProcessor
30
+ from ...loaders import (
31
+ FromSingleFileMixin,
32
+ IPAdapterMixin,
33
+ StableDiffusionXLLoraLoaderMixin,
34
+ TextualInversionLoaderMixin,
35
+ )
36
+ from ...models import AutoencoderKL, ImageProjection, MultiAdapter, T2IAdapter, UNet2DConditionModel
37
+ from ...models.lora import adjust_lora_scale_text_encoder
38
+ from ...schedulers import KarrasDiffusionSchedulers
39
+ from ...utils import (
40
+ PIL_INTERPOLATION,
41
+ USE_PEFT_BACKEND,
42
+ deprecate,
43
+ is_torch_xla_available,
44
+ logging,
45
+ replace_example_docstring,
46
+ scale_lora_layers,
47
+ unscale_lora_layers,
48
+ )
49
+ from ...utils.torch_utils import randn_tensor
50
+ from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin
51
+ from ..stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput
52
+
53
+
54
+ if is_torch_xla_available():
55
+ import torch_xla.core.xla_model as xm
56
+
57
+ XLA_AVAILABLE = True
58
+ else:
59
+ XLA_AVAILABLE = False
60
+
61
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
62
+
63
+
64
+ EXAMPLE_DOC_STRING = """
65
+ Examples:
66
+ ```py
67
+ >>> import torch
68
+ >>> from diffusers import T2IAdapter, StableDiffusionXLAdapterPipeline, DDPMScheduler
69
+ >>> from diffusers.utils import load_image
70
+
71
+ >>> sketch_image = load_image("https://huggingface.co/Adapter/t2iadapter/resolve/main/sketch.png").convert("L")
72
+
73
+ >>> model_id = "stabilityai/stable-diffusion-xl-base-1.0"
74
+
75
+ >>> adapter = T2IAdapter.from_pretrained(
76
+ ... "Adapter/t2iadapter",
77
+ ... subfolder="sketch_sdxl_1.0",
78
+ ... torch_dtype=torch.float16,
79
+ ... adapter_type="full_adapter_xl",
80
+ ... )
81
+ >>> scheduler = DDPMScheduler.from_pretrained(model_id, subfolder="scheduler")
82
+
83
+ >>> pipe = StableDiffusionXLAdapterPipeline.from_pretrained(
84
+ ... model_id, adapter=adapter, torch_dtype=torch.float16, variant="fp16", scheduler=scheduler
85
+ ... ).to("cuda")
86
+
87
+ >>> generator = torch.manual_seed(42)
88
+ >>> sketch_image_out = pipe(
89
+ ... prompt="a photo of a dog in real world, high quality",
90
+ ... negative_prompt="extra digit, fewer digits, cropped, worst quality, low quality",
91
+ ... image=sketch_image,
92
+ ... generator=generator,
93
+ ... guidance_scale=7.5,
94
+ ... ).images[0]
95
+ ```
96
+ """
97
+
98
+
99
+ def _preprocess_adapter_image(image, height, width):
100
+ if isinstance(image, torch.Tensor):
101
+ return image
102
+ elif isinstance(image, PIL.Image.Image):
103
+ image = [image]
104
+
105
+ if isinstance(image[0], PIL.Image.Image):
106
+ image = [np.array(i.resize((width, height), resample=PIL_INTERPOLATION["lanczos"])) for i in image]
107
+ image = [
108
+ i[None, ..., None] if i.ndim == 2 else i[None, ...] for i in image
109
+ ] # expand [h, w] or [h, w, c] to [b, h, w, c]
110
+ image = np.concatenate(image, axis=0)
111
+ image = np.array(image).astype(np.float32) / 255.0
112
+ image = image.transpose(0, 3, 1, 2)
113
+ image = torch.from_numpy(image)
114
+ elif isinstance(image[0], torch.Tensor):
115
+ if image[0].ndim == 3:
116
+ image = torch.stack(image, dim=0)
117
+ elif image[0].ndim == 4:
118
+ image = torch.cat(image, dim=0)
119
+ else:
120
+ raise ValueError(
121
+ f"Invalid image tensor! Expecting image tensor with 3 or 4 dimension, but receive: {image[0].ndim}"
122
+ )
123
+ return image
124
+
125
+
126
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg
127
+ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):
128
+ r"""
129
+ Rescales `noise_cfg` tensor based on `guidance_rescale` to improve image quality and fix overexposure. Based on
130
+ Section 3.4 from [Common Diffusion Noise Schedules and Sample Steps are
131
+ Flawed](https://huggingface.co/papers/2305.08891).
132
+
133
+ Args:
134
+ noise_cfg (`torch.Tensor`):
135
+ The predicted noise tensor for the guided diffusion process.
136
+ noise_pred_text (`torch.Tensor`):
137
+ The predicted noise tensor for the text-guided diffusion process.
138
+ guidance_rescale (`float`, *optional*, defaults to 0.0):
139
+ A rescale factor applied to the noise predictions.
140
+
141
+ Returns:
142
+ noise_cfg (`torch.Tensor`): The rescaled noise prediction tensor.
143
+ """
144
+ std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)
145
+ std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)
146
+ # rescale the results from guidance (fixes overexposure)
147
+ noise_pred_rescaled = noise_cfg * (std_text / std_cfg)
148
+ # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images
149
+ noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg
150
+ return noise_cfg
151
+
152
+
153
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
154
+ def retrieve_timesteps(
155
+ scheduler,
156
+ num_inference_steps: int | None = None,
157
+ device: str | torch.device | None = None,
158
+ timesteps: list[int] | None = None,
159
+ sigmas: list[float] | None = None,
160
+ **kwargs,
161
+ ):
162
+ r"""
163
+ Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
164
+ custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
165
+
166
+ Args:
167
+ scheduler (`SchedulerMixin`):
168
+ The scheduler to get timesteps from.
169
+ num_inference_steps (`int`):
170
+ The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
171
+ must be `None`.
172
+ device (`str` or `torch.device`, *optional*):
173
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
174
+ timesteps (`list[int]`, *optional*):
175
+ Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
176
+ `num_inference_steps` and `sigmas` must be `None`.
177
+ sigmas (`list[float]`, *optional*):
178
+ Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
179
+ `num_inference_steps` and `timesteps` must be `None`.
180
+
181
+ Returns:
182
+ `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
183
+ second element is the number of inference steps.
184
+ """
185
+ if timesteps is not None and sigmas is not None:
186
+ raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
187
+ if timesteps is not None:
188
+ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
189
+ if not accepts_timesteps:
190
+ raise ValueError(
191
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
192
+ f" timestep schedules. Please check whether you are using the correct scheduler."
193
+ )
194
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
195
+ timesteps = scheduler.timesteps
196
+ num_inference_steps = len(timesteps)
197
+ elif sigmas is not None:
198
+ accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
199
+ if not accept_sigmas:
200
+ raise ValueError(
201
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
202
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
203
+ )
204
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
205
+ timesteps = scheduler.timesteps
206
+ num_inference_steps = len(timesteps)
207
+ else:
208
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
209
+ timesteps = scheduler.timesteps
210
+ return timesteps, num_inference_steps
211
+
212
+
213
+ class StableDiffusionXLAdapterPipeline(
214
+ DiffusionPipeline,
215
+ StableDiffusionMixin,
216
+ TextualInversionLoaderMixin,
217
+ StableDiffusionXLLoraLoaderMixin,
218
+ IPAdapterMixin,
219
+ FromSingleFileMixin,
220
+ ):
221
+ r"""
222
+ Pipeline for text-to-image generation using Stable Diffusion augmented with T2I-Adapter
223
+ https://huggingface.co/papers/2302.08453
224
+
225
+ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
226
+ library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
227
+
228
+ The pipeline also inherits the following loading methods:
229
+ - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings
230
+ - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files
231
+ - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights
232
+ - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights
233
+ - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters
234
+
235
+ Args:
236
+ adapter ([`T2IAdapter`] or [`MultiAdapter`] or `list[T2IAdapter]`):
237
+ Provides additional conditioning to the unet during the denoising process. If you set multiple Adapter as a
238
+ list, the outputs from each Adapter are added together to create one combined additional conditioning.
239
+ adapter_weights (`list[float]`, *optional*, defaults to None):
240
+ list of floats representing the weight which will be multiply to each adapter's output before adding them
241
+ together.
242
+ vae ([`AutoencoderKL`]):
243
+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
244
+ text_encoder ([`CLIPTextModel`]):
245
+ Frozen text-encoder. Stable Diffusion uses the text portion of
246
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
247
+ the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
248
+ tokenizer (`CLIPTokenizer`):
249
+ Tokenizer of class
250
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
251
+ unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.
252
+ scheduler ([`SchedulerMixin`]):
253
+ A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
254
+ [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
255
+ safety_checker ([`StableDiffusionSafetyChecker`]):
256
+ Classification module that estimates whether generated images could be considered offensive or harmful.
257
+ Please, refer to the [model card](https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5) for
258
+ details.
259
+ feature_extractor ([`CLIPImageProcessor`]):
260
+ Model that extracts features from generated images to be used as inputs for the `safety_checker`.
261
+ """
262
+
263
+ model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->unet->vae"
264
+ _optional_components = [
265
+ "tokenizer",
266
+ "tokenizer_2",
267
+ "text_encoder",
268
+ "text_encoder_2",
269
+ "feature_extractor",
270
+ "image_encoder",
271
+ ]
272
+
273
+ def __init__(
274
+ self,
275
+ vae: AutoencoderKL,
276
+ text_encoder: CLIPTextModel,
277
+ text_encoder_2: CLIPTextModelWithProjection,
278
+ tokenizer: CLIPTokenizer,
279
+ tokenizer_2: CLIPTokenizer,
280
+ unet: UNet2DConditionModel,
281
+ adapter: T2IAdapter | MultiAdapter | list[T2IAdapter],
282
+ scheduler: KarrasDiffusionSchedulers,
283
+ force_zeros_for_empty_prompt: bool = True,
284
+ feature_extractor: CLIPImageProcessor = None,
285
+ image_encoder: CLIPVisionModelWithProjection = None,
286
+ ):
287
+ super().__init__()
288
+
289
+ self.register_modules(
290
+ vae=vae,
291
+ text_encoder=text_encoder,
292
+ text_encoder_2=text_encoder_2,
293
+ tokenizer=tokenizer,
294
+ tokenizer_2=tokenizer_2,
295
+ unet=unet,
296
+ adapter=adapter,
297
+ scheduler=scheduler,
298
+ feature_extractor=feature_extractor,
299
+ image_encoder=image_encoder,
300
+ )
301
+ self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt)
302
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
303
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
304
+ self.default_sample_size = (
305
+ self.unet.config.sample_size
306
+ if hasattr(self, "unet") and self.unet is not None and hasattr(self.unet.config, "sample_size")
307
+ else 128
308
+ )
309
+
310
+ # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.encode_prompt
311
+ def encode_prompt(
312
+ self,
313
+ prompt: str,
314
+ prompt_2: str | None = None,
315
+ device: torch.device | None = None,
316
+ num_images_per_prompt: int = 1,
317
+ do_classifier_free_guidance: bool = True,
318
+ negative_prompt: str | None = None,
319
+ negative_prompt_2: str | None = None,
320
+ prompt_embeds: torch.Tensor | None = None,
321
+ negative_prompt_embeds: torch.Tensor | None = None,
322
+ pooled_prompt_embeds: torch.Tensor | None = None,
323
+ negative_pooled_prompt_embeds: torch.Tensor | None = None,
324
+ lora_scale: float | None = None,
325
+ clip_skip: int | None = None,
326
+ ):
327
+ r"""
328
+ Encodes the prompt into text encoder hidden states.
329
+
330
+ Args:
331
+ prompt (`str` or `list[str]`, *optional*):
332
+ prompt to be encoded
333
+ prompt_2 (`str` or `list[str]`, *optional*):
334
+ The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
335
+ used in both text-encoders
336
+ device: (`torch.device`):
337
+ torch device
338
+ num_images_per_prompt (`int`):
339
+ number of images that should be generated per prompt
340
+ do_classifier_free_guidance (`bool`):
341
+ whether to use classifier free guidance or not
342
+ negative_prompt (`str` or `list[str]`, *optional*):
343
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
344
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
345
+ less than `1`).
346
+ negative_prompt_2 (`str` or `list[str]`, *optional*):
347
+ The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
348
+ `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
349
+ prompt_embeds (`torch.Tensor`, *optional*):
350
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
351
+ provided, text embeddings will be generated from `prompt` input argument.
352
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
353
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
354
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
355
+ argument.
356
+ pooled_prompt_embeds (`torch.Tensor`, *optional*):
357
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
358
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
359
+ negative_pooled_prompt_embeds (`torch.Tensor`, *optional*):
360
+ Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
361
+ weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
362
+ input argument.
363
+ lora_scale (`float`, *optional*):
364
+ A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
365
+ clip_skip (`int`, *optional*):
366
+ Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
367
+ the output of the pre-final layer will be used for computing the prompt embeddings.
368
+ """
369
+ device = device or self._execution_device
370
+
371
+ # set lora scale so that monkey patched LoRA
372
+ # function of text encoder can correctly access it
373
+ if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin):
374
+ self._lora_scale = lora_scale
375
+
376
+ # dynamically adjust the LoRA scale
377
+ if self.text_encoder is not None:
378
+ if not USE_PEFT_BACKEND:
379
+ adjust_lora_scale_text_encoder(self.text_encoder, lora_scale)
380
+ else:
381
+ scale_lora_layers(self.text_encoder, lora_scale)
382
+
383
+ if self.text_encoder_2 is not None:
384
+ if not USE_PEFT_BACKEND:
385
+ adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale)
386
+ else:
387
+ scale_lora_layers(self.text_encoder_2, lora_scale)
388
+
389
+ prompt = [prompt] if isinstance(prompt, str) else prompt
390
+
391
+ if prompt is not None:
392
+ batch_size = len(prompt)
393
+ else:
394
+ batch_size = prompt_embeds.shape[0]
395
+
396
+ # Define tokenizers and text encoders
397
+ tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2]
398
+ text_encoders = (
399
+ [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2]
400
+ )
401
+
402
+ if prompt_embeds is None:
403
+ prompt_2 = prompt_2 or prompt
404
+ prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2
405
+
406
+ # textual inversion: process multi-vector tokens if necessary
407
+ prompt_embeds_list = []
408
+ prompts = [prompt, prompt_2]
409
+ for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders):
410
+ if isinstance(self, TextualInversionLoaderMixin):
411
+ prompt = self.maybe_convert_prompt(prompt, tokenizer)
412
+
413
+ text_inputs = tokenizer(
414
+ prompt,
415
+ padding="max_length",
416
+ max_length=tokenizer.model_max_length,
417
+ truncation=True,
418
+ return_tensors="pt",
419
+ )
420
+
421
+ text_input_ids = text_inputs.input_ids
422
+ untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
423
+
424
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(
425
+ text_input_ids, untruncated_ids
426
+ ):
427
+ removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1])
428
+ logger.warning(
429
+ "The following part of your input was truncated because CLIP can only handle sequences up to"
430
+ f" {tokenizer.model_max_length} tokens: {removed_text}"
431
+ )
432
+
433
+ prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True)
434
+
435
+ # We are only ALWAYS interested in the pooled output of the final text encoder
436
+ if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2:
437
+ pooled_prompt_embeds = prompt_embeds[0]
438
+
439
+ if clip_skip is None:
440
+ prompt_embeds = prompt_embeds.hidden_states[-2]
441
+ else:
442
+ # "2" because SDXL always indexes from the penultimate layer.
443
+ prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)]
444
+
445
+ prompt_embeds_list.append(prompt_embeds)
446
+
447
+ prompt_embeds = torch.concat(prompt_embeds_list, dim=-1)
448
+
449
+ # get unconditional embeddings for classifier free guidance
450
+ zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt
451
+ if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt:
452
+ negative_prompt_embeds = torch.zeros_like(prompt_embeds)
453
+ negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds)
454
+ elif do_classifier_free_guidance and negative_prompt_embeds is None:
455
+ negative_prompt = negative_prompt or ""
456
+ negative_prompt_2 = negative_prompt_2 or negative_prompt
457
+
458
+ # normalize str to list
459
+ negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
460
+ negative_prompt_2 = (
461
+ batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2
462
+ )
463
+
464
+ uncond_tokens: list[str]
465
+ if prompt is not None and type(prompt) is not type(negative_prompt):
466
+ raise TypeError(
467
+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
468
+ f" {type(prompt)}."
469
+ )
470
+ elif batch_size != len(negative_prompt):
471
+ raise ValueError(
472
+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
473
+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
474
+ " the batch size of `prompt`."
475
+ )
476
+ else:
477
+ uncond_tokens = [negative_prompt, negative_prompt_2]
478
+
479
+ negative_prompt_embeds_list = []
480
+ for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders):
481
+ if isinstance(self, TextualInversionLoaderMixin):
482
+ negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer)
483
+
484
+ max_length = prompt_embeds.shape[1]
485
+ uncond_input = tokenizer(
486
+ negative_prompt,
487
+ padding="max_length",
488
+ max_length=max_length,
489
+ truncation=True,
490
+ return_tensors="pt",
491
+ )
492
+
493
+ negative_prompt_embeds = text_encoder(
494
+ uncond_input.input_ids.to(device),
495
+ output_hidden_states=True,
496
+ )
497
+
498
+ # We are only ALWAYS interested in the pooled output of the final text encoder
499
+ if negative_pooled_prompt_embeds is None and negative_prompt_embeds[0].ndim == 2:
500
+ negative_pooled_prompt_embeds = negative_prompt_embeds[0]
501
+ negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2]
502
+
503
+ negative_prompt_embeds_list.append(negative_prompt_embeds)
504
+
505
+ negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1)
506
+
507
+ if self.text_encoder_2 is not None:
508
+ prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)
509
+ else:
510
+ prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device)
511
+
512
+ bs_embed, seq_len, _ = prompt_embeds.shape
513
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
514
+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
515
+ prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
516
+
517
+ if do_classifier_free_guidance:
518
+ # duplicate unconditional embeddings for each generation per prompt, using mps friendly method
519
+ seq_len = negative_prompt_embeds.shape[1]
520
+
521
+ if self.text_encoder_2 is not None:
522
+ negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)
523
+ else:
524
+ negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device)
525
+
526
+ negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
527
+ negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
528
+
529
+ pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
530
+ bs_embed * num_images_per_prompt, -1
531
+ )
532
+ if do_classifier_free_guidance:
533
+ negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
534
+ bs_embed * num_images_per_prompt, -1
535
+ )
536
+
537
+ if self.text_encoder is not None:
538
+ if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND:
539
+ # Retrieve the original scale by scaling back the LoRA layers
540
+ unscale_lora_layers(self.text_encoder, lora_scale)
541
+
542
+ if self.text_encoder_2 is not None:
543
+ if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND:
544
+ # Retrieve the original scale by scaling back the LoRA layers
545
+ unscale_lora_layers(self.text_encoder_2, lora_scale)
546
+
547
+ return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds
548
+
549
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image
550
+ def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None):
551
+ dtype = next(self.image_encoder.parameters()).dtype
552
+
553
+ if not isinstance(image, torch.Tensor):
554
+ image = self.feature_extractor(image, return_tensors="pt").pixel_values
555
+
556
+ image = image.to(device=device, dtype=dtype)
557
+ if output_hidden_states:
558
+ image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2]
559
+ image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0)
560
+ uncond_image_enc_hidden_states = self.image_encoder(
561
+ torch.zeros_like(image), output_hidden_states=True
562
+ ).hidden_states[-2]
563
+ uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave(
564
+ num_images_per_prompt, dim=0
565
+ )
566
+ return image_enc_hidden_states, uncond_image_enc_hidden_states
567
+ else:
568
+ image_embeds = self.image_encoder(image).image_embeds
569
+ image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0)
570
+ uncond_image_embeds = torch.zeros_like(image_embeds)
571
+
572
+ return image_embeds, uncond_image_embeds
573
+
574
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds
575
+ def prepare_ip_adapter_image_embeds(
576
+ self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance
577
+ ):
578
+ image_embeds = []
579
+ if do_classifier_free_guidance:
580
+ negative_image_embeds = []
581
+ if ip_adapter_image_embeds is None:
582
+ if not isinstance(ip_adapter_image, list):
583
+ ip_adapter_image = [ip_adapter_image]
584
+
585
+ if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers):
586
+ raise ValueError(
587
+ f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters."
588
+ )
589
+
590
+ for single_ip_adapter_image, image_proj_layer in zip(
591
+ ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers
592
+ ):
593
+ output_hidden_state = not isinstance(image_proj_layer, ImageProjection)
594
+ single_image_embeds, single_negative_image_embeds = self.encode_image(
595
+ single_ip_adapter_image, device, 1, output_hidden_state
596
+ )
597
+
598
+ image_embeds.append(single_image_embeds[None, :])
599
+ if do_classifier_free_guidance:
600
+ negative_image_embeds.append(single_negative_image_embeds[None, :])
601
+ else:
602
+ for single_image_embeds in ip_adapter_image_embeds:
603
+ if do_classifier_free_guidance:
604
+ single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2)
605
+ negative_image_embeds.append(single_negative_image_embeds)
606
+ image_embeds.append(single_image_embeds)
607
+
608
+ ip_adapter_image_embeds = []
609
+ for i, single_image_embeds in enumerate(image_embeds):
610
+ single_image_embeds = torch.cat([single_image_embeds] * num_images_per_prompt, dim=0)
611
+ if do_classifier_free_guidance:
612
+ single_negative_image_embeds = torch.cat([negative_image_embeds[i]] * num_images_per_prompt, dim=0)
613
+ single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds], dim=0)
614
+
615
+ single_image_embeds = single_image_embeds.to(device=device)
616
+ ip_adapter_image_embeds.append(single_image_embeds)
617
+
618
+ return ip_adapter_image_embeds
619
+
620
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs
621
+ def prepare_extra_step_kwargs(self, generator, eta):
622
+ # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
623
+ # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
624
+ # eta corresponds to η in DDIM paper: https://huggingface.co/papers/2010.02502
625
+ # and should be between [0, 1]
626
+
627
+ accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
628
+ extra_step_kwargs = {}
629
+ if accepts_eta:
630
+ extra_step_kwargs["eta"] = eta
631
+
632
+ # check if the scheduler accepts generator
633
+ accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
634
+ if accepts_generator:
635
+ extra_step_kwargs["generator"] = generator
636
+ return extra_step_kwargs
637
+
638
+ # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.check_inputs
639
+ def check_inputs(
640
+ self,
641
+ prompt,
642
+ prompt_2,
643
+ height,
644
+ width,
645
+ callback_steps,
646
+ negative_prompt=None,
647
+ negative_prompt_2=None,
648
+ prompt_embeds=None,
649
+ negative_prompt_embeds=None,
650
+ pooled_prompt_embeds=None,
651
+ negative_pooled_prompt_embeds=None,
652
+ ip_adapter_image=None,
653
+ ip_adapter_image_embeds=None,
654
+ callback_on_step_end_tensor_inputs=None,
655
+ ):
656
+ if height % 8 != 0 or width % 8 != 0:
657
+ raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
658
+
659
+ if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0):
660
+ raise ValueError(
661
+ f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
662
+ f" {type(callback_steps)}."
663
+ )
664
+
665
+ if callback_on_step_end_tensor_inputs is not None and not all(
666
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
667
+ ):
668
+ raise ValueError(
669
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
670
+ )
671
+
672
+ if prompt is not None and prompt_embeds is not None:
673
+ raise ValueError(
674
+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
675
+ " only forward one of the two."
676
+ )
677
+ elif prompt_2 is not None and prompt_embeds is not None:
678
+ raise ValueError(
679
+ f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
680
+ " only forward one of the two."
681
+ )
682
+ elif prompt is None and prompt_embeds is None:
683
+ raise ValueError(
684
+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
685
+ )
686
+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
687
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
688
+ elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)):
689
+ raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}")
690
+
691
+ if negative_prompt is not None and negative_prompt_embeds is not None:
692
+ raise ValueError(
693
+ f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
694
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
695
+ )
696
+ elif negative_prompt_2 is not None and negative_prompt_embeds is not None:
697
+ raise ValueError(
698
+ f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:"
699
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
700
+ )
701
+
702
+ if prompt_embeds is not None and negative_prompt_embeds is not None:
703
+ if prompt_embeds.shape != negative_prompt_embeds.shape:
704
+ raise ValueError(
705
+ "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"
706
+ f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"
707
+ f" {negative_prompt_embeds.shape}."
708
+ )
709
+
710
+ if prompt_embeds is not None and pooled_prompt_embeds is None:
711
+ raise ValueError(
712
+ "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`."
713
+ )
714
+
715
+ if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None:
716
+ raise ValueError(
717
+ "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`."
718
+ )
719
+
720
+ if ip_adapter_image is not None and ip_adapter_image_embeds is not None:
721
+ raise ValueError(
722
+ "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined."
723
+ )
724
+
725
+ if ip_adapter_image_embeds is not None:
726
+ if not isinstance(ip_adapter_image_embeds, list):
727
+ raise ValueError(
728
+ f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}"
729
+ )
730
+ elif ip_adapter_image_embeds[0].ndim not in [3, 4]:
731
+ raise ValueError(
732
+ f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D"
733
+ )
734
+
735
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents
736
+ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):
737
+ shape = (
738
+ batch_size,
739
+ num_channels_latents,
740
+ int(height) // self.vae_scale_factor,
741
+ int(width) // self.vae_scale_factor,
742
+ )
743
+ if isinstance(generator, list) and len(generator) != batch_size:
744
+ raise ValueError(
745
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
746
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
747
+ )
748
+
749
+ if latents is None:
750
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
751
+ else:
752
+ latents = latents.to(device)
753
+
754
+ # scale the initial noise by the standard deviation required by the scheduler
755
+ latents = latents * self.scheduler.init_noise_sigma
756
+ return latents
757
+
758
+ # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline._get_add_time_ids
759
+ def _get_add_time_ids(
760
+ self, original_size, crops_coords_top_left, target_size, dtype, text_encoder_projection_dim=None
761
+ ):
762
+ add_time_ids = list(original_size + crops_coords_top_left + target_size)
763
+
764
+ passed_add_embed_dim = (
765
+ self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim
766
+ )
767
+ expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features
768
+
769
+ if expected_add_embed_dim != passed_add_embed_dim:
770
+ raise ValueError(
771
+ f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`."
772
+ )
773
+
774
+ add_time_ids = torch.tensor([add_time_ids], dtype=dtype)
775
+ return add_time_ids
776
+
777
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.StableDiffusionUpscalePipeline.upcast_vae
778
+ def upcast_vae(self):
779
+ deprecate(
780
+ "upcast_vae",
781
+ "1.0.0",
782
+ "`upcast_vae` is deprecated. Please use `pipe.vae.to(torch.float32)`. For more details, please refer to: https://github.com/huggingface/diffusers/pull/12619#issue-3606633695.",
783
+ )
784
+ self.vae.to(dtype=torch.float32)
785
+
786
+ # Copied from diffusers.pipelines.t2i_adapter.pipeline_stable_diffusion_adapter.StableDiffusionAdapterPipeline._default_height_width
787
+ def _default_height_width(self, height, width, image):
788
+ # NOTE: It is possible that a list of images have different
789
+ # dimensions for each image, so just checking the first image
790
+ # is not _exactly_ correct, but it is simple.
791
+ while isinstance(image, list):
792
+ image = image[0]
793
+
794
+ if height is None:
795
+ if isinstance(image, PIL.Image.Image):
796
+ height = image.height
797
+ elif isinstance(image, torch.Tensor):
798
+ height = image.shape[-2]
799
+
800
+ # round down to nearest multiple of `self.adapter.downscale_factor`
801
+ height = (height // self.adapter.downscale_factor) * self.adapter.downscale_factor
802
+
803
+ if width is None:
804
+ if isinstance(image, PIL.Image.Image):
805
+ width = image.width
806
+ elif isinstance(image, torch.Tensor):
807
+ width = image.shape[-1]
808
+
809
+ # round down to nearest multiple of `self.adapter.downscale_factor`
810
+ width = (width // self.adapter.downscale_factor) * self.adapter.downscale_factor
811
+
812
+ return height, width
813
+
814
+ # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding
815
+ def get_guidance_scale_embedding(
816
+ self, w: torch.Tensor, embedding_dim: int = 512, dtype: torch.dtype = torch.float32
817
+ ) -> torch.Tensor:
818
+ """
819
+ See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298
820
+
821
+ Args:
822
+ w (`torch.Tensor`):
823
+ Generate embedding vectors with a specified guidance scale to subsequently enrich timestep embeddings.
824
+ embedding_dim (`int`, *optional*, defaults to 512):
825
+ Dimension of the embeddings to generate.
826
+ dtype (`torch.dtype`, *optional*, defaults to `torch.float32`):
827
+ Data type of the generated embeddings.
828
+
829
+ Returns:
830
+ `torch.Tensor`: Embedding vectors with shape `(len(w), embedding_dim)`.
831
+ """
832
+ assert len(w.shape) == 1
833
+ w = w * 1000.0
834
+
835
+ half_dim = embedding_dim // 2
836
+ emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1)
837
+ emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb)
838
+ emb = w.to(dtype)[:, None] * emb[None, :]
839
+ emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)
840
+ if embedding_dim % 2 == 1: # zero pad
841
+ emb = torch.nn.functional.pad(emb, (0, 1))
842
+ assert emb.shape == (w.shape[0], embedding_dim)
843
+ return emb
844
+
845
+ @property
846
+ def guidance_scale(self):
847
+ return self._guidance_scale
848
+
849
+ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
850
+ # of the Imagen paper: https://huggingface.co/papers/2205.11487 . `guidance_scale = 1`
851
+ # corresponds to doing no classifier free guidance.
852
+ @property
853
+ def do_classifier_free_guidance(self):
854
+ return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None
855
+
856
+ @torch.no_grad()
857
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
858
+ def __call__(
859
+ self,
860
+ prompt: str | list[str] = None,
861
+ prompt_2: str | list[str] | None = None,
862
+ image: PipelineImageInput = None,
863
+ height: int | None = None,
864
+ width: int | None = None,
865
+ num_inference_steps: int = 50,
866
+ timesteps: list[int] = None,
867
+ sigmas: list[float] = None,
868
+ denoising_end: float | None = None,
869
+ guidance_scale: float = 5.0,
870
+ negative_prompt: str | list[str] | None = None,
871
+ negative_prompt_2: str | list[str] | None = None,
872
+ num_images_per_prompt: int | None = 1,
873
+ eta: float = 0.0,
874
+ generator: torch.Generator | list[torch.Generator] | None = None,
875
+ latents: torch.Tensor | None = None,
876
+ prompt_embeds: torch.Tensor | None = None,
877
+ negative_prompt_embeds: torch.Tensor | None = None,
878
+ pooled_prompt_embeds: torch.Tensor | None = None,
879
+ negative_pooled_prompt_embeds: torch.Tensor | None = None,
880
+ ip_adapter_image: PipelineImageInput | None = None,
881
+ ip_adapter_image_embeds: list[torch.Tensor] | None = None,
882
+ output_type: str | None = "pil",
883
+ return_dict: bool = True,
884
+ callback: Callable[[int, int, torch.Tensor], None] | None = None,
885
+ callback_steps: int = 1,
886
+ cross_attention_kwargs: dict[str, Any] | None = None,
887
+ guidance_rescale: float = 0.0,
888
+ original_size: tuple[int, int] | None = None,
889
+ crops_coords_top_left: tuple[int, int] = (0, 0),
890
+ target_size: tuple[int, int] | None = None,
891
+ negative_original_size: tuple[int, int] | None = None,
892
+ negative_crops_coords_top_left: tuple[int, int] = (0, 0),
893
+ negative_target_size: tuple[int, int] | None = None,
894
+ adapter_conditioning_scale: float | list[float] = 1.0,
895
+ adapter_conditioning_factor: float = 1.0,
896
+ clip_skip: int | None = None,
897
+ ):
898
+ r"""
899
+ Function invoked when calling the pipeline for generation.
900
+
901
+ Args:
902
+ prompt (`str` or `list[str]`, *optional*):
903
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
904
+ instead.
905
+ prompt_2 (`str` or `list[str]`, *optional*):
906
+ The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
907
+ used in both text-encoders
908
+ image (`torch.Tensor`, `PIL.Image.Image`, `list[torch.Tensor]` or `list[PIL.Image.Image]` or `list[list[PIL.Image.Image]]`):
909
+ The Adapter input condition. Adapter uses this input condition to generate guidance to Unet. If the
910
+ type is specified as `torch.Tensor`, it is passed to Adapter as is. PIL.Image.Image` can also be
911
+ accepted as an image. The control image is automatically resized to fit the output image.
912
+ height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
913
+ The height in pixels of the generated image. Anything below 512 pixels won't work well for
914
+ [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
915
+ and checkpoints that are not specifically fine-tuned on low resolutions.
916
+ width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
917
+ The width in pixels of the generated image. Anything below 512 pixels won't work well for
918
+ [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
919
+ and checkpoints that are not specifically fine-tuned on low resolutions.
920
+ num_inference_steps (`int`, *optional*, defaults to 50):
921
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
922
+ expense of slower inference.
923
+ timesteps (`list[int]`, *optional*):
924
+ Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument
925
+ in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is
926
+ passed will be used. Must be in descending order.
927
+ sigmas (`list[float]`, *optional*):
928
+ Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
929
+ their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
930
+ will be used.
931
+ denoising_end (`float`, *optional*):
932
+ When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be
933
+ completed before it is intentionally prematurely terminated. As a result, the returned sample will
934
+ still retain a substantial amount of noise as determined by the discrete timesteps selected by the
935
+ scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a
936
+ "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image
937
+ Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output)
938
+ guidance_scale (`float`, *optional*, defaults to 5.0):
939
+ Guidance scale as defined in [Classifier-Free Diffusion
940
+ Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2.
941
+ of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
942
+ `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to
943
+ the text `prompt`, usually at the expense of lower image quality.
944
+ negative_prompt (`str` or `list[str]`, *optional*):
945
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
946
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
947
+ less than `1`).
948
+ negative_prompt_2 (`str` or `list[str]`, *optional*):
949
+ The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
950
+ `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
951
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
952
+ The number of images to generate per prompt.
953
+ eta (`float`, *optional*, defaults to 0.0):
954
+ Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only
955
+ applies to [`schedulers.DDIMScheduler`], will be ignored for others.
956
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
957
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
958
+ to make generation deterministic.
959
+ latents (`torch.Tensor`, *optional*):
960
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
961
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
962
+ tensor will be generated by sampling using the supplied random `generator`.
963
+ prompt_embeds (`torch.Tensor`, *optional*):
964
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
965
+ provided, text embeddings will be generated from `prompt` input argument.
966
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
967
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
968
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
969
+ argument.
970
+ pooled_prompt_embeds (`torch.Tensor`, *optional*):
971
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
972
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
973
+ negative_pooled_prompt_embeds (`torch.Tensor`, *optional*):
974
+ Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
975
+ weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
976
+ input argument.
977
+ ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.
978
+ ip_adapter_image_embeds (`list[torch.Tensor]`, *optional*):
979
+ Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of
980
+ IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should
981
+ contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not
982
+ provided, embeddings are computed from the `ip_adapter_image` input argument.
983
+ output_type (`str`, *optional*, defaults to `"pil"`):
984
+ The output format of the generate image. Choose between
985
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
986
+ return_dict (`bool`, *optional*, defaults to `True`):
987
+ Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionAdapterPipelineOutput`]
988
+ instead of a plain tuple.
989
+ callback (`Callable`, *optional*):
990
+ A function that will be called every `callback_steps` steps during inference. The function will be
991
+ called with the following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`.
992
+ callback_steps (`int`, *optional*, defaults to 1):
993
+ The frequency at which the `callback` function will be called. If not specified, the callback will be
994
+ called at every step.
995
+ cross_attention_kwargs (`dict`, *optional*):
996
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
997
+ `self.processor` in
998
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
999
+ guidance_rescale (`float`, *optional*, defaults to 0.0):
1000
+ Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are
1001
+ Flawed](https://huggingface.co/papers/2305.08891) `guidance_scale` is defined as `φ` in equation 16. of
1002
+ [Common Diffusion Noise Schedules and Sample Steps are
1003
+ Flawed](https://huggingface.co/papers/2305.08891). Guidance rescale factor should fix overexposure when
1004
+ using zero terminal SNR.
1005
+ original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
1006
+ If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled.
1007
+ `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as
1008
+ explained in section 2.2 of
1009
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
1010
+ crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)):
1011
+ `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position
1012
+ `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting
1013
+ `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of
1014
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
1015
+ target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
1016
+ For most cases, `target_size` should be set to the desired height and width of the generated image. If
1017
+ not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in
1018
+ section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
1019
+ section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
1020
+ negative_original_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
1021
+ To negatively condition the generation process based on a specific image resolution. Part of SDXL's
1022
+ micro-conditioning as explained in section 2.2 of
1023
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
1024
+ information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
1025
+ negative_crops_coords_top_left (`tuple[int]`, *optional*, defaults to (0, 0)):
1026
+ To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's
1027
+ micro-conditioning as explained in section 2.2 of
1028
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
1029
+ information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
1030
+ negative_target_size (`tuple[int]`, *optional*, defaults to (1024, 1024)):
1031
+ To negatively condition the generation process based on a target image resolution. It should be as same
1032
+ as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of
1033
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
1034
+ information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
1035
+ adapter_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 1.0):
1036
+ The outputs of the adapter are multiplied by `adapter_conditioning_scale` before they are added to the
1037
+ residual in the original unet. If multiple adapters are specified in init, you can set the
1038
+ corresponding scale as a list.
1039
+ adapter_conditioning_factor (`float`, *optional*, defaults to 1.0):
1040
+ The fraction of timesteps for which adapter should be applied. If `adapter_conditioning_factor` is
1041
+ `0.0`, adapter is not applied at all. If `adapter_conditioning_factor` is `1.0`, adapter is applied for
1042
+ all timesteps. If `adapter_conditioning_factor` is `0.5`, adapter is applied for half of the timesteps.
1043
+ clip_skip (`int`, *optional*):
1044
+ Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
1045
+ the output of the pre-final layer will be used for computing the prompt embeddings.
1046
+
1047
+ Examples:
1048
+
1049
+ Returns:
1050
+ [`~pipelines.stable_diffusion.StableDiffusionAdapterPipelineOutput`] or `tuple`:
1051
+ [`~pipelines.stable_diffusion.StableDiffusionAdapterPipelineOutput`] if `return_dict` is True, otherwise a
1052
+ `tuple`. When returning a tuple, the first element is a list with the generated images.
1053
+ """
1054
+ # 0. Default height and width to unet
1055
+
1056
+ height, width = self._default_height_width(height, width, image)
1057
+ device = self._execution_device
1058
+
1059
+ if isinstance(self.adapter, MultiAdapter):
1060
+ adapter_input = []
1061
+
1062
+ for one_image in image:
1063
+ one_image = _preprocess_adapter_image(one_image, height, width)
1064
+ one_image = one_image.to(device=device, dtype=self.adapter.dtype)
1065
+ adapter_input.append(one_image)
1066
+ else:
1067
+ adapter_input = _preprocess_adapter_image(image, height, width)
1068
+ adapter_input = adapter_input.to(device=device, dtype=self.adapter.dtype)
1069
+ original_size = original_size or (height, width)
1070
+ target_size = target_size or (height, width)
1071
+
1072
+ # 1. Check inputs. Raise error if not correct
1073
+ self.check_inputs(
1074
+ prompt,
1075
+ prompt_2,
1076
+ height,
1077
+ width,
1078
+ callback_steps,
1079
+ negative_prompt,
1080
+ negative_prompt_2,
1081
+ prompt_embeds,
1082
+ negative_prompt_embeds,
1083
+ pooled_prompt_embeds,
1084
+ negative_pooled_prompt_embeds,
1085
+ ip_adapter_image,
1086
+ ip_adapter_image_embeds,
1087
+ )
1088
+
1089
+ self._guidance_scale = guidance_scale
1090
+
1091
+ # 2. Define call parameters
1092
+ if prompt is not None and isinstance(prompt, str):
1093
+ batch_size = 1
1094
+ elif prompt is not None and isinstance(prompt, list):
1095
+ batch_size = len(prompt)
1096
+ else:
1097
+ batch_size = prompt_embeds.shape[0]
1098
+
1099
+ device = self._execution_device
1100
+
1101
+ # 3.1 Encode input prompt
1102
+ (
1103
+ prompt_embeds,
1104
+ negative_prompt_embeds,
1105
+ pooled_prompt_embeds,
1106
+ negative_pooled_prompt_embeds,
1107
+ ) = self.encode_prompt(
1108
+ prompt=prompt,
1109
+ prompt_2=prompt_2,
1110
+ device=device,
1111
+ num_images_per_prompt=num_images_per_prompt,
1112
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
1113
+ negative_prompt=negative_prompt,
1114
+ negative_prompt_2=negative_prompt_2,
1115
+ prompt_embeds=prompt_embeds,
1116
+ negative_prompt_embeds=negative_prompt_embeds,
1117
+ pooled_prompt_embeds=pooled_prompt_embeds,
1118
+ negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
1119
+ clip_skip=clip_skip,
1120
+ )
1121
+
1122
+ # 3.2 Encode ip_adapter_image
1123
+ if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
1124
+ image_embeds = self.prepare_ip_adapter_image_embeds(
1125
+ ip_adapter_image,
1126
+ ip_adapter_image_embeds,
1127
+ device,
1128
+ batch_size * num_images_per_prompt,
1129
+ self.do_classifier_free_guidance,
1130
+ )
1131
+
1132
+ # 4. Prepare timesteps
1133
+ if XLA_AVAILABLE:
1134
+ timestep_device = "cpu"
1135
+ else:
1136
+ timestep_device = device
1137
+ timesteps, num_inference_steps = retrieve_timesteps(
1138
+ self.scheduler, num_inference_steps, timestep_device, timesteps, sigmas
1139
+ )
1140
+
1141
+ # 5. Prepare latent variables
1142
+ num_channels_latents = self.unet.config.in_channels
1143
+ latents = self.prepare_latents(
1144
+ batch_size * num_images_per_prompt,
1145
+ num_channels_latents,
1146
+ height,
1147
+ width,
1148
+ prompt_embeds.dtype,
1149
+ device,
1150
+ generator,
1151
+ latents,
1152
+ )
1153
+
1154
+ # 6.1 Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
1155
+ extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
1156
+
1157
+ # 6.2 Optionally get Guidance Scale Embedding
1158
+ timestep_cond = None
1159
+ if self.unet.config.time_cond_proj_dim is not None:
1160
+ guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)
1161
+ timestep_cond = self.get_guidance_scale_embedding(
1162
+ guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim
1163
+ ).to(device=device, dtype=latents.dtype)
1164
+
1165
+ # 7. Prepare added time ids & embeddings & adapter features
1166
+ if isinstance(self.adapter, MultiAdapter):
1167
+ adapter_state = self.adapter(adapter_input, adapter_conditioning_scale)
1168
+ for k, v in enumerate(adapter_state):
1169
+ adapter_state[k] = v
1170
+ else:
1171
+ adapter_state = self.adapter(adapter_input)
1172
+ for k, v in enumerate(adapter_state):
1173
+ adapter_state[k] = v * adapter_conditioning_scale
1174
+ if num_images_per_prompt > 1:
1175
+ for k, v in enumerate(adapter_state):
1176
+ adapter_state[k] = v.repeat(num_images_per_prompt, 1, 1, 1)
1177
+ if self.do_classifier_free_guidance:
1178
+ for k, v in enumerate(adapter_state):
1179
+ adapter_state[k] = torch.cat([v] * 2, dim=0)
1180
+
1181
+ add_text_embeds = pooled_prompt_embeds
1182
+ if self.text_encoder_2 is None:
1183
+ text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1])
1184
+ else:
1185
+ text_encoder_projection_dim = self.text_encoder_2.config.projection_dim
1186
+
1187
+ add_time_ids = self._get_add_time_ids(
1188
+ original_size,
1189
+ crops_coords_top_left,
1190
+ target_size,
1191
+ dtype=prompt_embeds.dtype,
1192
+ text_encoder_projection_dim=text_encoder_projection_dim,
1193
+ )
1194
+ if negative_original_size is not None and negative_target_size is not None:
1195
+ negative_add_time_ids = self._get_add_time_ids(
1196
+ negative_original_size,
1197
+ negative_crops_coords_top_left,
1198
+ negative_target_size,
1199
+ dtype=prompt_embeds.dtype,
1200
+ text_encoder_projection_dim=text_encoder_projection_dim,
1201
+ )
1202
+ else:
1203
+ negative_add_time_ids = add_time_ids
1204
+
1205
+ if self.do_classifier_free_guidance:
1206
+ prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
1207
+ add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0)
1208
+ add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], dim=0)
1209
+
1210
+ prompt_embeds = prompt_embeds.to(device)
1211
+ add_text_embeds = add_text_embeds.to(device)
1212
+ add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1)
1213
+
1214
+ # 8. Denoising loop
1215
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
1216
+ # Apply denoising_end
1217
+ if denoising_end is not None and isinstance(denoising_end, float) and denoising_end > 0 and denoising_end < 1:
1218
+ discrete_timestep_cutoff = int(
1219
+ round(
1220
+ self.scheduler.config.num_train_timesteps
1221
+ - (denoising_end * self.scheduler.config.num_train_timesteps)
1222
+ )
1223
+ )
1224
+ num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps)))
1225
+ timesteps = timesteps[:num_inference_steps]
1226
+
1227
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
1228
+ for i, t in enumerate(timesteps):
1229
+ # expand the latents if we are doing classifier free guidance
1230
+ latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
1231
+
1232
+ latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
1233
+
1234
+ added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
1235
+
1236
+ if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
1237
+ added_cond_kwargs["image_embeds"] = image_embeds
1238
+
1239
+ # predict the noise residual
1240
+ if i < int(num_inference_steps * adapter_conditioning_factor):
1241
+ down_intrablock_additional_residuals = [state.clone() for state in adapter_state]
1242
+ else:
1243
+ down_intrablock_additional_residuals = None
1244
+
1245
+ noise_pred = self.unet(
1246
+ latent_model_input,
1247
+ t,
1248
+ encoder_hidden_states=prompt_embeds,
1249
+ timestep_cond=timestep_cond,
1250
+ cross_attention_kwargs=cross_attention_kwargs,
1251
+ down_intrablock_additional_residuals=down_intrablock_additional_residuals,
1252
+ added_cond_kwargs=added_cond_kwargs,
1253
+ return_dict=False,
1254
+ )[0]
1255
+
1256
+ # perform guidance
1257
+ if self.do_classifier_free_guidance:
1258
+ noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
1259
+ noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
1260
+
1261
+ if self.do_classifier_free_guidance and guidance_rescale > 0.0:
1262
+ # Based on 3.4. in https://huggingface.co/papers/2305.08891
1263
+ noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale)
1264
+
1265
+ # compute the previous noisy sample x_t -> x_t-1
1266
+ latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
1267
+
1268
+ # call the callback, if provided
1269
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
1270
+ progress_bar.update()
1271
+ if callback is not None and i % callback_steps == 0:
1272
+ step_idx = i // getattr(self.scheduler, "order", 1)
1273
+ callback(step_idx, t, latents)
1274
+
1275
+ if XLA_AVAILABLE:
1276
+ xm.mark_step()
1277
+
1278
+ if not output_type == "latent":
1279
+ # make sure the VAE is in float32 mode, as it overflows in float16
1280
+ needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast
1281
+
1282
+ if needs_upcasting:
1283
+ self.upcast_vae()
1284
+ latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)
1285
+
1286
+ image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0]
1287
+
1288
+ # cast back to fp16 if needed
1289
+ if needs_upcasting:
1290
+ self.vae.to(dtype=torch.float16)
1291
+ else:
1292
+ image = latents
1293
+ return StableDiffusionXLPipelineOutput(images=image)
1294
+
1295
+ image = self.image_processor.postprocess(image, output_type=output_type)
1296
+
1297
+ # Offload all models
1298
+ self.maybe_free_model_hooks()
1299
+
1300
+ if not return_dict:
1301
+ return (image,)
1302
+
1303
+ return StableDiffusionXLPipelineOutput(images=image)
western_ai/europe/diffusers/src/diffusers/pipelines/transformers_loading_utils.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2025 The HuggingFace Inc. team.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ import contextlib
16
+ import os
17
+ import tempfile
18
+ from typing import TYPE_CHECKING
19
+
20
+ from huggingface_hub import DDUFEntry
21
+ from tqdm import tqdm
22
+
23
+ from ..utils import is_safetensors_available, is_transformers_available, is_transformers_version
24
+
25
+
26
+ if TYPE_CHECKING:
27
+ from transformers import PreTrainedModel, PreTrainedTokenizer
28
+
29
+ if is_transformers_available():
30
+ from transformers import PreTrainedModel, PreTrainedTokenizer
31
+
32
+ if is_safetensors_available():
33
+ import safetensors.torch
34
+
35
+
36
+ def _load_tokenizer_from_dduf(
37
+ cls: "PreTrainedTokenizer", name: str, dduf_entries: dict[str, DDUFEntry], **kwargs
38
+ ) -> "PreTrainedTokenizer":
39
+ """
40
+ Load a tokenizer from a DDUF archive.
41
+
42
+ In practice, `transformers` do not provide a way to load a tokenizer from a DDUF archive. This function is a
43
+ workaround by extracting the tokenizer files from the DDUF archive and loading the tokenizer from the extracted
44
+ files. There is an extra cost of extracting the files, but of limited impact as the tokenizer files are usually
45
+ small-ish.
46
+ """
47
+ with tempfile.TemporaryDirectory() as tmp_dir:
48
+ for entry_name, entry in dduf_entries.items():
49
+ if entry_name.startswith(name + "/"):
50
+ tmp_entry_path = os.path.join(tmp_dir, *entry_name.split("/"))
51
+ # need to create intermediary directory if they don't exist
52
+ os.makedirs(os.path.dirname(tmp_entry_path), exist_ok=True)
53
+ with open(tmp_entry_path, "wb") as f:
54
+ with entry.as_mmap() as mm:
55
+ f.write(mm)
56
+ return cls.from_pretrained(os.path.dirname(tmp_entry_path), **kwargs)
57
+
58
+
59
+ def _load_transformers_model_from_dduf(
60
+ cls: "PreTrainedModel", name: str, dduf_entries: dict[str, DDUFEntry], **kwargs
61
+ ) -> "PreTrainedModel":
62
+ """
63
+ Load a transformers model from a DDUF archive.
64
+
65
+ In practice, `transformers` do not provide a way to load a model from a DDUF archive. This function is a workaround
66
+ by instantiating a model from the config file and loading the weights from the DDUF archive directly.
67
+ """
68
+ config_file = dduf_entries.get(f"{name}/config.json")
69
+ if config_file is None:
70
+ raise EnvironmentError(
71
+ f"Could not find a config.json file for component {name} in DDUF file (contains {dduf_entries.keys()})."
72
+ )
73
+ generation_config = dduf_entries.get(f"{name}/generation_config.json", None)
74
+
75
+ weight_files = [
76
+ entry
77
+ for entry_name, entry in dduf_entries.items()
78
+ if entry_name.startswith(f"{name}/") and entry_name.endswith(".safetensors")
79
+ ]
80
+ if not weight_files:
81
+ raise EnvironmentError(
82
+ f"Could not find any weight file for component {name} in DDUF file (contains {dduf_entries.keys()})."
83
+ )
84
+ if not is_safetensors_available():
85
+ raise EnvironmentError(
86
+ "Safetensors is not available, cannot load model from DDUF. Please `pip install safetensors`."
87
+ )
88
+ if is_transformers_version("<", "4.47.0"):
89
+ raise ImportError(
90
+ "You need to install `transformers>4.47.0` in order to load a transformers model from a DDUF file. "
91
+ "You can install it with: `pip install --upgrade transformers`"
92
+ )
93
+
94
+ with tempfile.TemporaryDirectory() as tmp_dir:
95
+ from transformers import AutoConfig, GenerationConfig
96
+
97
+ tmp_config_file = os.path.join(tmp_dir, "config.json")
98
+ with open(tmp_config_file, "w") as f:
99
+ f.write(config_file.read_text())
100
+ config = AutoConfig.from_pretrained(tmp_config_file)
101
+ if generation_config is not None:
102
+ tmp_generation_config_file = os.path.join(tmp_dir, "generation_config.json")
103
+ with open(tmp_generation_config_file, "w") as f:
104
+ f.write(generation_config.read_text())
105
+ generation_config = GenerationConfig.from_pretrained(tmp_generation_config_file)
106
+ state_dict = {}
107
+ with contextlib.ExitStack() as stack:
108
+ for entry in tqdm(weight_files, desc="Loading state_dict"): # Loop over safetensors files
109
+ # Memory-map the safetensors file
110
+ mmap = stack.enter_context(entry.as_mmap())
111
+ # Load tensors from the memory-mapped file
112
+ tensors = safetensors.torch.load(mmap)
113
+ # Update the state dictionary with tensors
114
+ state_dict.update(tensors)
115
+ # `from_pretrained` sets the model to eval mode by default, which is the
116
+ # correct behavior for inference. Do not call `model.train()` here.
117
+ return cls.from_pretrained(
118
+ pretrained_model_name_or_path=None,
119
+ config=config,
120
+ generation_config=generation_config,
121
+ state_dict=state_dict,
122
+ **kwargs,
123
+ )
western_ai/europe/diffusers/src/diffusers/pipelines/visualcloze/__init__.py ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import TYPE_CHECKING
2
+
3
+ from ...utils import (
4
+ DIFFUSERS_SLOW_IMPORT,
5
+ OptionalDependencyNotAvailable,
6
+ _LazyModule,
7
+ get_objects_from_module,
8
+ is_torch_available,
9
+ is_transformers_available,
10
+ )
11
+
12
+
13
+ _dummy_objects = {}
14
+ _import_structure = {}
15
+
16
+
17
+ try:
18
+ if not (is_transformers_available() and is_torch_available()):
19
+ raise OptionalDependencyNotAvailable()
20
+ except OptionalDependencyNotAvailable:
21
+ from ...utils import dummy_torch_and_transformers_objects # noqa F403
22
+
23
+ _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects))
24
+ else:
25
+ _import_structure["pipeline_visualcloze_combined"] = ["VisualClozePipeline"]
26
+ _import_structure["pipeline_visualcloze_generation"] = ["VisualClozeGenerationPipeline"]
27
+
28
+
29
+ if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT:
30
+ try:
31
+ if not (is_transformers_available() and is_torch_available()):
32
+ raise OptionalDependencyNotAvailable()
33
+
34
+ except OptionalDependencyNotAvailable:
35
+ from ...utils.dummy_torch_and_transformers_objects import *
36
+ else:
37
+ from .pipeline_visualcloze_combined import VisualClozePipeline
38
+ from .pipeline_visualcloze_generation import VisualClozeGenerationPipeline
39
+
40
+
41
+ else:
42
+ import sys
43
+
44
+ sys.modules[__name__] = _LazyModule(
45
+ __name__,
46
+ globals()["__file__"],
47
+ _import_structure,
48
+ module_spec=__spec__,
49
+ )
50
+
51
+ for name, value in _dummy_objects.items():
52
+ setattr(sys.modules[__name__], name, value)
western_ai/europe/diffusers/src/diffusers/pipelines/visualcloze/pipeline_visualcloze_combined.py ADDED
@@ -0,0 +1,440 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 VisualCloze team and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from typing import Any, Callable
16
+
17
+ import torch
18
+ from PIL import Image
19
+ from transformers import CLIPTextModel, CLIPTokenizer, T5EncoderModel, T5TokenizerFast
20
+
21
+ from ...loaders import FluxLoraLoaderMixin, FromSingleFileMixin, TextualInversionLoaderMixin
22
+ from ...models.autoencoders import AutoencoderKL
23
+ from ...models.transformers import FluxTransformer2DModel
24
+ from ...schedulers import FlowMatchEulerDiscreteScheduler
25
+ from ...utils import is_torch_xla_available, logging, replace_example_docstring
26
+ from ..flux.pipeline_flux_fill import FluxFillPipeline as VisualClozeUpsamplingPipeline
27
+ from ..flux.pipeline_output import FluxPipelineOutput
28
+ from ..pipeline_utils import DiffusionPipeline
29
+ from .pipeline_visualcloze_generation import VisualClozeGenerationPipeline
30
+
31
+
32
+ if is_torch_xla_available():
33
+ XLA_AVAILABLE = True
34
+ else:
35
+ XLA_AVAILABLE = False
36
+
37
+
38
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
39
+
40
+ EXAMPLE_DOC_STRING = """
41
+ Examples:
42
+ ```python
43
+ >>> import torch
44
+ >>> from diffusers import VisualClozePipeline
45
+ >>> from diffusers.utils import load_image
46
+
47
+ >>> image_paths = [
48
+ ... # in-context examples
49
+ ... [
50
+ ... load_image(
51
+ ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/visualcloze/visualcloze_mask2image_incontext-example-1_mask.jpg"
52
+ ... ),
53
+ ... load_image(
54
+ ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/visualcloze/visualcloze_mask2image_incontext-example-1_image.jpg"
55
+ ... ),
56
+ ... ],
57
+ ... # query with the target image
58
+ ... [
59
+ ... load_image(
60
+ ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/visualcloze/visualcloze_mask2image_query_mask.jpg"
61
+ ... ),
62
+ ... None, # No image needed for the target image
63
+ ... ],
64
+ ... ]
65
+ >>> task_prompt = "In each row, a logical task is demonstrated to achieve [IMAGE2] an aesthetically pleasing photograph based on [IMAGE1] sam 2-generated masks with rich color coding."
66
+ >>> content_prompt = "Majestic photo of a golden eagle perched on a rocky outcrop in a mountainous landscape. The eagle is positioned in the right foreground, facing left, with its sharp beak and keen eyes prominently visible. Its plumage is a mix of dark brown and golden hues, with intricate feather details. The background features a soft-focus view of snow-capped mountains under a cloudy sky, creating a serene and grandiose atmosphere. The foreground includes rugged rocks and patches of green moss. Photorealistic, medium depth of field, soft natural lighting, cool color palette, high contrast, sharp focus on the eagle, blurred background, tranquil, majestic, wildlife photography."
67
+ >>> pipe = VisualClozePipeline.from_pretrained(
68
+ ... "VisualCloze/VisualClozePipeline-384", resolution=384, torch_dtype=torch.bfloat16
69
+ ... )
70
+ >>> pipe.to("cuda")
71
+
72
+ >>> image = pipe(
73
+ ... task_prompt=task_prompt,
74
+ ... content_prompt=content_prompt,
75
+ ... image=image_paths,
76
+ ... upsampling_width=1344,
77
+ ... upsampling_height=768,
78
+ ... upsampling_strength=0.4,
79
+ ... guidance_scale=30,
80
+ ... num_inference_steps=30,
81
+ ... max_sequence_length=512,
82
+ ... generator=torch.Generator("cpu").manual_seed(0),
83
+ ... ).images[0][0]
84
+ >>> image.save("visualcloze.png")
85
+ ```
86
+ """
87
+
88
+
89
+ class VisualClozePipeline(
90
+ DiffusionPipeline,
91
+ FluxLoraLoaderMixin,
92
+ FromSingleFileMixin,
93
+ TextualInversionLoaderMixin,
94
+ ):
95
+ r"""
96
+ The VisualCloze pipeline for image generation with visual context. Reference:
97
+ https://github.com/lzyhha/VisualCloze/tree/main. This pipeline is designed to generate images based on visual
98
+ in-context examples.
99
+
100
+ Args:
101
+ transformer ([`FluxTransformer2DModel`]):
102
+ Conditional Transformer (MMDiT) architecture to denoise the encoded image latents.
103
+ scheduler ([`FlowMatchEulerDiscreteScheduler`]):
104
+ A scheduler to be used in combination with `transformer` to denoise the encoded image latents.
105
+ vae ([`AutoencoderKL`]):
106
+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
107
+ text_encoder ([`CLIPTextModel`]):
108
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
109
+ the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
110
+ text_encoder_2 ([`T5EncoderModel`]):
111
+ [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5EncoderModel), specifically
112
+ the [google/t5-v1_1-xxl](https://huggingface.co/google/t5-v1_1-xxl) variant.
113
+ tokenizer (`CLIPTokenizer`):
114
+ Tokenizer of class
115
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/en/model_doc/clip#transformers.CLIPTokenizer).
116
+ tokenizer_2 (`T5TokenizerFast`):
117
+ Second Tokenizer of class
118
+ [T5TokenizerFast](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5TokenizerFast).
119
+ resolution (`int`, *optional*, defaults to 384):
120
+ The resolution of each image when concatenating images from the query and in-context examples.
121
+ """
122
+
123
+ model_cpu_offload_seq = "text_encoder->text_encoder_2->transformer->vae"
124
+ _optional_components = []
125
+ _callback_tensor_inputs = ["latents", "prompt_embeds"]
126
+
127
+ def __init__(
128
+ self,
129
+ scheduler: FlowMatchEulerDiscreteScheduler,
130
+ vae: AutoencoderKL,
131
+ text_encoder: CLIPTextModel,
132
+ tokenizer: CLIPTokenizer,
133
+ text_encoder_2: T5EncoderModel,
134
+ tokenizer_2: T5TokenizerFast,
135
+ transformer: FluxTransformer2DModel,
136
+ resolution: int = 384,
137
+ ):
138
+ super().__init__()
139
+
140
+ self.register_modules(
141
+ vae=vae,
142
+ text_encoder=text_encoder,
143
+ text_encoder_2=text_encoder_2,
144
+ tokenizer=tokenizer,
145
+ tokenizer_2=tokenizer_2,
146
+ transformer=transformer,
147
+ scheduler=scheduler,
148
+ )
149
+
150
+ self.generation_pipe = VisualClozeGenerationPipeline(
151
+ vae=vae,
152
+ text_encoder=text_encoder,
153
+ text_encoder_2=text_encoder_2,
154
+ tokenizer=tokenizer,
155
+ tokenizer_2=tokenizer_2,
156
+ transformer=transformer,
157
+ scheduler=scheduler,
158
+ resolution=resolution,
159
+ )
160
+ self.upsampling_pipe = VisualClozeUpsamplingPipeline(
161
+ vae=vae,
162
+ text_encoder=text_encoder,
163
+ text_encoder_2=text_encoder_2,
164
+ tokenizer=tokenizer,
165
+ tokenizer_2=tokenizer_2,
166
+ transformer=transformer,
167
+ scheduler=scheduler,
168
+ )
169
+
170
+ def check_inputs(
171
+ self,
172
+ image,
173
+ task_prompt,
174
+ content_prompt,
175
+ upsampling_height,
176
+ upsampling_width,
177
+ strength,
178
+ prompt_embeds=None,
179
+ pooled_prompt_embeds=None,
180
+ callback_on_step_end_tensor_inputs=None,
181
+ max_sequence_length=None,
182
+ ):
183
+ if strength < 0 or strength > 1:
184
+ raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}")
185
+
186
+ if upsampling_height is not None and upsampling_height % (self.vae_scale_factor * 2) != 0:
187
+ logger.warning(
188
+ f"`upsampling_height`has to be divisible by {self.vae_scale_factor * 2} but are {upsampling_height}. Dimensions will be resized accordingly"
189
+ )
190
+ if upsampling_width is not None and upsampling_width % (self.vae_scale_factor * 2) != 0:
191
+ logger.warning(
192
+ f"`upsampling_width` have to be divisible by {self.vae_scale_factor * 2} but are {upsampling_width}. Dimensions will be resized accordingly"
193
+ )
194
+
195
+ if callback_on_step_end_tensor_inputs is not None and not all(
196
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
197
+ ):
198
+ raise ValueError(
199
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
200
+ )
201
+
202
+ # Validate prompt inputs
203
+ if (task_prompt is not None or content_prompt is not None) and prompt_embeds is not None:
204
+ raise ValueError("Cannot provide both text `task_prompt` + `content_prompt` and `prompt_embeds`. ")
205
+
206
+ if task_prompt is None and content_prompt is None and prompt_embeds is None:
207
+ raise ValueError("Must provide either `task_prompt` + `content_prompt` or pre-computed `prompt_embeds`. ")
208
+
209
+ # Validate prompt types and consistency
210
+ if task_prompt is None:
211
+ raise ValueError("`task_prompt` is missing.")
212
+
213
+ if task_prompt is not None and not isinstance(task_prompt, (str, list)):
214
+ raise ValueError(f"`task_prompt` must be str or list, got {type(task_prompt)}")
215
+
216
+ if content_prompt is not None and not isinstance(content_prompt, (str, list)):
217
+ raise ValueError(f"`content_prompt` must be str or list, got {type(content_prompt)}")
218
+
219
+ if isinstance(task_prompt, list) or isinstance(content_prompt, list):
220
+ if not isinstance(task_prompt, list) or not isinstance(content_prompt, list):
221
+ raise ValueError(
222
+ f"`task_prompt` and `content_prompt` must both be lists, or both be of type str or None, "
223
+ f"got {type(task_prompt)} and {type(content_prompt)}"
224
+ )
225
+ if len(content_prompt) != len(task_prompt):
226
+ raise ValueError("`task_prompt` and `content_prompt` must have the same length whe they are lists.")
227
+
228
+ for sample in image:
229
+ if not isinstance(sample, list) or not isinstance(sample[0], list):
230
+ raise ValueError("Each sample in the batch must have a 2D list of images.")
231
+ if len({len(row) for row in sample}) != 1:
232
+ raise ValueError("Each in-context example and query should contain the same number of images.")
233
+ if not any(img is None for img in sample[-1]):
234
+ raise ValueError("There are no targets in the query, which should be represented as None.")
235
+ for row in sample[:-1]:
236
+ if any(img is None for img in row):
237
+ raise ValueError("Images are missing in in-context examples.")
238
+
239
+ # Validate embeddings
240
+ if prompt_embeds is not None and pooled_prompt_embeds is None:
241
+ raise ValueError(
242
+ "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`."
243
+ )
244
+
245
+ # Validate sequence length
246
+ if max_sequence_length is not None and max_sequence_length > 512:
247
+ raise ValueError(f"max_sequence_length cannot exceed 512, got {max_sequence_length}")
248
+
249
+ @torch.no_grad()
250
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
251
+ def __call__(
252
+ self,
253
+ task_prompt: str | list[str] = None,
254
+ content_prompt: str | list[str] = None,
255
+ image: torch.FloatTensor | None = None,
256
+ upsampling_height: int | None = None,
257
+ upsampling_width: int | None = None,
258
+ num_inference_steps: int = 50,
259
+ sigmas: list[float] | None = None,
260
+ guidance_scale: float = 30.0,
261
+ num_images_per_prompt: int | None = 1,
262
+ generator: torch.Generator | list[torch.Generator] | None = None,
263
+ latents: torch.FloatTensor | None = None,
264
+ prompt_embeds: torch.FloatTensor | None = None,
265
+ pooled_prompt_embeds: torch.FloatTensor | None = None,
266
+ output_type: str | None = "pil",
267
+ return_dict: bool = True,
268
+ joint_attention_kwargs: dict[str, Any] | None = None,
269
+ callback_on_step_end: Callable[[int, int], None] | None = None,
270
+ callback_on_step_end_tensor_inputs: list[str] = ["latents"],
271
+ max_sequence_length: int = 512,
272
+ upsampling_strength: float = 1.0,
273
+ ):
274
+ r"""
275
+ Function invoked when calling the VisualCloze pipeline for generation.
276
+
277
+ Args:
278
+ task_prompt (`str` or `list[str]`, *optional*):
279
+ The prompt or prompts to define the task intention.
280
+ content_prompt (`str` or `list[str]`, *optional*):
281
+ The prompt or prompts to define the content or caption of the target image to be generated.
282
+ image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`):
283
+ `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both
284
+ numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list
285
+ or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a
286
+ list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)`.
287
+ upsampling_height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
288
+ The height in pixels of the generated image (i.e., output image) after upsampling via SDEdit. By
289
+ default, the image is upsampled by a factor of three, and the base resolution is determined by the
290
+ resolution parameter of the pipeline. When only one of `upsampling_height` or `upsampling_width` is
291
+ specified, the other will be automatically set based on the aspect ratio.
292
+ upsampling_width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
293
+ The width in pixels of the generated image (i.e., output image) after upsampling via SDEdit. By
294
+ default, the image is upsampled by a factor of three, and the base resolution is determined by the
295
+ resolution parameter of the pipeline. When only one of `upsampling_height` or `upsampling_width` is
296
+ specified, the other will be automatically set based on the aspect ratio.
297
+ num_inference_steps (`int`, *optional*, defaults to 50):
298
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
299
+ expense of slower inference.
300
+ sigmas (`list[float]`, *optional*):
301
+ Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
302
+ their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
303
+ will be used.
304
+ guidance_scale (`float`, *optional*, defaults to 30.0):
305
+ Guidance scale as defined in [Classifier-Free Diffusion
306
+ Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2.
307
+ of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
308
+ `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to
309
+ the text `prompt`, usually at the expense of lower image quality.
310
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
311
+ The number of images to generate per prompt.
312
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
313
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
314
+ to make generation deterministic.
315
+ latents (`torch.FloatTensor`, *optional*):
316
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
317
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
318
+ tensor will be generated by sampling using the supplied random `generator`.
319
+ prompt_embeds (`torch.FloatTensor`, *optional*):
320
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
321
+ provided, text embeddings will be generated from `prompt` input argument.
322
+ pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
323
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
324
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
325
+ output_type (`str`, *optional*, defaults to `"pil"`):
326
+ The output format of the generate image. Choose between
327
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
328
+ return_dict (`bool`, *optional*, defaults to `True`):
329
+ Whether or not to return a [`~pipelines.flux.FluxPipelineOutput`] instead of a plain tuple.
330
+ joint_attention_kwargs (`dict`, *optional*):
331
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
332
+ `self.processor` in
333
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
334
+ callback_on_step_end (`Callable`, *optional*):
335
+ A function that calls at the end of each denoising steps during the inference. The function is called
336
+ with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
337
+ callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
338
+ `callback_on_step_end_tensor_inputs`.
339
+ callback_on_step_end_tensor_inputs (`list`, *optional*):
340
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
341
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
342
+ `._callback_tensor_inputs` attribute of your pipeline class.
343
+ max_sequence_length (`int` defaults to 512): Maximum sequence length to use with the `prompt`.
344
+ upsampling_strength (`float`, *optional*, defaults to 1.0):
345
+ Indicates extent to transform the reference `image` when upsampling the results. Must be between 0 and
346
+ 1. The generated image is used as a starting point and more noise is added the higher the
347
+ `upsampling_strength`. The number of denoising steps depends on the amount of noise initially added.
348
+ When `upsampling_strength` is 1, added noise is maximum and the denoising process runs for the full
349
+ number of iterations specified in `num_inference_steps`. A value of 0 skips the upsampling step and
350
+ output the results at the resolution of `self.resolution`.
351
+
352
+ Examples:
353
+
354
+ Returns:
355
+ [`~pipelines.flux.FluxPipelineOutput`] or `tuple`: [`~pipelines.flux.FluxPipelineOutput`] if `return_dict`
356
+ is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the generated
357
+ images.
358
+ """
359
+
360
+ generation_output = self.generation_pipe(
361
+ task_prompt=task_prompt,
362
+ content_prompt=content_prompt,
363
+ image=image,
364
+ num_inference_steps=num_inference_steps,
365
+ sigmas=sigmas,
366
+ guidance_scale=guidance_scale,
367
+ num_images_per_prompt=num_images_per_prompt,
368
+ generator=generator,
369
+ latents=latents,
370
+ prompt_embeds=prompt_embeds,
371
+ pooled_prompt_embeds=pooled_prompt_embeds,
372
+ joint_attention_kwargs=joint_attention_kwargs,
373
+ callback_on_step_end=callback_on_step_end,
374
+ callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs,
375
+ max_sequence_length=max_sequence_length,
376
+ output_type=output_type if upsampling_strength == 0 else "pil",
377
+ )
378
+ if upsampling_strength == 0:
379
+ if not return_dict:
380
+ return (generation_output,)
381
+
382
+ return FluxPipelineOutput(images=generation_output)
383
+
384
+ # Upsampling the generated images
385
+ # 1. Prepare the input images and prompts
386
+ if not isinstance(content_prompt, (list)):
387
+ content_prompt = [content_prompt]
388
+ n_target_per_sample = []
389
+ upsampling_image = []
390
+ upsampling_mask = []
391
+ upsampling_prompt = []
392
+ upsampling_generator = generator if isinstance(generator, (torch.Generator,)) else []
393
+ for i in range(len(generation_output.images)):
394
+ n_target_per_sample.append(len(generation_output.images[i]))
395
+ for image in generation_output.images[i]:
396
+ upsampling_image.append(image)
397
+ upsampling_mask.append(Image.new("RGB", image.size, (255, 255, 255)))
398
+ upsampling_prompt.append(
399
+ content_prompt[i % len(content_prompt)] if content_prompt[i % len(content_prompt)] else ""
400
+ )
401
+ if not isinstance(generator, (torch.Generator,)):
402
+ upsampling_generator.append(generator[i % len(content_prompt)])
403
+
404
+ # 2. Apply the denosing loop
405
+ upsampling_output = self.upsampling_pipe(
406
+ prompt=upsampling_prompt,
407
+ image=upsampling_image,
408
+ mask_image=upsampling_mask,
409
+ height=upsampling_height,
410
+ width=upsampling_width,
411
+ strength=upsampling_strength,
412
+ num_inference_steps=num_inference_steps,
413
+ sigmas=sigmas,
414
+ guidance_scale=guidance_scale,
415
+ generator=upsampling_generator,
416
+ output_type=output_type,
417
+ joint_attention_kwargs=joint_attention_kwargs,
418
+ callback_on_step_end=callback_on_step_end,
419
+ callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs,
420
+ max_sequence_length=max_sequence_length,
421
+ )
422
+ image = upsampling_output.images
423
+
424
+ output = []
425
+ if output_type == "pil":
426
+ # Each sample in the batch may have multiple output images. When returning as PIL images,
427
+ # these images cannot be concatenated. Therefore, for each sample,
428
+ # a list is used to represent all the output images.
429
+ output = []
430
+ start = 0
431
+ for n in n_target_per_sample:
432
+ output.append(image[start : start + n])
433
+ start += n
434
+ else:
435
+ output = image
436
+
437
+ if not return_dict:
438
+ return (output,)
439
+
440
+ return FluxPipelineOutput(images=output)
western_ai/europe/diffusers/src/diffusers/pipelines/visualcloze/pipeline_visualcloze_generation.py ADDED
@@ -0,0 +1,977 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 VisualCloze team and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from typing import Any, Callable
16
+
17
+ import numpy as np
18
+ import torch
19
+ from transformers import CLIPTextModel, CLIPTokenizer, T5EncoderModel, T5TokenizerFast
20
+
21
+ from ...loaders import FluxLoraLoaderMixin, FromSingleFileMixin, TextualInversionLoaderMixin
22
+ from ...models.autoencoders import AutoencoderKL
23
+ from ...models.transformers import FluxTransformer2DModel
24
+ from ...schedulers import FlowMatchEulerDiscreteScheduler
25
+ from ...utils import (
26
+ USE_PEFT_BACKEND,
27
+ deprecate,
28
+ is_torch_xla_available,
29
+ logging,
30
+ replace_example_docstring,
31
+ scale_lora_layers,
32
+ unscale_lora_layers,
33
+ )
34
+ from ...utils.torch_utils import randn_tensor
35
+ from ..flux.pipeline_flux_fill import calculate_shift, retrieve_latents, retrieve_timesteps
36
+ from ..flux.pipeline_output import FluxPipelineOutput
37
+ from ..pipeline_utils import DiffusionPipeline
38
+ from .visualcloze_utils import VisualClozeProcessor
39
+
40
+
41
+ if is_torch_xla_available():
42
+ import torch_xla.core.xla_model as xm
43
+
44
+ XLA_AVAILABLE = True
45
+ else:
46
+ XLA_AVAILABLE = False
47
+
48
+
49
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
50
+
51
+
52
+ EXAMPLE_DOC_STRING = """
53
+ Examples:
54
+ ```python
55
+ >>> import torch
56
+ >>> from diffusers import VisualClozeGenerationPipeline, FluxFillPipeline as VisualClozeUpsamplingPipeline
57
+ >>> from diffusers.utils import load_image
58
+ >>> from PIL import Image
59
+
60
+ >>> image_paths = [
61
+ ... # in-context examples
62
+ ... [
63
+ ... load_image(
64
+ ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/visualcloze/visualcloze_mask2image_incontext-example-1_mask.jpg"
65
+ ... ),
66
+ ... load_image(
67
+ ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/visualcloze/visualcloze_mask2image_incontext-example-1_image.jpg"
68
+ ... ),
69
+ ... ],
70
+ ... # query with the target image
71
+ ... [
72
+ ... load_image(
73
+ ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/visualcloze/visualcloze_mask2image_query_mask.jpg"
74
+ ... ),
75
+ ... None, # No image needed for the target image
76
+ ... ],
77
+ ... ]
78
+ >>> task_prompt = "In each row, a logical task is demonstrated to achieve [IMAGE2] an aesthetically pleasing photograph based on [IMAGE1] sam 2-generated masks with rich color coding."
79
+ >>> content_prompt = "Majestic photo of a golden eagle perched on a rocky outcrop in a mountainous landscape. The eagle is positioned in the right foreground, facing left, with its sharp beak and keen eyes prominently visible. Its plumage is a mix of dark brown and golden hues, with intricate feather details. The background features a soft-focus view of snow-capped mountains under a cloudy sky, creating a serene and grandiose atmosphere. The foreground includes rugged rocks and patches of green moss. Photorealistic, medium depth of field, soft natural lighting, cool color palette, high contrast, sharp focus on the eagle, blurred background, tranquil, majestic, wildlife photography."
80
+ >>> pipe = VisualClozeGenerationPipeline.from_pretrained(
81
+ ... "VisualCloze/VisualClozePipeline-384", resolution=384, torch_dtype=torch.bfloat16
82
+ ... )
83
+ >>> pipe.to("cuda")
84
+
85
+ >>> image = pipe(
86
+ ... task_prompt=task_prompt,
87
+ ... content_prompt=content_prompt,
88
+ ... image=image_paths,
89
+ ... guidance_scale=30,
90
+ ... num_inference_steps=30,
91
+ ... max_sequence_length=512,
92
+ ... generator=torch.Generator("cpu").manual_seed(0),
93
+ ... ).images[0][0]
94
+
95
+ >>> # optional, upsampling the generated image
96
+ >>> pipe_upsample = VisualClozeUpsamplingPipeline.from_pipe(pipe)
97
+ >>> pipe_upsample.to("cuda")
98
+
99
+ >>> mask_image = Image.new("RGB", image.size, (255, 255, 255))
100
+
101
+ >>> image = pipe_upsample(
102
+ ... image=image,
103
+ ... mask_image=mask_image,
104
+ ... prompt=content_prompt,
105
+ ... width=1344,
106
+ ... height=768,
107
+ ... strength=0.4,
108
+ ... guidance_scale=30,
109
+ ... num_inference_steps=30,
110
+ ... max_sequence_length=512,
111
+ ... generator=torch.Generator("cpu").manual_seed(0),
112
+ ... ).images[0]
113
+
114
+ >>> image.save("visualcloze.png")
115
+ ```
116
+ """
117
+
118
+
119
+ class VisualClozeGenerationPipeline(
120
+ DiffusionPipeline,
121
+ FluxLoraLoaderMixin,
122
+ FromSingleFileMixin,
123
+ TextualInversionLoaderMixin,
124
+ ):
125
+ r"""
126
+ The VisualCloze pipeline for image generation with visual context. Reference:
127
+ https://github.com/lzyhha/VisualCloze/tree/main This pipeline is designed to generate images based on visual
128
+ in-context examples.
129
+
130
+ Args:
131
+ transformer ([`FluxTransformer2DModel`]):
132
+ Conditional Transformer (MMDiT) architecture to denoise the encoded image latents.
133
+ scheduler ([`FlowMatchEulerDiscreteScheduler`]):
134
+ A scheduler to be used in combination with `transformer` to denoise the encoded image latents.
135
+ vae ([`AutoencoderKL`]):
136
+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
137
+ text_encoder ([`CLIPTextModel`]):
138
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
139
+ the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
140
+ text_encoder_2 ([`T5EncoderModel`]):
141
+ [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5EncoderModel), specifically
142
+ the [google/t5-v1_1-xxl](https://huggingface.co/google/t5-v1_1-xxl) variant.
143
+ tokenizer (`CLIPTokenizer`):
144
+ Tokenizer of class
145
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/en/model_doc/clip#transformers.CLIPTokenizer).
146
+ tokenizer_2 (`T5TokenizerFast`):
147
+ Second Tokenizer of class
148
+ [T5TokenizerFast](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5TokenizerFast).
149
+ resolution (`int`, *optional*, defaults to 384):
150
+ The resolution of each image when concatenating images from the query and in-context examples.
151
+ """
152
+
153
+ model_cpu_offload_seq = "text_encoder->text_encoder_2->transformer->vae"
154
+ _optional_components = []
155
+ _callback_tensor_inputs = ["latents", "prompt_embeds"]
156
+
157
+ def __init__(
158
+ self,
159
+ scheduler: FlowMatchEulerDiscreteScheduler,
160
+ vae: AutoencoderKL,
161
+ text_encoder: CLIPTextModel,
162
+ tokenizer: CLIPTokenizer,
163
+ text_encoder_2: T5EncoderModel,
164
+ tokenizer_2: T5TokenizerFast,
165
+ transformer: FluxTransformer2DModel,
166
+ resolution: int = 384,
167
+ ):
168
+ super().__init__()
169
+
170
+ self.register_modules(
171
+ vae=vae,
172
+ text_encoder=text_encoder,
173
+ text_encoder_2=text_encoder_2,
174
+ tokenizer=tokenizer,
175
+ tokenizer_2=tokenizer_2,
176
+ transformer=transformer,
177
+ scheduler=scheduler,
178
+ )
179
+ self.resolution = resolution
180
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
181
+ # Flux latents are turned into 2x2 patches and packed. This means the latent width and height has to be divisible
182
+ # by the patch size. So the vae scale factor is multiplied by the patch size to account for this
183
+ self.latent_channels = self.vae.config.latent_channels if getattr(self, "vae", None) else 16
184
+ self.image_processor = VisualClozeProcessor(
185
+ vae_scale_factor=self.vae_scale_factor * 2, vae_latent_channels=self.latent_channels, resolution=resolution
186
+ )
187
+ self.tokenizer_max_length = (
188
+ self.tokenizer.model_max_length if hasattr(self, "tokenizer") and self.tokenizer is not None else 77
189
+ )
190
+ self.default_sample_size = 128
191
+
192
+ # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_t5_prompt_embeds
193
+ def _get_t5_prompt_embeds(
194
+ self,
195
+ prompt: str | list[str] = None,
196
+ num_images_per_prompt: int = 1,
197
+ max_sequence_length: int = 512,
198
+ device: torch.device | None = None,
199
+ dtype: torch.dtype | None = None,
200
+ ):
201
+ device = device or self._execution_device
202
+ dtype = dtype or self.text_encoder.dtype
203
+
204
+ prompt = [prompt] if isinstance(prompt, str) else prompt
205
+ batch_size = len(prompt)
206
+
207
+ if isinstance(self, TextualInversionLoaderMixin):
208
+ prompt = self.maybe_convert_prompt(prompt, self.tokenizer_2)
209
+
210
+ text_inputs = self.tokenizer_2(
211
+ prompt,
212
+ padding="max_length",
213
+ max_length=max_sequence_length,
214
+ truncation=True,
215
+ return_length=False,
216
+ return_overflowing_tokens=False,
217
+ return_tensors="pt",
218
+ )
219
+ text_input_ids = text_inputs.input_ids
220
+ untruncated_ids = self.tokenizer_2(prompt, padding="longest", return_tensors="pt").input_ids
221
+
222
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids):
223
+ removed_text = self.tokenizer_2.batch_decode(untruncated_ids[:, self.tokenizer_max_length - 1 : -1])
224
+ logger.warning(
225
+ "The following part of your input was truncated because `max_sequence_length` is set to "
226
+ f" {max_sequence_length} tokens: {removed_text}"
227
+ )
228
+
229
+ prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0]
230
+
231
+ dtype = self.text_encoder_2.dtype
232
+ prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
233
+
234
+ _, seq_len, _ = prompt_embeds.shape
235
+
236
+ # duplicate text embeddings and attention mask for each generation per prompt, using mps friendly method
237
+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
238
+ prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
239
+
240
+ return prompt_embeds
241
+
242
+ # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_clip_prompt_embeds
243
+ def _get_clip_prompt_embeds(
244
+ self,
245
+ prompt: str | list[str],
246
+ num_images_per_prompt: int = 1,
247
+ device: torch.device | None = None,
248
+ ):
249
+ device = device or self._execution_device
250
+
251
+ prompt = [prompt] if isinstance(prompt, str) else prompt
252
+ batch_size = len(prompt)
253
+
254
+ if isinstance(self, TextualInversionLoaderMixin):
255
+ prompt = self.maybe_convert_prompt(prompt, self.tokenizer)
256
+
257
+ text_inputs = self.tokenizer(
258
+ prompt,
259
+ padding="max_length",
260
+ max_length=self.tokenizer_max_length,
261
+ truncation=True,
262
+ return_overflowing_tokens=False,
263
+ return_length=False,
264
+ return_tensors="pt",
265
+ )
266
+
267
+ text_input_ids = text_inputs.input_ids
268
+ untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
269
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids):
270
+ removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer_max_length - 1 : -1])
271
+ logger.warning(
272
+ "The following part of your input was truncated because CLIP can only handle sequences up to"
273
+ f" {self.tokenizer_max_length} tokens: {removed_text}"
274
+ )
275
+ prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False)
276
+
277
+ # Use pooled output of CLIPTextModel
278
+ prompt_embeds = prompt_embeds.pooler_output
279
+ prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
280
+
281
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
282
+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt)
283
+ prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, -1)
284
+
285
+ return prompt_embeds
286
+
287
+ # Modified from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.encode_prompt
288
+ def encode_prompt(
289
+ self,
290
+ layout_prompt: str | list[str],
291
+ task_prompt: str | list[str],
292
+ content_prompt: str | list[str],
293
+ device: torch.device | None = None,
294
+ num_images_per_prompt: int = 1,
295
+ prompt_embeds: torch.FloatTensor | None = None,
296
+ pooled_prompt_embeds: torch.FloatTensor | None = None,
297
+ max_sequence_length: int = 512,
298
+ lora_scale: float | None = None,
299
+ ):
300
+ r"""
301
+
302
+ Args:
303
+ layout_prompt (`str` or `list[str]`, *optional*):
304
+ The prompt or prompts to define the number of in-context examples and the number of images involved in
305
+ the task.
306
+ task_prompt (`str` or `list[str]`, *optional*):
307
+ The prompt or prompts to define the task intention.
308
+ content_prompt (`str` or `list[str]`, *optional*):
309
+ The prompt or prompts to define the content or caption of the target image to be generated.
310
+ device: (`torch.device`):
311
+ torch device
312
+ num_images_per_prompt (`int`):
313
+ number of images that should be generated per prompt
314
+ prompt_embeds (`torch.FloatTensor`, *optional*):
315
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
316
+ provided, text embeddings will be generated from `prompt` input argument.
317
+ pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
318
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
319
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
320
+ lora_scale (`float`, *optional*):
321
+ A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
322
+ """
323
+ device = device or self._execution_device
324
+
325
+ # set lora scale so that monkey patched LoRA
326
+ # function of text encoder can correctly access it
327
+ if lora_scale is not None and isinstance(self, FluxLoraLoaderMixin):
328
+ self._lora_scale = lora_scale
329
+
330
+ # dynamically adjust the LoRA scale
331
+ if self.text_encoder is not None and USE_PEFT_BACKEND:
332
+ scale_lora_layers(self.text_encoder, lora_scale)
333
+ if self.text_encoder_2 is not None and USE_PEFT_BACKEND:
334
+ scale_lora_layers(self.text_encoder_2, lora_scale)
335
+
336
+ if isinstance(layout_prompt, str):
337
+ layout_prompt = [layout_prompt]
338
+ task_prompt = [task_prompt]
339
+ content_prompt = [content_prompt]
340
+
341
+ def _preprocess(prompt, content=False):
342
+ if prompt is not None:
343
+ return f"The last image of the last row depicts: {prompt}" if content else prompt
344
+ else:
345
+ return ""
346
+
347
+ prompt = [
348
+ f"{_preprocess(layout_prompt[i])} {_preprocess(task_prompt[i])} {_preprocess(content_prompt[i], content=True)}".strip()
349
+ for i in range(len(layout_prompt))
350
+ ]
351
+ pooled_prompt_embeds = self._get_clip_prompt_embeds(
352
+ prompt=prompt,
353
+ device=device,
354
+ num_images_per_prompt=num_images_per_prompt,
355
+ )
356
+ prompt_embeds = self._get_t5_prompt_embeds(
357
+ prompt=prompt,
358
+ num_images_per_prompt=num_images_per_prompt,
359
+ max_sequence_length=max_sequence_length,
360
+ device=device,
361
+ )
362
+
363
+ if self.text_encoder is not None:
364
+ if isinstance(self, FluxLoraLoaderMixin) and USE_PEFT_BACKEND:
365
+ # Retrieve the original scale by scaling back the LoRA layers
366
+ unscale_lora_layers(self.text_encoder, lora_scale)
367
+
368
+ if self.text_encoder_2 is not None:
369
+ if isinstance(self, FluxLoraLoaderMixin) and USE_PEFT_BACKEND:
370
+ # Retrieve the original scale by scaling back the LoRA layers
371
+ unscale_lora_layers(self.text_encoder_2, lora_scale)
372
+
373
+ dtype = self.text_encoder.dtype if self.text_encoder is not None else self.transformer.dtype
374
+ text_ids = torch.zeros(prompt_embeds.shape[1], 3).to(device=device, dtype=dtype)
375
+
376
+ return prompt_embeds, pooled_prompt_embeds, text_ids
377
+
378
+ # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3_inpaint.StableDiffusion3InpaintPipeline._encode_vae_image
379
+ def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator):
380
+ if isinstance(generator, list):
381
+ image_latents = [
382
+ retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i])
383
+ for i in range(image.shape[0])
384
+ ]
385
+ image_latents = torch.cat(image_latents, dim=0)
386
+ else:
387
+ image_latents = retrieve_latents(self.vae.encode(image), generator=generator)
388
+
389
+ image_latents = (image_latents - self.vae.config.shift_factor) * self.vae.config.scaling_factor
390
+
391
+ return image_latents
392
+
393
+ # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3_img2img.StableDiffusion3Img2ImgPipeline.get_timesteps
394
+ def get_timesteps(self, num_inference_steps, strength, device):
395
+ # get the original timestep using init_timestep
396
+ init_timestep = min(num_inference_steps * strength, num_inference_steps)
397
+
398
+ t_start = int(max(num_inference_steps - init_timestep, 0))
399
+ timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :]
400
+ if hasattr(self.scheduler, "set_begin_index"):
401
+ self.scheduler.set_begin_index(t_start * self.scheduler.order)
402
+
403
+ return timesteps, num_inference_steps - t_start
404
+
405
+ def check_inputs(
406
+ self,
407
+ image,
408
+ task_prompt,
409
+ content_prompt,
410
+ prompt_embeds=None,
411
+ pooled_prompt_embeds=None,
412
+ callback_on_step_end_tensor_inputs=None,
413
+ max_sequence_length=None,
414
+ ):
415
+ if callback_on_step_end_tensor_inputs is not None and not all(
416
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
417
+ ):
418
+ raise ValueError(
419
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
420
+ )
421
+
422
+ # Validate prompt inputs
423
+ if (task_prompt is not None or content_prompt is not None) and prompt_embeds is not None:
424
+ raise ValueError("Cannot provide both text `task_prompt` + `content_prompt` and `prompt_embeds`. ")
425
+
426
+ if task_prompt is None and content_prompt is None and prompt_embeds is None:
427
+ raise ValueError("Must provide either `task_prompt` + `content_prompt` or pre-computed `prompt_embeds`. ")
428
+
429
+ # Validate prompt types and consistency
430
+ if task_prompt is None:
431
+ raise ValueError("`task_prompt` is missing.")
432
+
433
+ if task_prompt is not None and not isinstance(task_prompt, (str, list)):
434
+ raise ValueError(f"`task_prompt` must be str or list, got {type(task_prompt)}")
435
+
436
+ if content_prompt is not None and not isinstance(content_prompt, (str, list)):
437
+ raise ValueError(f"`content_prompt` must be str or list, got {type(content_prompt)}")
438
+
439
+ if isinstance(task_prompt, list) or isinstance(content_prompt, list):
440
+ if not isinstance(task_prompt, list) or not isinstance(content_prompt, list):
441
+ raise ValueError(
442
+ f"`task_prompt` and `content_prompt` must both be lists, or both be of type str or None, "
443
+ f"got {type(task_prompt)} and {type(content_prompt)}"
444
+ )
445
+ if len(content_prompt) != len(task_prompt):
446
+ raise ValueError("`task_prompt` and `content_prompt` must have the same length whe they are lists.")
447
+
448
+ for sample in image:
449
+ if not isinstance(sample, list) or not isinstance(sample[0], list):
450
+ raise ValueError("Each sample in the batch must have a 2D list of images.")
451
+ if len({len(row) for row in sample}) != 1:
452
+ raise ValueError("Each in-context example and query should contain the same number of images.")
453
+ if not any(img is None for img in sample[-1]):
454
+ raise ValueError("There are no targets in the query, which should be represented as None.")
455
+ for row in sample[:-1]:
456
+ if any(img is None for img in row):
457
+ raise ValueError("Images are missing in in-context examples.")
458
+
459
+ # Validate embeddings
460
+ if prompt_embeds is not None and pooled_prompt_embeds is None:
461
+ raise ValueError(
462
+ "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`."
463
+ )
464
+
465
+ # Validate sequence length
466
+ if max_sequence_length is not None and max_sequence_length > 512:
467
+ raise ValueError(f"max_sequence_length cannot exceed 512, got {max_sequence_length}")
468
+
469
+ @staticmethod
470
+ def _prepare_latent_image_ids(image, vae_scale_factor, device, dtype):
471
+ latent_image_ids = []
472
+
473
+ for idx, img in enumerate(image, start=1):
474
+ img = img.squeeze(0)
475
+ channels, height, width = img.shape
476
+
477
+ num_patches_h = height // vae_scale_factor // 2
478
+ num_patches_w = width // vae_scale_factor // 2
479
+
480
+ patch_ids = torch.zeros(num_patches_h, num_patches_w, 3, device=device, dtype=dtype)
481
+ patch_ids[..., 0] = idx
482
+ patch_ids[..., 1] = torch.arange(num_patches_h, device=device, dtype=dtype)[:, None]
483
+ patch_ids[..., 2] = torch.arange(num_patches_w, device=device, dtype=dtype)[None, :]
484
+
485
+ patch_ids = patch_ids.reshape(-1, 3)
486
+ latent_image_ids.append(patch_ids)
487
+
488
+ return torch.cat(latent_image_ids, dim=0)
489
+
490
+ @staticmethod
491
+ # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._pack_latents
492
+ def _pack_latents(latents, batch_size, num_channels_latents, height, width):
493
+ latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2)
494
+ latents = latents.permute(0, 2, 4, 1, 3, 5)
495
+ latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4)
496
+
497
+ return latents
498
+
499
+ @staticmethod
500
+ def _unpack_latents(latents, sizes, vae_scale_factor):
501
+ batch_size, num_patches, channels = latents.shape
502
+
503
+ start = 0
504
+ unpacked_latents = []
505
+ for i in range(len(sizes)):
506
+ cur_size = sizes[i]
507
+ height = cur_size[0][0] // vae_scale_factor
508
+ width = sum([size[1] for size in cur_size]) // vae_scale_factor
509
+
510
+ end = start + (height * width) // 4
511
+
512
+ cur_latents = latents[:, start:end]
513
+ cur_latents = cur_latents.view(batch_size, height // 2, width // 2, channels // 4, 2, 2)
514
+ cur_latents = cur_latents.permute(0, 3, 1, 4, 2, 5)
515
+ cur_latents = cur_latents.reshape(batch_size, channels // (2 * 2), height, width)
516
+
517
+ unpacked_latents.append(cur_latents)
518
+
519
+ start = end
520
+
521
+ return unpacked_latents
522
+
523
+ def enable_vae_slicing(self):
524
+ r"""
525
+ Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to
526
+ compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.
527
+ """
528
+ depr_message = f"Calling `enable_vae_slicing()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.enable_slicing()`."
529
+ deprecate(
530
+ "enable_vae_slicing",
531
+ "0.40.0",
532
+ depr_message,
533
+ )
534
+ self.vae.enable_slicing()
535
+
536
+ def disable_vae_slicing(self):
537
+ r"""
538
+ Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to
539
+ computing decoding in one step.
540
+ """
541
+ depr_message = f"Calling `disable_vae_slicing()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.disable_slicing()`."
542
+ deprecate(
543
+ "disable_vae_slicing",
544
+ "0.40.0",
545
+ depr_message,
546
+ )
547
+ self.vae.disable_slicing()
548
+
549
+ def enable_vae_tiling(self):
550
+ r"""
551
+ Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to
552
+ compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow
553
+ processing larger images.
554
+ """
555
+ depr_message = f"Calling `enable_vae_tiling()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.enable_tiling()`."
556
+ deprecate(
557
+ "enable_vae_tiling",
558
+ "0.40.0",
559
+ depr_message,
560
+ )
561
+ self.vae.enable_tiling()
562
+
563
+ def disable_vae_tiling(self):
564
+ r"""
565
+ Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to
566
+ computing decoding in one step.
567
+ """
568
+ depr_message = f"Calling `disable_vae_tiling()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.disable_tiling()`."
569
+ deprecate(
570
+ "disable_vae_tiling",
571
+ "0.40.0",
572
+ depr_message,
573
+ )
574
+ self.vae.disable_tiling()
575
+
576
+ def _prepare_latents(self, image, mask, gen, vae_scale_factor, device, dtype):
577
+ """Helper function to prepare latents for a single batch."""
578
+ # Concatenate images and masks along width dimension
579
+ image = [torch.cat(img, dim=3).to(device=device, dtype=dtype) for img in image]
580
+ mask = [torch.cat(m, dim=3).to(device=device, dtype=dtype) for m in mask]
581
+
582
+ # Generate latent image IDs
583
+ latent_image_ids = self._prepare_latent_image_ids(image, vae_scale_factor, device, dtype)
584
+
585
+ # For initial encoding, use actual images
586
+ image_latent = [self._encode_vae_image(img, gen) for img in image]
587
+ masked_image_latent = [img.clone() for img in image_latent]
588
+
589
+ for i in range(len(image_latent)):
590
+ # Rearrange latents and masks for patch processing
591
+ num_channels_latents, height, width = image_latent[i].shape[1:]
592
+ image_latent[i] = self._pack_latents(image_latent[i], 1, num_channels_latents, height, width)
593
+ masked_image_latent[i] = self._pack_latents(masked_image_latent[i], 1, num_channels_latents, height, width)
594
+
595
+ # Rearrange masks for patch processing
596
+ num_channels_latents, height, width = mask[i].shape[1:]
597
+ mask[i] = mask[i].view(
598
+ 1,
599
+ num_channels_latents,
600
+ height // vae_scale_factor,
601
+ vae_scale_factor,
602
+ width // vae_scale_factor,
603
+ vae_scale_factor,
604
+ )
605
+ mask[i] = mask[i].permute(0, 1, 3, 5, 2, 4)
606
+ mask[i] = mask[i].reshape(
607
+ 1,
608
+ num_channels_latents * (vae_scale_factor**2),
609
+ height // vae_scale_factor,
610
+ width // vae_scale_factor,
611
+ )
612
+ mask[i] = self._pack_latents(
613
+ mask[i],
614
+ 1,
615
+ num_channels_latents * (vae_scale_factor**2),
616
+ height // vae_scale_factor,
617
+ width // vae_scale_factor,
618
+ )
619
+
620
+ # Concatenate along batch dimension
621
+ image_latent = torch.cat(image_latent, dim=1)
622
+ masked_image_latent = torch.cat(masked_image_latent, dim=1)
623
+ mask = torch.cat(mask, dim=1)
624
+
625
+ return image_latent, masked_image_latent, mask, latent_image_ids
626
+
627
+ def prepare_latents(
628
+ self,
629
+ input_image,
630
+ input_mask,
631
+ timestep,
632
+ batch_size,
633
+ dtype,
634
+ device,
635
+ generator,
636
+ vae_scale_factor,
637
+ ):
638
+ if isinstance(generator, list) and len(generator) != batch_size:
639
+ raise ValueError(
640
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
641
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
642
+ )
643
+
644
+ # Process each batch
645
+ masked_image_latents = []
646
+ image_latents = []
647
+ masks = []
648
+ latent_image_ids = []
649
+
650
+ for i in range(len(input_image)):
651
+ _image_latent, _masked_image_latent, _mask, _latent_image_ids = self._prepare_latents(
652
+ input_image[i],
653
+ input_mask[i],
654
+ generator if isinstance(generator, torch.Generator) else generator[i],
655
+ vae_scale_factor,
656
+ device,
657
+ dtype,
658
+ )
659
+ masked_image_latents.append(_masked_image_latent)
660
+ image_latents.append(_image_latent)
661
+ masks.append(_mask)
662
+ latent_image_ids.append(_latent_image_ids)
663
+
664
+ # Concatenate all batches
665
+ masked_image_latents = torch.cat(masked_image_latents, dim=0)
666
+ image_latents = torch.cat(image_latents, dim=0)
667
+ masks = torch.cat(masks, dim=0)
668
+
669
+ # Handle batch size expansion
670
+ if batch_size > masked_image_latents.shape[0]:
671
+ if batch_size % masked_image_latents.shape[0] == 0:
672
+ # Expand batches by repeating
673
+ additional_image_per_prompt = batch_size // masked_image_latents.shape[0]
674
+ masked_image_latents = torch.cat([masked_image_latents] * additional_image_per_prompt, dim=0)
675
+ image_latents = torch.cat([image_latents] * additional_image_per_prompt, dim=0)
676
+ masks = torch.cat([masks] * additional_image_per_prompt, dim=0)
677
+ else:
678
+ raise ValueError(
679
+ f"Cannot expand batch size from {masked_image_latents.shape[0]} to {batch_size}. "
680
+ "Batch sizes must be multiples of each other."
681
+ )
682
+
683
+ # Add noise to latents
684
+ noises = randn_tensor(image_latents.shape, generator=generator, device=device, dtype=dtype)
685
+ latents = self.scheduler.scale_noise(image_latents, timestep, noises).to(dtype=dtype)
686
+
687
+ # Combine masked latents with masks
688
+ masked_image_latents = torch.cat((masked_image_latents, masks), dim=-1).to(dtype=dtype)
689
+
690
+ return latents, masked_image_latents, latent_image_ids[0]
691
+
692
+ @property
693
+ def guidance_scale(self):
694
+ return self._guidance_scale
695
+
696
+ @property
697
+ def joint_attention_kwargs(self):
698
+ return self._joint_attention_kwargs
699
+
700
+ @property
701
+ def num_timesteps(self):
702
+ return self._num_timesteps
703
+
704
+ @property
705
+ def interrupt(self):
706
+ return self._interrupt
707
+
708
+ @torch.no_grad()
709
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
710
+ def __call__(
711
+ self,
712
+ task_prompt: str | list[str] = None,
713
+ content_prompt: str | list[str] = None,
714
+ image: torch.FloatTensor | None = None,
715
+ num_inference_steps: int = 50,
716
+ sigmas: list[float] | None = None,
717
+ guidance_scale: float = 30.0,
718
+ num_images_per_prompt: int | None = 1,
719
+ generator: torch.Generator | list[torch.Generator] | None = None,
720
+ latents: torch.FloatTensor | None = None,
721
+ prompt_embeds: torch.FloatTensor | None = None,
722
+ pooled_prompt_embeds: torch.FloatTensor | None = None,
723
+ output_type: str | None = "pil",
724
+ return_dict: bool = True,
725
+ joint_attention_kwargs: dict[str, Any] | None = None,
726
+ callback_on_step_end: Callable[[int, int], None] | None = None,
727
+ callback_on_step_end_tensor_inputs: list[str] = ["latents"],
728
+ max_sequence_length: int = 512,
729
+ ):
730
+ r"""
731
+ Function invoked when calling the VisualCloze pipeline for generation.
732
+
733
+ Args:
734
+ task_prompt (`str` or `list[str]`, *optional*):
735
+ The prompt or prompts to define the task intention.
736
+ content_prompt (`str` or `list[str]`, *optional*):
737
+ The prompt or prompts to define the content or caption of the target image to be generated.
738
+ image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`):
739
+ `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both
740
+ numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list
741
+ or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a
742
+ list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)`.
743
+ num_inference_steps (`int`, *optional*, defaults to 50):
744
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
745
+ expense of slower inference.
746
+ sigmas (`list[float]`, *optional*):
747
+ Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
748
+ their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
749
+ will be used.
750
+ guidance_scale (`float`, *optional*, defaults to 30.0):
751
+ Guidance scale as defined in [Classifier-Free Diffusion
752
+ Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2.
753
+ of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
754
+ `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to
755
+ the text `prompt`, usually at the expense of lower image quality.
756
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
757
+ The number of images to generate per prompt.
758
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
759
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
760
+ to make generation deterministic.
761
+ latents (`torch.FloatTensor`, *optional*):
762
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
763
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
764
+ tensor will be generated by sampling using the supplied random `generator`.
765
+ prompt_embeds (`torch.FloatTensor`, *optional*):
766
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
767
+ provided, text embeddings will be generated from `prompt` input argument.
768
+ pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
769
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
770
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
771
+ output_type (`str`, *optional*, defaults to `"pil"`):
772
+ The output format of the generate image. Choose between
773
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
774
+ return_dict (`bool`, *optional*, defaults to `True`):
775
+ Whether or not to return a [`~pipelines.flux.FluxPipelineOutput`] instead of a plain tuple.
776
+ joint_attention_kwargs (`dict`, *optional*):
777
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
778
+ `self.processor` in
779
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
780
+ callback_on_step_end (`Callable`, *optional*):
781
+ A function that calls at the end of each denoising steps during the inference. The function is called
782
+ with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
783
+ callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
784
+ `callback_on_step_end_tensor_inputs`.
785
+ callback_on_step_end_tensor_inputs (`list`, *optional*):
786
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
787
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
788
+ `._callback_tensor_inputs` attribute of your pipeline class.
789
+ max_sequence_length (`int` defaults to 512): Maximum sequence length to use with the `prompt`.
790
+
791
+ Examples:
792
+
793
+ Returns:
794
+ [`~pipelines.flux.FluxPipelineOutput`] or `tuple`: [`~pipelines.flux.FluxPipelineOutput`] if `return_dict`
795
+ is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the generated
796
+ images.
797
+ """
798
+
799
+ # 1. Check inputs. Raise error if not correct
800
+ self.check_inputs(
801
+ image,
802
+ task_prompt,
803
+ content_prompt,
804
+ prompt_embeds=prompt_embeds,
805
+ pooled_prompt_embeds=pooled_prompt_embeds,
806
+ callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs,
807
+ max_sequence_length=max_sequence_length,
808
+ )
809
+
810
+ self._guidance_scale = guidance_scale
811
+ self._joint_attention_kwargs = joint_attention_kwargs
812
+ self._interrupt = False
813
+
814
+ processor_output = self.image_processor.preprocess(
815
+ task_prompt, content_prompt, image, vae_scale_factor=self.vae_scale_factor
816
+ )
817
+
818
+ # 2. Define call parameters
819
+ if processor_output["task_prompt"] is not None and isinstance(processor_output["task_prompt"], str):
820
+ batch_size = 1
821
+ elif processor_output["task_prompt"] is not None and isinstance(processor_output["task_prompt"], list):
822
+ batch_size = len(processor_output["task_prompt"])
823
+
824
+ device = self._execution_device
825
+
826
+ # 3. Prepare prompt embeddings
827
+ lora_scale = (
828
+ self.joint_attention_kwargs.get("scale", None) if self.joint_attention_kwargs is not None else None
829
+ )
830
+ prompt_embeds, pooled_prompt_embeds, text_ids = self.encode_prompt(
831
+ layout_prompt=processor_output["layout_prompt"],
832
+ task_prompt=processor_output["task_prompt"],
833
+ content_prompt=processor_output["content_prompt"],
834
+ prompt_embeds=prompt_embeds,
835
+ pooled_prompt_embeds=pooled_prompt_embeds,
836
+ device=device,
837
+ num_images_per_prompt=num_images_per_prompt,
838
+ max_sequence_length=max_sequence_length,
839
+ lora_scale=lora_scale,
840
+ )
841
+
842
+ # 4. Prepare timesteps
843
+ # Calculate sequence length and shift factor
844
+ image_seq_len = sum(
845
+ (size[0] // self.vae_scale_factor // 2) * (size[1] // self.vae_scale_factor // 2)
846
+ for sample in processor_output["image_size"][0]
847
+ for size in sample
848
+ )
849
+
850
+ # Calculate noise schedule parameters
851
+ mu = calculate_shift(
852
+ image_seq_len,
853
+ self.scheduler.config.get("base_image_seq_len", 256),
854
+ self.scheduler.config.get("max_image_seq_len", 4096),
855
+ self.scheduler.config.get("base_shift", 0.5),
856
+ self.scheduler.config.get("max_shift", 1.15),
857
+ )
858
+
859
+ # Get timesteps
860
+ sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas
861
+ timesteps, num_inference_steps = retrieve_timesteps(
862
+ self.scheduler,
863
+ num_inference_steps,
864
+ device,
865
+ sigmas=sigmas,
866
+ mu=mu,
867
+ )
868
+ timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, 1.0, device)
869
+
870
+ # 5. Prepare latent variables
871
+ latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
872
+ latents, masked_image_latents, latent_image_ids = self.prepare_latents(
873
+ processor_output["init_image"],
874
+ processor_output["mask"],
875
+ latent_timestep,
876
+ batch_size * num_images_per_prompt,
877
+ prompt_embeds.dtype,
878
+ device,
879
+ generator,
880
+ vae_scale_factor=self.vae_scale_factor,
881
+ )
882
+
883
+ # Calculate warmup steps
884
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
885
+ self._num_timesteps = len(timesteps)
886
+
887
+ # Prepare guidance
888
+ if self.transformer.config.guidance_embeds:
889
+ guidance = torch.full([1], guidance_scale, device=device, dtype=torch.float32)
890
+ guidance = guidance.expand(latents.shape[0])
891
+ else:
892
+ guidance = None
893
+
894
+ # 6. Denoising loop
895
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
896
+ for i, t in enumerate(timesteps):
897
+ if self.interrupt:
898
+ continue
899
+
900
+ # Broadcast to batch dimension in a way that's compatible with ONNX/Core ML
901
+ timestep = t.expand(latents.shape[0]).to(latents.dtype)
902
+ latent_model_input = torch.cat((latents, masked_image_latents), dim=2)
903
+
904
+ noise_pred = self.transformer(
905
+ hidden_states=latent_model_input,
906
+ timestep=timestep / 1000,
907
+ guidance=guidance,
908
+ pooled_projections=pooled_prompt_embeds,
909
+ encoder_hidden_states=prompt_embeds,
910
+ txt_ids=text_ids,
911
+ img_ids=latent_image_ids,
912
+ joint_attention_kwargs=self.joint_attention_kwargs,
913
+ return_dict=False,
914
+ )[0]
915
+
916
+ # Compute the previous noisy sample x_t -> x_t-1
917
+ latents_dtype = latents.dtype
918
+ latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
919
+
920
+ if latents.dtype != latents_dtype:
921
+ if torch.backends.mps.is_available():
922
+ # Some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
923
+ latents = latents.to(latents_dtype)
924
+
925
+ if callback_on_step_end is not None:
926
+ callback_kwargs = {}
927
+ for k in callback_on_step_end_tensor_inputs:
928
+ callback_kwargs[k] = locals()[k]
929
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
930
+
931
+ latents = callback_outputs.pop("latents", latents)
932
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
933
+
934
+ # Call the callback, if provided
935
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
936
+ progress_bar.update()
937
+
938
+ # XLA optimization
939
+ if XLA_AVAILABLE:
940
+ xm.mark_step()
941
+
942
+ # 7. Post-process the image
943
+ # Crop the target image
944
+ # Since the generated image is a concatenation of the conditional and target regions,
945
+ # we need to extract only the target regions based on their positions
946
+ image = []
947
+ if output_type == "latent":
948
+ image = latents
949
+ else:
950
+ for b in range(len(latents)):
951
+ cur_image_size = processor_output["image_size"][b % batch_size]
952
+ cur_target_position = processor_output["target_position"][b % batch_size]
953
+ cur_latent = self._unpack_latents(latents[b].unsqueeze(0), cur_image_size, self.vae_scale_factor)[-1]
954
+ cur_latent = (cur_latent / self.vae.config.scaling_factor) + self.vae.config.shift_factor
955
+ cur_image = self.vae.decode(cur_latent, return_dict=False)[0]
956
+ cur_image = self.image_processor.postprocess(cur_image, output_type=output_type)[0]
957
+
958
+ start = 0
959
+ cropped = []
960
+ for i, size in enumerate(cur_image_size[-1]):
961
+ if cur_target_position[i]:
962
+ if output_type == "pil":
963
+ cropped.append(cur_image.crop((start, 0, start + size[1], size[0])))
964
+ else:
965
+ cropped.append(cur_image[0 : size[0], start : start + size[1]])
966
+ start += size[1]
967
+ image.append(cropped)
968
+ if output_type != "pil":
969
+ image = np.concatenate([arr[None] for sub_image in image for arr in sub_image], axis=0)
970
+
971
+ # Offload all models
972
+ self.maybe_free_model_hooks()
973
+
974
+ if not return_dict:
975
+ return (image,)
976
+
977
+ return FluxPipelineOutput(images=image)
western_ai/europe/diffusers/src/diffusers/pipelines/visualcloze/visualcloze_utils.py ADDED
@@ -0,0 +1,250 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 VisualCloze team and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ import torch
17
+ from PIL import Image
18
+
19
+ from ...image_processor import VaeImageProcessor
20
+
21
+
22
+ class VisualClozeProcessor(VaeImageProcessor):
23
+ """
24
+ Image processor for the VisualCloze pipeline.
25
+
26
+ This processor handles the preprocessing of images for visual cloze tasks, including resizing, normalization, and
27
+ mask generation.
28
+
29
+ Args:
30
+ resolution (int, optional):
31
+ Target resolution for processing images. Each image will be resized to this resolution before being
32
+ concatenated to avoid the out-of-memory error. Defaults to 384.
33
+ *args: Additional arguments passed to [~image_processor.VaeImageProcessor]
34
+ **kwargs: Additional keyword arguments passed to [~image_processor.VaeImageProcessor]
35
+ """
36
+
37
+ def __init__(self, *args, resolution: int = 384, **kwargs):
38
+ super().__init__(*args, **kwargs)
39
+ self.resolution = resolution
40
+
41
+ def preprocess_image(
42
+ self, input_images: list[list[Image.Image | None]], vae_scale_factor: int
43
+ ) -> tuple[list[list[torch.Tensor]], list[list[list[int]]], list[int]]:
44
+ """
45
+ Preprocesses input images for the VisualCloze pipeline.
46
+
47
+ This function handles the preprocessing of input images by:
48
+ 1. Resizing and cropping images to maintain consistent dimensions
49
+ 2. Converting images to the Tensor format for the VAE
50
+ 3. Normalizing pixel values
51
+ 4. Tracking image sizes and positions of target images
52
+
53
+ Args:
54
+ input_images (list[list[Image.Image | None]]):
55
+ A nested list of PIL Images where:
56
+ - Outer list represents different samples, including in-context examples and the query
57
+ - Inner list contains images for the task
58
+ - In the last row, condition images are provided and the target images are placed as None
59
+ vae_scale_factor (int):
60
+ The scale factor used by the VAE for resizing images
61
+
62
+ Returns:
63
+ tuple containing:
64
+ - list[list[torch.Tensor]]: Preprocessed images in tensor format
65
+ - list[list[list[int]]]: Dimensions of each processed image [height, width]
66
+ - list[int]: Target positions indicating which images are to be generated
67
+ """
68
+ n_samples, n_task_images = len(input_images), len(input_images[0])
69
+ divisible = 2 * vae_scale_factor
70
+
71
+ processed_images: list[list[Image.Image]] = [[] for _ in range(n_samples)]
72
+ resize_size: list[tuple[int, int] | None] = [None for _ in range(n_samples)]
73
+ target_position: list[int] = []
74
+
75
+ # Process each sample
76
+ for i in range(n_samples):
77
+ # Determine size from first non-None image
78
+ for j in range(n_task_images):
79
+ if input_images[i][j] is not None:
80
+ aspect_ratio = input_images[i][j].width / input_images[i][j].height
81
+ target_area = self.resolution * self.resolution
82
+ new_h = int((target_area / aspect_ratio) ** 0.5)
83
+ new_w = int(new_h * aspect_ratio)
84
+
85
+ new_w = max(new_w // divisible, 1) * divisible
86
+ new_h = max(new_h // divisible, 1) * divisible
87
+ resize_size[i] = (new_w, new_h)
88
+ break
89
+
90
+ # Process all images in the sample
91
+ for j in range(n_task_images):
92
+ if input_images[i][j] is not None:
93
+ target = self._resize_and_crop(input_images[i][j], resize_size[i][0], resize_size[i][1])
94
+ processed_images[i].append(target)
95
+ if i == n_samples - 1:
96
+ target_position.append(0)
97
+ else:
98
+ blank = Image.new("RGB", resize_size[i] or (self.resolution, self.resolution), (0, 0, 0))
99
+ processed_images[i].append(blank)
100
+ if i == n_samples - 1:
101
+ target_position.append(1)
102
+
103
+ # Ensure consistent width for multiple target images when there are multiple target images
104
+ if len(target_position) > 1 and sum(target_position) > 1:
105
+ new_w = resize_size[n_samples - 1][0] or 384
106
+ for i in range(len(processed_images)):
107
+ for j in range(len(processed_images[i])):
108
+ if processed_images[i][j] is not None:
109
+ new_h = int(processed_images[i][j].height * (new_w / processed_images[i][j].width))
110
+ new_w = int(new_w / 16) * 16
111
+ new_h = int(new_h / 16) * 16
112
+ processed_images[i][j] = self._resize_and_crop(processed_images[i][j], new_h, new_w)
113
+
114
+ # Convert to tensors and normalize
115
+ image_sizes = []
116
+ for i in range(len(processed_images)):
117
+ image_sizes.append([[img.height, img.width] for img in processed_images[i]])
118
+ for j, image in enumerate(processed_images[i]):
119
+ image = self.pil_to_numpy(image)
120
+ image = self.numpy_to_pt(image)
121
+ image = self.normalize(image)
122
+ processed_images[i][j] = image
123
+
124
+ return processed_images, image_sizes, target_position
125
+
126
+ def preprocess_mask(
127
+ self, input_images: list[list[Image.Image]], target_position: list[int]
128
+ ) -> list[list[torch.Tensor]]:
129
+ """
130
+ Generate masks for the VisualCloze pipeline.
131
+
132
+ Args:
133
+ input_images (list[list[Image.Image]]):
134
+ Processed images from preprocess_image
135
+ target_position (list[int]):
136
+ Binary list marking the positions of target images (1 for target, 0 for condition)
137
+
138
+ Returns:
139
+ list[list[torch.Tensor]]:
140
+ A nested list of mask tensors (1 for target positions, 0 for condition images)
141
+ """
142
+ mask = []
143
+ for i, row in enumerate(input_images):
144
+ if i == len(input_images) - 1: # Query row
145
+ row_masks = [
146
+ torch.full((1, 1, row[0].shape[2], row[0].shape[3]), fill_value=m) for m in target_position
147
+ ]
148
+ else: # In-context examples
149
+ row_masks = [
150
+ torch.full((1, 1, row[0].shape[2], row[0].shape[3]), fill_value=0) for _ in target_position
151
+ ]
152
+ mask.append(row_masks)
153
+ return mask
154
+
155
+ def preprocess_image_upsampling(
156
+ self,
157
+ input_images: list[list[Image.Image]],
158
+ height: int,
159
+ width: int,
160
+ ) -> tuple[list[list[Image.Image]], list[list[list[int]]]]:
161
+ """Process images for the upsampling stage in the VisualCloze pipeline.
162
+
163
+ Args:
164
+ input_images: Input image to process
165
+ height: Target height
166
+ width: Target width
167
+
168
+ Returns:
169
+ tuple of processed image and its size
170
+ """
171
+ image = self.resize(input_images[0][0], height, width)
172
+ image = self.pil_to_numpy(image) # to np
173
+ image = self.numpy_to_pt(image) # to pt
174
+ image = self.normalize(image)
175
+
176
+ input_images[0][0] = image
177
+ image_sizes = [[[height, width]]]
178
+ return input_images, image_sizes
179
+
180
+ def preprocess_mask_upsampling(self, input_images: list[list[Image.Image]]) -> list[list[torch.Tensor]]:
181
+ return [[torch.ones((1, 1, input_images[0][0].shape[2], input_images[0][0].shape[3]))]]
182
+
183
+ def get_layout_prompt(self, size: tuple[int, int]) -> str:
184
+ layout_instruction = (
185
+ f"A grid layout with {size[0]} rows and {size[1]} columns, displaying {size[0] * size[1]} images arranged side by side.",
186
+ )
187
+ return layout_instruction
188
+
189
+ def preprocess(
190
+ self,
191
+ task_prompt: str | list[str],
192
+ content_prompt: str | list[str],
193
+ input_images: list[list[list[str | None]]] | None = None,
194
+ height: int | None = None,
195
+ width: int | None = None,
196
+ upsampling: bool = False,
197
+ vae_scale_factor: int = 16,
198
+ ) -> dict:
199
+ """Process visual cloze inputs.
200
+
201
+ Args:
202
+ task_prompt: Task description(s)
203
+ content_prompt: Content description(s)
204
+ input_images: list of images or None for the target images
205
+ height: Optional target height for upsampling stage
206
+ width: Optional target width for upsampling stage
207
+ upsampling: Whether this is in the upsampling processing stage
208
+
209
+ Returns:
210
+ dictionary containing processed images, masks, prompts and metadata
211
+ """
212
+ if isinstance(task_prompt, str):
213
+ task_prompt = [task_prompt]
214
+ content_prompt = [content_prompt]
215
+ input_images = [input_images]
216
+
217
+ output = {
218
+ "init_image": [],
219
+ "mask": [],
220
+ "task_prompt": task_prompt if not upsampling else [None for _ in range(len(task_prompt))],
221
+ "content_prompt": content_prompt,
222
+ "layout_prompt": [],
223
+ "target_position": [],
224
+ "image_size": [],
225
+ }
226
+ for i in range(len(task_prompt)):
227
+ if upsampling:
228
+ layout_prompt = None
229
+ else:
230
+ layout_prompt = self.get_layout_prompt((len(input_images[i]), len(input_images[i][0])))
231
+
232
+ if upsampling:
233
+ cur_processed_images, cur_image_size = self.preprocess_image_upsampling(
234
+ input_images[i], height=height, width=width
235
+ )
236
+ cur_mask = self.preprocess_mask_upsampling(cur_processed_images)
237
+ else:
238
+ cur_processed_images, cur_image_size, cur_target_position = self.preprocess_image(
239
+ input_images[i], vae_scale_factor=vae_scale_factor
240
+ )
241
+ cur_mask = self.preprocess_mask(cur_processed_images, cur_target_position)
242
+
243
+ output["target_position"].append(cur_target_position)
244
+
245
+ output["image_size"].append(cur_image_size)
246
+ output["init_image"].append(cur_processed_images)
247
+ output["mask"].append(cur_mask)
248
+ output["layout_prompt"].append(layout_prompt)
249
+
250
+ return output
western_ai/europe/diffusers/src/diffusers/pipelines/wan/__init__.py ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import TYPE_CHECKING
2
+
3
+ from ...utils import (
4
+ DIFFUSERS_SLOW_IMPORT,
5
+ OptionalDependencyNotAvailable,
6
+ _LazyModule,
7
+ get_objects_from_module,
8
+ is_torch_available,
9
+ is_transformers_available,
10
+ )
11
+
12
+
13
+ _dummy_objects = {}
14
+ _import_structure = {}
15
+
16
+
17
+ try:
18
+ if not (is_transformers_available() and is_torch_available()):
19
+ raise OptionalDependencyNotAvailable()
20
+ except OptionalDependencyNotAvailable:
21
+ from ...utils import dummy_torch_and_transformers_objects # noqa F403
22
+
23
+ _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects))
24
+ else:
25
+ _import_structure["pipeline_wan"] = ["WanPipeline"]
26
+ _import_structure["pipeline_wan_animate"] = ["WanAnimatePipeline"]
27
+ _import_structure["pipeline_wan_i2v"] = ["WanImageToVideoPipeline"]
28
+ _import_structure["pipeline_wan_vace"] = ["WanVACEPipeline"]
29
+ _import_structure["pipeline_wan_video2video"] = ["WanVideoToVideoPipeline"]
30
+ if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT:
31
+ try:
32
+ if not (is_transformers_available() and is_torch_available()):
33
+ raise OptionalDependencyNotAvailable()
34
+
35
+ except OptionalDependencyNotAvailable:
36
+ from ...utils.dummy_torch_and_transformers_objects import *
37
+ else:
38
+ from .pipeline_wan import WanPipeline
39
+ from .pipeline_wan_animate import WanAnimatePipeline
40
+ from .pipeline_wan_i2v import WanImageToVideoPipeline
41
+ from .pipeline_wan_vace import WanVACEPipeline
42
+ from .pipeline_wan_video2video import WanVideoToVideoPipeline
43
+ else:
44
+ import sys
45
+
46
+ sys.modules[__name__] = _LazyModule(
47
+ __name__,
48
+ globals()["__file__"],
49
+ _import_structure,
50
+ module_spec=__spec__,
51
+ )
52
+
53
+ for name, value in _dummy_objects.items():
54
+ setattr(sys.modules[__name__], name, value)
western_ai/europe/diffusers/src/diffusers/pipelines/wan/image_processor.py ADDED
@@ -0,0 +1,184 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 The Wan Team and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ import numpy as np
17
+ import PIL.Image
18
+ import torch
19
+
20
+ from ...configuration_utils import register_to_config
21
+ from ...image_processor import VaeImageProcessor
22
+ from ...utils import PIL_INTERPOLATION
23
+
24
+
25
+ class WanAnimateImageProcessor(VaeImageProcessor):
26
+ r"""
27
+ Image processor to preprocess the reference (character) image for the Wan Animate model.
28
+
29
+ Args:
30
+ do_resize (`bool`, *optional*, defaults to `True`):
31
+ Whether to downscale the image's (height, width) dimensions to multiples of `vae_scale_factor`. Can accept
32
+ `height` and `width` arguments from [`image_processor.VaeImageProcessor.preprocess`] method.
33
+ vae_scale_factor (`int`, *optional*, defaults to `8`):
34
+ VAE (spatial) scale factor. If `do_resize` is `True`, the image is automatically resized to multiples of
35
+ this factor.
36
+ vae_latent_channels (`int`, *optional*, defaults to `16`):
37
+ VAE latent channels.
38
+ spatial_patch_size (`tuple[int, int]`, *optional*, defaults to `(2, 2)`):
39
+ The spatial patch size used by the diffusion transformer. For Wan models, this is typically (2, 2).
40
+ resample (`str`, *optional*, defaults to `lanczos`):
41
+ Resampling filter to use when resizing the image.
42
+ do_normalize (`bool`, *optional*, defaults to `True`):
43
+ Whether to normalize the image to [-1,1].
44
+ do_binarize (`bool`, *optional*, defaults to `False`):
45
+ Whether to binarize the image to 0/1.
46
+ do_convert_rgb (`bool`, *optional*, defaults to be `False`):
47
+ Whether to convert the images to RGB format.
48
+ do_convert_grayscale (`bool`, *optional*, defaults to be `False`):
49
+ Whether to convert the images to grayscale format.
50
+ fill_color (`str` or `float` or `tuple[float, ...]`, *optional*, defaults to `None`):
51
+ An optional fill color when `resize_mode` is set to `"fill"`. This will fill the empty space with that
52
+ color instead of filling with data from the image. Any valid `color` argument to `PIL.Image.new` is valid;
53
+ if `None`, will default to filling with data from `image`.
54
+ """
55
+
56
+ @register_to_config
57
+ def __init__(
58
+ self,
59
+ do_resize: bool = True,
60
+ vae_scale_factor: int = 8,
61
+ vae_latent_channels: int = 16,
62
+ spatial_patch_size: tuple[int, int] = (2, 2),
63
+ resample: str = "lanczos",
64
+ reducing_gap: int = None,
65
+ do_normalize: bool = True,
66
+ do_binarize: bool = False,
67
+ do_convert_rgb: bool = False,
68
+ do_convert_grayscale: bool = False,
69
+ fill_color: str | float | tuple[float, ...] | None = 0,
70
+ ):
71
+ super().__init__()
72
+ if do_convert_rgb and do_convert_grayscale:
73
+ raise ValueError(
74
+ "`do_convert_rgb` and `do_convert_grayscale` can not both be set to `True`,"
75
+ " if you intended to convert the image into RGB format, please set `do_convert_grayscale = False`.",
76
+ " if you intended to convert the image into grayscale format, please set `do_convert_rgb = False`",
77
+ )
78
+
79
+ def _resize_and_fill(
80
+ self,
81
+ image: PIL.Image.Image,
82
+ width: int,
83
+ height: int,
84
+ ) -> PIL.Image.Image:
85
+ r"""
86
+ Resize the image to fit within the specified width and height, maintaining the aspect ratio, and then center
87
+ the image within the dimensions, filling empty with data from image.
88
+
89
+ Args:
90
+ image (`PIL.Image.Image`):
91
+ The image to resize and fill.
92
+ width (`int`):
93
+ The width to resize the image to.
94
+ height (`int`):
95
+ The height to resize the image to.
96
+
97
+ Returns:
98
+ `PIL.Image.Image`:
99
+ The resized and filled image.
100
+ """
101
+
102
+ ratio = width / height
103
+ src_ratio = image.width / image.height
104
+ fill_with_image_data = self.config.fill_color is None
105
+ fill_color = self.config.fill_color or 0
106
+
107
+ src_w = width if ratio < src_ratio else image.width * height // image.height
108
+ src_h = height if ratio >= src_ratio else image.height * width // image.width
109
+
110
+ resized = image.resize((src_w, src_h), resample=PIL_INTERPOLATION[self.config.resample])
111
+ res = PIL.Image.new("RGB", (width, height), color=fill_color)
112
+ res.paste(resized, box=(width // 2 - src_w // 2, height // 2 - src_h // 2))
113
+
114
+ if fill_with_image_data:
115
+ if ratio < src_ratio:
116
+ fill_height = height // 2 - src_h // 2
117
+ if fill_height > 0:
118
+ res.paste(resized.resize((width, fill_height), box=(0, 0, width, 0)), box=(0, 0))
119
+ res.paste(
120
+ resized.resize((width, fill_height), box=(0, resized.height, width, resized.height)),
121
+ box=(0, fill_height + src_h),
122
+ )
123
+ elif ratio > src_ratio:
124
+ fill_width = width // 2 - src_w // 2
125
+ if fill_width > 0:
126
+ res.paste(resized.resize((fill_width, height), box=(0, 0, 0, height)), box=(0, 0))
127
+ res.paste(
128
+ resized.resize((fill_width, height), box=(resized.width, 0, resized.width, height)),
129
+ box=(fill_width + src_w, 0),
130
+ )
131
+
132
+ return res
133
+
134
+ def get_default_height_width(
135
+ self,
136
+ image: PIL.Image.Image | np.ndarray | torch.Tensor,
137
+ height: int | None = None,
138
+ width: int | None = None,
139
+ ) -> tuple[int, int]:
140
+ r"""
141
+ Returns the height and width of the image, downscaled to the next integer multiple of `vae_scale_factor`.
142
+
143
+ Args:
144
+ image (`PIL.Image.Image | np.ndarray | torch.Tensor`):
145
+ The image input, which can be a PIL image, NumPy array, or PyTorch tensor. If it is a NumPy array, it
146
+ should have shape `[batch, height, width]` or `[batch, height, width, channels]`. If it is a PyTorch
147
+ tensor, it should have shape `[batch, channels, height, width]`.
148
+ height (`int | None`, *optional*, defaults to `None`):
149
+ The height of the preprocessed image. If `None`, the height of the `image` input will be used.
150
+ width (`int | None`, *optional*, defaults to `None`):
151
+ The width of the preprocessed image. If `None`, the width of the `image` input will be used.
152
+
153
+ Returns:
154
+ `tuple[int, int]`:
155
+ A tuple containing the height and width, both resized to the nearest integer multiple of
156
+ `vae_scale_factor * spatial_patch_size`.
157
+ """
158
+
159
+ if height is None:
160
+ if isinstance(image, PIL.Image.Image):
161
+ height = image.height
162
+ elif isinstance(image, torch.Tensor):
163
+ height = image.shape[2]
164
+ else:
165
+ height = image.shape[1]
166
+
167
+ if width is None:
168
+ if isinstance(image, PIL.Image.Image):
169
+ width = image.width
170
+ elif isinstance(image, torch.Tensor):
171
+ width = image.shape[3]
172
+ else:
173
+ width = image.shape[2]
174
+
175
+ max_area = width * height
176
+ aspect_ratio = height / width
177
+ mod_value_h = self.config.vae_scale_factor * self.config.spatial_patch_size[0]
178
+ mod_value_w = self.config.vae_scale_factor * self.config.spatial_patch_size[1]
179
+
180
+ # Try to preserve the aspect ratio
181
+ height = round(np.sqrt(max_area * aspect_ratio)) // mod_value_h * mod_value_h
182
+ width = round(np.sqrt(max_area / aspect_ratio)) // mod_value_w * mod_value_w
183
+
184
+ return height, width
western_ai/europe/diffusers/src/diffusers/pipelines/wan/pipeline_output.py ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+
3
+ import torch
4
+
5
+ from diffusers.utils import BaseOutput
6
+
7
+
8
+ @dataclass
9
+ class WanPipelineOutput(BaseOutput):
10
+ r"""
11
+ Output class for Wan pipelines.
12
+
13
+ Args:
14
+ frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]):
15
+ list of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing
16
+ denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape
17
+ `(batch_size, num_frames, channels, height, width)`.
18
+ """
19
+
20
+ frames: torch.Tensor
western_ai/europe/diffusers/src/diffusers/pipelines/wan/pipeline_wan.py ADDED
@@ -0,0 +1,678 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 The Wan Team and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import html
16
+ from typing import Any, Callable
17
+
18
+ import regex as re
19
+ import torch
20
+ from transformers import AutoTokenizer, UMT5EncoderModel
21
+
22
+ from ...callbacks import MultiPipelineCallbacks, PipelineCallback
23
+ from ...loaders import WanLoraLoaderMixin
24
+ from ...models import AutoencoderKLWan, WanTransformer3DModel
25
+ from ...schedulers import FlowMatchEulerDiscreteScheduler
26
+ from ...utils import is_ftfy_available, is_torch_xla_available, logging, replace_example_docstring
27
+ from ...utils.torch_utils import randn_tensor
28
+ from ...video_processor import VideoProcessor
29
+ from ..pipeline_utils import DiffusionPipeline
30
+ from .pipeline_output import WanPipelineOutput
31
+
32
+
33
+ if is_torch_xla_available():
34
+ import torch_xla.core.xla_model as xm
35
+
36
+ XLA_AVAILABLE = True
37
+ else:
38
+ XLA_AVAILABLE = False
39
+
40
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
41
+
42
+ if is_ftfy_available():
43
+ import ftfy
44
+
45
+
46
+ EXAMPLE_DOC_STRING = """
47
+ Examples:
48
+ ```python
49
+ >>> import torch
50
+ >>> from diffusers.utils import export_to_video
51
+ >>> from diffusers import AutoencoderKLWan, WanPipeline
52
+ >>> from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
53
+
54
+ >>> # Available models: Wan-AI/Wan2.1-T2V-14B-Diffusers, Wan-AI/Wan2.1-T2V-1.3B-Diffusers
55
+ >>> model_id = "Wan-AI/Wan2.1-T2V-14B-Diffusers"
56
+ >>> vae = AutoencoderKLWan.from_pretrained(model_id, subfolder="vae", torch_dtype=torch.float32)
57
+ >>> pipe = WanPipeline.from_pretrained(model_id, vae=vae, torch_dtype=torch.bfloat16)
58
+ >>> flow_shift = 5.0 # 5.0 for 720P, 3.0 for 480P
59
+ >>> pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config, flow_shift=flow_shift)
60
+ >>> pipe.to("cuda")
61
+
62
+ >>> prompt = "A cat and a dog baking a cake together in a kitchen. The cat is carefully measuring flour, while the dog is stirring the batter with a wooden spoon. The kitchen is cozy, with sunlight streaming through the window."
63
+ >>> negative_prompt = "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards"
64
+
65
+ >>> output = pipe(
66
+ ... prompt=prompt,
67
+ ... negative_prompt=negative_prompt,
68
+ ... height=720,
69
+ ... width=1280,
70
+ ... num_frames=81,
71
+ ... guidance_scale=5.0,
72
+ ... ).frames[0]
73
+ >>> export_to_video(output, "output.mp4", fps=16)
74
+ ```
75
+ """
76
+
77
+
78
+ def basic_clean(text):
79
+ if is_ftfy_available():
80
+ text = ftfy.fix_text(text)
81
+ text = html.unescape(html.unescape(text))
82
+ return text.strip()
83
+
84
+
85
+ def whitespace_clean(text):
86
+ text = re.sub(r"\s+", " ", text)
87
+ text = text.strip()
88
+ return text
89
+
90
+
91
+ def prompt_clean(text):
92
+ text = whitespace_clean(basic_clean(text))
93
+ return text
94
+
95
+
96
+ class WanPipeline(DiffusionPipeline, WanLoraLoaderMixin):
97
+ r"""
98
+ Pipeline for text-to-video generation using Wan.
99
+
100
+ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods
101
+ implemented for all pipelines (downloading, saving, running on a particular device, etc.).
102
+
103
+ Args:
104
+ tokenizer ([`T5Tokenizer`]):
105
+ Tokenizer from [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5Tokenizer),
106
+ specifically the [google/umt5-xxl](https://huggingface.co/google/umt5-xxl) variant.
107
+ text_encoder ([`T5EncoderModel`]):
108
+ [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5EncoderModel), specifically
109
+ the [google/umt5-xxl](https://huggingface.co/google/umt5-xxl) variant.
110
+ transformer ([`WanTransformer3DModel`]):
111
+ Conditional Transformer to denoise the input latents.
112
+ scheduler ([`UniPCMultistepScheduler`]):
113
+ A scheduler to be used in combination with `transformer` to denoise the encoded image latents.
114
+ vae ([`AutoencoderKLWan`]):
115
+ Variational Auto-Encoder (VAE) Model to encode and decode videos to and from latent representations.
116
+ transformer_2 ([`WanTransformer3DModel`], *optional*):
117
+ Conditional Transformer to denoise the input latents during the low-noise stage. If provided, enables
118
+ two-stage denoising where `transformer` handles high-noise stages and `transformer_2` handles low-noise
119
+ stages. If not provided, only `transformer` is used.
120
+ boundary_ratio (`float`, *optional*, defaults to `None`):
121
+ Ratio of total timesteps to use as the boundary for switching between transformers in two-stage denoising.
122
+ The actual boundary timestep is calculated as `boundary_ratio * num_train_timesteps`. When provided,
123
+ `transformer` handles timesteps >= boundary_timestep and `transformer_2` handles timesteps <
124
+ boundary_timestep. If `None`, only `transformer` is used for the entire denoising process.
125
+ """
126
+
127
+ model_cpu_offload_seq = "text_encoder->transformer->transformer_2->vae"
128
+ _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"]
129
+ _optional_components = ["transformer", "transformer_2"]
130
+
131
+ def __init__(
132
+ self,
133
+ tokenizer: AutoTokenizer,
134
+ text_encoder: UMT5EncoderModel,
135
+ vae: AutoencoderKLWan,
136
+ scheduler: FlowMatchEulerDiscreteScheduler,
137
+ transformer: WanTransformer3DModel | None = None,
138
+ transformer_2: WanTransformer3DModel | None = None,
139
+ boundary_ratio: float | None = None,
140
+ expand_timesteps: bool = False, # Wan2.2 ti2v
141
+ ):
142
+ super().__init__()
143
+
144
+ self.register_modules(
145
+ vae=vae,
146
+ text_encoder=text_encoder,
147
+ tokenizer=tokenizer,
148
+ transformer=transformer,
149
+ scheduler=scheduler,
150
+ transformer_2=transformer_2,
151
+ )
152
+ self.register_to_config(boundary_ratio=boundary_ratio)
153
+ self.register_to_config(expand_timesteps=expand_timesteps)
154
+ self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if getattr(self, "vae", None) else 4
155
+ self.vae_scale_factor_spatial = self.vae.config.scale_factor_spatial if getattr(self, "vae", None) else 8
156
+ self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial)
157
+
158
+ def _get_t5_prompt_embeds(
159
+ self,
160
+ prompt: str | list[str] = None,
161
+ num_videos_per_prompt: int = 1,
162
+ max_sequence_length: int = 226,
163
+ device: torch.device | None = None,
164
+ dtype: torch.dtype | None = None,
165
+ ):
166
+ device = device or self._execution_device
167
+ dtype = dtype or self.text_encoder.dtype
168
+
169
+ prompt = [prompt] if isinstance(prompt, str) else prompt
170
+ prompt = [prompt_clean(u) for u in prompt]
171
+ batch_size = len(prompt)
172
+
173
+ text_inputs = self.tokenizer(
174
+ prompt,
175
+ padding="max_length",
176
+ max_length=max_sequence_length,
177
+ truncation=True,
178
+ add_special_tokens=True,
179
+ return_attention_mask=True,
180
+ return_tensors="pt",
181
+ )
182
+ text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask
183
+ seq_lens = mask.gt(0).sum(dim=1).long()
184
+
185
+ prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state
186
+ prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
187
+ prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)]
188
+ prompt_embeds = torch.stack(
189
+ [torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0
190
+ )
191
+
192
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
193
+ _, seq_len, _ = prompt_embeds.shape
194
+ prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1)
195
+ prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
196
+
197
+ return prompt_embeds
198
+
199
+ def encode_prompt(
200
+ self,
201
+ prompt: str | list[str],
202
+ negative_prompt: str | list[str] | None = None,
203
+ do_classifier_free_guidance: bool = True,
204
+ num_videos_per_prompt: int = 1,
205
+ prompt_embeds: torch.Tensor | None = None,
206
+ negative_prompt_embeds: torch.Tensor | None = None,
207
+ max_sequence_length: int = 226,
208
+ device: torch.device | None = None,
209
+ dtype: torch.dtype | None = None,
210
+ ):
211
+ r"""
212
+ Encodes the prompt into text encoder hidden states.
213
+
214
+ Args:
215
+ prompt (`str` or `list[str]`, *optional*):
216
+ prompt to be encoded
217
+ negative_prompt (`str` or `list[str]`, *optional*):
218
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
219
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
220
+ less than `1`).
221
+ do_classifier_free_guidance (`bool`, *optional*, defaults to `True`):
222
+ Whether to use classifier free guidance or not.
223
+ num_videos_per_prompt (`int`, *optional*, defaults to 1):
224
+ Number of videos that should be generated per prompt. torch device to place the resulting embeddings on
225
+ prompt_embeds (`torch.Tensor`, *optional*):
226
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
227
+ provided, text embeddings will be generated from `prompt` input argument.
228
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
229
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
230
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
231
+ argument.
232
+ device: (`torch.device`, *optional*):
233
+ torch device
234
+ dtype: (`torch.dtype`, *optional*):
235
+ torch dtype
236
+ """
237
+ device = device or self._execution_device
238
+
239
+ prompt = [prompt] if isinstance(prompt, str) else prompt
240
+ if prompt is not None:
241
+ batch_size = len(prompt)
242
+ else:
243
+ batch_size = prompt_embeds.shape[0]
244
+
245
+ if prompt_embeds is None:
246
+ prompt_embeds = self._get_t5_prompt_embeds(
247
+ prompt=prompt,
248
+ num_videos_per_prompt=num_videos_per_prompt,
249
+ max_sequence_length=max_sequence_length,
250
+ device=device,
251
+ dtype=dtype,
252
+ )
253
+
254
+ if do_classifier_free_guidance and negative_prompt_embeds is None:
255
+ negative_prompt = negative_prompt or ""
256
+ negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
257
+
258
+ if prompt is not None and type(prompt) is not type(negative_prompt):
259
+ raise TypeError(
260
+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
261
+ f" {type(prompt)}."
262
+ )
263
+ elif batch_size != len(negative_prompt):
264
+ raise ValueError(
265
+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
266
+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
267
+ " the batch size of `prompt`."
268
+ )
269
+
270
+ negative_prompt_embeds = self._get_t5_prompt_embeds(
271
+ prompt=negative_prompt,
272
+ num_videos_per_prompt=num_videos_per_prompt,
273
+ max_sequence_length=max_sequence_length,
274
+ device=device,
275
+ dtype=dtype,
276
+ )
277
+
278
+ return prompt_embeds, negative_prompt_embeds
279
+
280
+ def check_inputs(
281
+ self,
282
+ prompt,
283
+ negative_prompt,
284
+ height,
285
+ width,
286
+ prompt_embeds=None,
287
+ negative_prompt_embeds=None,
288
+ callback_on_step_end_tensor_inputs=None,
289
+ guidance_scale_2=None,
290
+ ):
291
+ if height % 16 != 0 or width % 16 != 0:
292
+ raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.")
293
+
294
+ if callback_on_step_end_tensor_inputs is not None and not all(
295
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
296
+ ):
297
+ raise ValueError(
298
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
299
+ )
300
+
301
+ if prompt is not None and prompt_embeds is not None:
302
+ raise ValueError(
303
+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
304
+ " only forward one of the two."
305
+ )
306
+ elif negative_prompt is not None and negative_prompt_embeds is not None:
307
+ raise ValueError(
308
+ f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`: {negative_prompt_embeds}. Please make sure to"
309
+ " only forward one of the two."
310
+ )
311
+ elif prompt is None and prompt_embeds is None:
312
+ raise ValueError(
313
+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
314
+ )
315
+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
316
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
317
+ elif negative_prompt is not None and (
318
+ not isinstance(negative_prompt, str) and not isinstance(negative_prompt, list)
319
+ ):
320
+ raise ValueError(f"`negative_prompt` has to be of type `str` or `list` but is {type(negative_prompt)}")
321
+
322
+ if self.config.boundary_ratio is None and guidance_scale_2 is not None:
323
+ raise ValueError("`guidance_scale_2` is only supported when the pipeline's `boundary_ratio` is not None.")
324
+
325
+ def prepare_latents(
326
+ self,
327
+ batch_size: int,
328
+ num_channels_latents: int = 16,
329
+ height: int = 480,
330
+ width: int = 832,
331
+ num_frames: int = 81,
332
+ dtype: torch.dtype | None = None,
333
+ device: torch.device | None = None,
334
+ generator: torch.Generator | list[torch.Generator] | None = None,
335
+ latents: torch.Tensor | None = None,
336
+ ) -> torch.Tensor:
337
+ if latents is not None:
338
+ return latents.to(device=device, dtype=dtype)
339
+
340
+ num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1
341
+ shape = (
342
+ batch_size,
343
+ num_channels_latents,
344
+ num_latent_frames,
345
+ int(height) // self.vae_scale_factor_spatial,
346
+ int(width) // self.vae_scale_factor_spatial,
347
+ )
348
+ if isinstance(generator, list) and len(generator) != batch_size:
349
+ raise ValueError(
350
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
351
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
352
+ )
353
+
354
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
355
+ return latents
356
+
357
+ @property
358
+ def guidance_scale(self):
359
+ return self._guidance_scale
360
+
361
+ @property
362
+ def do_classifier_free_guidance(self):
363
+ return self._guidance_scale > 1.0
364
+
365
+ @property
366
+ def num_timesteps(self):
367
+ return self._num_timesteps
368
+
369
+ @property
370
+ def current_timestep(self):
371
+ return self._current_timestep
372
+
373
+ @property
374
+ def interrupt(self):
375
+ return self._interrupt
376
+
377
+ @property
378
+ def attention_kwargs(self):
379
+ return self._attention_kwargs
380
+
381
+ @torch.no_grad()
382
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
383
+ def __call__(
384
+ self,
385
+ prompt: str | list[str] = None,
386
+ negative_prompt: str | list[str] = None,
387
+ height: int = 480,
388
+ width: int = 832,
389
+ num_frames: int = 81,
390
+ num_inference_steps: int = 50,
391
+ guidance_scale: float = 5.0,
392
+ guidance_scale_2: float | None = None,
393
+ num_videos_per_prompt: int | None = 1,
394
+ generator: torch.Generator | list[torch.Generator] | None = None,
395
+ latents: torch.Tensor | None = None,
396
+ prompt_embeds: torch.Tensor | None = None,
397
+ negative_prompt_embeds: torch.Tensor | None = None,
398
+ output_type: str | None = "np",
399
+ return_dict: bool = True,
400
+ attention_kwargs: dict[str, Any] | None = None,
401
+ callback_on_step_end: Callable[[int, int], None] | PipelineCallback | MultiPipelineCallbacks | None = None,
402
+ callback_on_step_end_tensor_inputs: list[str] = ["latents"],
403
+ max_sequence_length: int = 512,
404
+ ):
405
+ r"""
406
+ The call function to the pipeline for generation.
407
+
408
+ Args:
409
+ prompt (`str` or `list[str]`, *optional*):
410
+ The prompt or prompts to guide the image generation. If not defined, pass `prompt_embeds` instead.
411
+ negative_prompt (`str` or `list[str]`, *optional*):
412
+ The prompt or prompts to avoid during image generation. If not defined, pass `negative_prompt_embeds`
413
+ instead. Ignored when not using guidance (`guidance_scale` < `1`).
414
+ height (`int`, defaults to `480`):
415
+ The height in pixels of the generated image.
416
+ width (`int`, defaults to `832`):
417
+ The width in pixels of the generated image.
418
+ num_frames (`int`, defaults to `81`):
419
+ The number of frames in the generated video.
420
+ num_inference_steps (`int`, defaults to `50`):
421
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
422
+ expense of slower inference.
423
+ guidance_scale (`float`, defaults to `5.0`):
424
+ Guidance scale as defined in [Classifier-Free Diffusion
425
+ Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2.
426
+ of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
427
+ `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to
428
+ the text `prompt`, usually at the expense of lower image quality.
429
+ guidance_scale_2 (`float`, *optional*, defaults to `None`):
430
+ Guidance scale for the low-noise stage transformer (`transformer_2`). If `None` and the pipeline's
431
+ `boundary_ratio` is not None, uses the same value as `guidance_scale`. Only used when `transformer_2`
432
+ and the pipeline's `boundary_ratio` are not None.
433
+ num_videos_per_prompt (`int`, *optional*, defaults to 1):
434
+ The number of images to generate per prompt.
435
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
436
+ A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make
437
+ generation deterministic.
438
+ latents (`torch.Tensor`, *optional*):
439
+ Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image
440
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
441
+ tensor is generated by sampling using the supplied random `generator`.
442
+ prompt_embeds (`torch.Tensor`, *optional*):
443
+ Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not
444
+ provided, text embeddings are generated from the `prompt` input argument.
445
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
446
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If
447
+ not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.
448
+ output_type (`str`, *optional*, defaults to `"np"`):
449
+ The output format of the generated image. Choose between `PIL.Image` or `np.array`.
450
+ return_dict (`bool`, *optional*, defaults to `True`):
451
+ Whether or not to return a [`WanPipelineOutput`] instead of a plain tuple.
452
+ attention_kwargs (`dict`, *optional*):
453
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
454
+ `self.processor` in
455
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
456
+ callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*):
457
+ A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of
458
+ each denoising step during the inference. with the following arguments: `callback_on_step_end(self:
459
+ DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a
460
+ list of all tensors as specified by `callback_on_step_end_tensor_inputs`.
461
+ callback_on_step_end_tensor_inputs (`list`, *optional*):
462
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
463
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
464
+ `._callback_tensor_inputs` attribute of your pipeline class.
465
+ max_sequence_length (`int`, defaults to `512`):
466
+ The maximum sequence length of the text encoder. If the prompt is longer than this, it will be
467
+ truncated. If the prompt is shorter, it will be padded to this length.
468
+
469
+ Examples:
470
+
471
+ Returns:
472
+ [`~WanPipelineOutput`] or `tuple`:
473
+ If `return_dict` is `True`, [`WanPipelineOutput`] is returned, otherwise a `tuple` is returned where
474
+ the first element is a list with the generated images and the second element is a list of `bool`s
475
+ indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content.
476
+ """
477
+
478
+ if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
479
+ callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
480
+
481
+ # 1. Check inputs. Raise error if not correct
482
+ self.check_inputs(
483
+ prompt,
484
+ negative_prompt,
485
+ height,
486
+ width,
487
+ prompt_embeds,
488
+ negative_prompt_embeds,
489
+ callback_on_step_end_tensor_inputs,
490
+ guidance_scale_2,
491
+ )
492
+
493
+ if num_frames % self.vae_scale_factor_temporal != 1:
494
+ logger.warning(
495
+ f"`num_frames - 1` has to be divisible by {self.vae_scale_factor_temporal}. Rounding to the nearest number."
496
+ )
497
+ num_frames = num_frames // self.vae_scale_factor_temporal * self.vae_scale_factor_temporal + 1
498
+ num_frames = max(num_frames, 1)
499
+
500
+ patch_size = (
501
+ self.transformer.config.patch_size
502
+ if self.transformer is not None
503
+ else self.transformer_2.config.patch_size
504
+ )
505
+ h_multiple_of = self.vae_scale_factor_spatial * patch_size[1]
506
+ w_multiple_of = self.vae_scale_factor_spatial * patch_size[2]
507
+ calc_height = height // h_multiple_of * h_multiple_of
508
+ calc_width = width // w_multiple_of * w_multiple_of
509
+ if height != calc_height or width != calc_width:
510
+ logger.warning(
511
+ f"`height` and `width` must be multiples of ({h_multiple_of}, {w_multiple_of}) for proper patchification. "
512
+ f"Adjusting ({height}, {width}) -> ({calc_height}, {calc_width})."
513
+ )
514
+ height, width = calc_height, calc_width
515
+
516
+ if self.config.boundary_ratio is not None and guidance_scale_2 is None:
517
+ guidance_scale_2 = guidance_scale
518
+
519
+ self._guidance_scale = guidance_scale
520
+ self._guidance_scale_2 = guidance_scale_2
521
+ self._attention_kwargs = attention_kwargs
522
+ self._current_timestep = None
523
+ self._interrupt = False
524
+
525
+ device = self._execution_device
526
+
527
+ # 2. Define call parameters
528
+ if prompt is not None and isinstance(prompt, str):
529
+ batch_size = 1
530
+ elif prompt is not None and isinstance(prompt, list):
531
+ batch_size = len(prompt)
532
+ else:
533
+ batch_size = prompt_embeds.shape[0]
534
+
535
+ # 3. Encode input prompt
536
+ prompt_embeds, negative_prompt_embeds = self.encode_prompt(
537
+ prompt=prompt,
538
+ negative_prompt=negative_prompt,
539
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
540
+ num_videos_per_prompt=num_videos_per_prompt,
541
+ prompt_embeds=prompt_embeds,
542
+ negative_prompt_embeds=negative_prompt_embeds,
543
+ max_sequence_length=max_sequence_length,
544
+ device=device,
545
+ )
546
+
547
+ transformer_dtype = self.transformer.dtype if self.transformer is not None else self.transformer_2.dtype
548
+ prompt_embeds = prompt_embeds.to(transformer_dtype)
549
+ if negative_prompt_embeds is not None:
550
+ negative_prompt_embeds = negative_prompt_embeds.to(transformer_dtype)
551
+
552
+ # 4. Prepare timesteps
553
+ self.scheduler.set_timesteps(num_inference_steps, device=device)
554
+ timesteps = self.scheduler.timesteps
555
+
556
+ # 5. Prepare latent variables
557
+ num_channels_latents = (
558
+ self.transformer.config.in_channels
559
+ if self.transformer is not None
560
+ else self.transformer_2.config.in_channels
561
+ )
562
+ latents = self.prepare_latents(
563
+ batch_size * num_videos_per_prompt,
564
+ num_channels_latents,
565
+ height,
566
+ width,
567
+ num_frames,
568
+ torch.float32,
569
+ device,
570
+ generator,
571
+ latents,
572
+ )
573
+
574
+ mask = torch.ones(latents.shape, dtype=torch.float32, device=device)
575
+
576
+ # 6. Denoising loop
577
+ num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
578
+ self._num_timesteps = len(timesteps)
579
+
580
+ # We set the index here to remove DtoH sync, helpful especially during compilation.
581
+ # Check out more details here: https://github.com/huggingface/diffusers/pull/11696
582
+ self.scheduler.set_begin_index(0)
583
+
584
+ if self.config.boundary_ratio is not None:
585
+ boundary_timestep = self.config.boundary_ratio * self.scheduler.config.num_train_timesteps
586
+ else:
587
+ boundary_timestep = None
588
+
589
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
590
+ for i, t in enumerate(timesteps):
591
+ if self.interrupt:
592
+ continue
593
+
594
+ self._current_timestep = t
595
+
596
+ if boundary_timestep is None or t >= boundary_timestep:
597
+ # wan2.1 or high-noise stage in wan2.2
598
+ current_model = self.transformer
599
+ current_guidance_scale = guidance_scale
600
+ else:
601
+ # low-noise stage in wan2.2
602
+ current_model = self.transformer_2
603
+ current_guidance_scale = guidance_scale_2
604
+
605
+ latent_model_input = latents.to(transformer_dtype)
606
+ if self.config.expand_timesteps:
607
+ # seq_len: num_latent_frames * latent_height//2 * latent_width//2
608
+ temp_ts = (mask[0][0][:, ::2, ::2] * t).flatten()
609
+ # batch_size, seq_len
610
+ timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
611
+ else:
612
+ timestep = t.expand(latents.shape[0])
613
+
614
+ with current_model.cache_context("cond"):
615
+ noise_pred = current_model(
616
+ hidden_states=latent_model_input,
617
+ timestep=timestep,
618
+ encoder_hidden_states=prompt_embeds,
619
+ attention_kwargs=attention_kwargs,
620
+ return_dict=False,
621
+ )[0]
622
+
623
+ if self.do_classifier_free_guidance:
624
+ with current_model.cache_context("uncond"):
625
+ noise_uncond = current_model(
626
+ hidden_states=latent_model_input,
627
+ timestep=timestep,
628
+ encoder_hidden_states=negative_prompt_embeds,
629
+ attention_kwargs=attention_kwargs,
630
+ return_dict=False,
631
+ )[0]
632
+ noise_pred = noise_uncond + current_guidance_scale * (noise_pred - noise_uncond)
633
+
634
+ # compute the previous noisy sample x_t -> x_t-1
635
+ latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
636
+
637
+ if callback_on_step_end is not None:
638
+ callback_kwargs = {}
639
+ for k in callback_on_step_end_tensor_inputs:
640
+ callback_kwargs[k] = locals()[k]
641
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
642
+
643
+ latents = callback_outputs.pop("latents", latents)
644
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
645
+ negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
646
+
647
+ # call the callback, if provided
648
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
649
+ progress_bar.update()
650
+
651
+ if XLA_AVAILABLE:
652
+ xm.mark_step()
653
+
654
+ self._current_timestep = None
655
+
656
+ if not output_type == "latent":
657
+ latents = latents.to(self.vae.dtype)
658
+ latents_mean = (
659
+ torch.tensor(self.vae.config.latents_mean)
660
+ .view(1, self.vae.config.z_dim, 1, 1, 1)
661
+ .to(latents.device, latents.dtype)
662
+ )
663
+ latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
664
+ latents.device, latents.dtype
665
+ )
666
+ latents = latents / latents_std + latents_mean
667
+ video = self.vae.decode(latents, return_dict=False)[0]
668
+ video = self.video_processor.postprocess_video(video, output_type=output_type)
669
+ else:
670
+ video = latents
671
+
672
+ # Offload all models
673
+ self.maybe_free_model_hooks()
674
+
675
+ if not return_dict:
676
+ return (video,)
677
+
678
+ return WanPipelineOutput(frames=video)
western_ai/europe/diffusers/src/diffusers/pipelines/wan/pipeline_wan_animate.py ADDED
@@ -0,0 +1,1202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 The Wan Team and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import html
16
+ from copy import deepcopy
17
+ from typing import Any, Callable
18
+
19
+ import PIL
20
+ import regex as re
21
+ import torch
22
+ import torch.nn.functional as F
23
+ from transformers import AutoTokenizer, CLIPImageProcessor, CLIPVisionModel, UMT5EncoderModel
24
+
25
+ from ...callbacks import MultiPipelineCallbacks, PipelineCallback
26
+ from ...image_processor import PipelineImageInput
27
+ from ...loaders import WanLoraLoaderMixin
28
+ from ...models import AutoencoderKLWan, WanAnimateTransformer3DModel
29
+ from ...schedulers import UniPCMultistepScheduler
30
+ from ...utils import is_ftfy_available, is_torch_xla_available, logging, replace_example_docstring
31
+ from ...utils.torch_utils import randn_tensor
32
+ from ...video_processor import VideoProcessor
33
+ from ..pipeline_utils import DiffusionPipeline
34
+ from .image_processor import WanAnimateImageProcessor
35
+ from .pipeline_output import WanPipelineOutput
36
+
37
+
38
+ if is_torch_xla_available():
39
+ import torch_xla.core.xla_model as xm
40
+
41
+ XLA_AVAILABLE = True
42
+ else:
43
+ XLA_AVAILABLE = False
44
+
45
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
46
+
47
+ if is_ftfy_available():
48
+ import ftfy
49
+
50
+ EXAMPLE_DOC_STRING = """
51
+ Examples:
52
+ ```python
53
+ >>> import torch
54
+ >>> import numpy as np
55
+ >>> from diffusers import WanAnimatePipeline
56
+ >>> from diffusers.utils import export_to_video, load_image, load_video
57
+
58
+ >>> model_id = "Wan-AI/Wan2.2-Animate-14B-Diffusers"
59
+ >>> pipe = WanAnimatePipeline.from_pretrained(model_id, torch_dtype=torch.bfloat16)
60
+ >>> # Optionally upcast the Wan VAE to FP32
61
+ >>> pipe.vae.to(torch.float32)
62
+ >>> pipe.to("cuda")
63
+
64
+ >>> # Load the reference character image
65
+ >>> image = load_image(
66
+ ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg"
67
+ ... )
68
+
69
+ >>> # Load pose and face videos (preprocessed from reference video)
70
+ >>> # Note: Videos should be preprocessed to extract pose keypoints and face features
71
+ >>> # Refer to the Wan-Animate preprocessing documentation for details
72
+ >>> pose_video = load_video("path/to/pose_video.mp4")
73
+ >>> face_video = load_video("path/to/face_video.mp4")
74
+
75
+ >>> # CFG is generally not used for Wan Animate
76
+ >>> prompt = (
77
+ ... "An astronaut hatching from an egg, on the surface of the moon, the darkness and depth of space realised in "
78
+ ... "the background. High quality, ultrarealistic detail and breath-taking movie-like camera shot."
79
+ ... )
80
+
81
+ >>> # Animation mode: Animate the character with the motion from pose/face videos
82
+ >>> output = pipe(
83
+ ... image=image,
84
+ ... pose_video=pose_video,
85
+ ... face_video=face_video,
86
+ ... prompt=prompt,
87
+ ... height=height,
88
+ ... width=width,
89
+ ... segment_frame_length=77, # Frame length of each inference segment
90
+ ... guidance_scale=1.0,
91
+ ... num_inference_steps=20,
92
+ ... mode="animate",
93
+ ... ).frames[0]
94
+ >>> export_to_video(output, "output_animation.mp4", fps=30)
95
+
96
+ >>> # Replacement mode: Replace a character in the background video
97
+ >>> # Requires additional background_video and mask_video inputs
98
+ >>> background_video = load_video("path/to/background_video.mp4")
99
+ >>> mask_video = load_video("path/to/mask_video.mp4") # Black areas preserved, white areas generated
100
+ >>> output = pipe(
101
+ ... image=image,
102
+ ... pose_video=pose_video,
103
+ ... face_video=face_video,
104
+ ... background_video=background_video,
105
+ ... mask_video=mask_video,
106
+ ... prompt=prompt,
107
+ ... height=height,
108
+ ... width=width,
109
+ ... segment_frame_length=77, # Frame length of each inference segment
110
+ ... guidance_scale=1.0,
111
+ ... num_inference_steps=20,
112
+ ... mode="replace",
113
+ ... ).frames[0]
114
+ >>> export_to_video(output, "output_replacement.mp4", fps=30)
115
+ ```
116
+ """
117
+
118
+
119
+ def basic_clean(text):
120
+ text = ftfy.fix_text(text)
121
+ text = html.unescape(html.unescape(text))
122
+ return text.strip()
123
+
124
+
125
+ def whitespace_clean(text):
126
+ text = re.sub(r"\s+", " ", text)
127
+ text = text.strip()
128
+ return text
129
+
130
+
131
+ def prompt_clean(text):
132
+ text = whitespace_clean(basic_clean(text))
133
+ return text
134
+
135
+
136
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
137
+ def retrieve_latents(
138
+ encoder_output: torch.Tensor, generator: torch.Generator | None = None, sample_mode: str = "sample"
139
+ ):
140
+ if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
141
+ return encoder_output.latent_dist.sample(generator)
142
+ elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
143
+ return encoder_output.latent_dist.mode()
144
+ elif hasattr(encoder_output, "latents"):
145
+ return encoder_output.latents
146
+ else:
147
+ raise AttributeError("Could not access latents of provided encoder_output")
148
+
149
+
150
+ class WanAnimatePipeline(DiffusionPipeline, WanLoraLoaderMixin):
151
+ r"""
152
+ Pipeline for unified character animation and replacement using Wan-Animate.
153
+
154
+ WanAnimatePipeline takes a character image, pose video, and face video as input, and generates a video in two
155
+ modes:
156
+
157
+ 1. **Animation mode**: The model generates a video of the character image that mimics the human motion in the input
158
+ pose and face videos. The character is animated based on the provided motion controls, creating a new animated
159
+ video of the character.
160
+
161
+ 2. **Replacement mode**: The model replaces a character in a background video with the provided character image,
162
+ using the pose and face videos for motion control. This mode requires additional `background_video` and
163
+ `mask_video` inputs. The mask video should have black regions where the original content should be preserved and
164
+ white regions where the new character should be generated.
165
+
166
+ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods
167
+ implemented for all pipelines (downloading, saving, running on a particular device, etc.).
168
+
169
+ The pipeline also inherits the following loading methods:
170
+ - [`~loaders.WanLoraLoaderMixin.load_lora_weights`] for loading LoRA weights
171
+
172
+ Args:
173
+ tokenizer ([`T5Tokenizer`]):
174
+ Tokenizer from [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5Tokenizer),
175
+ specifically the [google/umt5-xxl](https://huggingface.co/google/umt5-xxl) variant.
176
+ text_encoder ([`T5EncoderModel`]):
177
+ [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5EncoderModel), specifically
178
+ the [google/umt5-xxl](https://huggingface.co/google/umt5-xxl) variant.
179
+ image_encoder ([`CLIPVisionModel`]):
180
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPVisionModel), specifically
181
+ the
182
+ [clip-vit-huge-patch14](https://github.com/mlfoundations/open_clip/blob/main/docs/PRETRAINED.md#vit-h14-xlm-roberta-large)
183
+ variant.
184
+ transformer ([`WanAnimateTransformer3DModel`]):
185
+ Conditional Transformer to denoise the input latents.
186
+ scheduler ([`UniPCMultistepScheduler`]):
187
+ A scheduler to be used in combination with `transformer` to denoise the encoded image latents.
188
+ vae ([`AutoencoderKLWan`]):
189
+ Variational Auto-Encoder (VAE) Model to encode and decode videos to and from latent representations.
190
+ image_processor ([`CLIPImageProcessor`]):
191
+ Image processor for preprocessing images before encoding.
192
+ """
193
+
194
+ model_cpu_offload_seq = "text_encoder->image_encoder->transformer->vae"
195
+ _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"]
196
+
197
+ def __init__(
198
+ self,
199
+ tokenizer: AutoTokenizer,
200
+ text_encoder: UMT5EncoderModel,
201
+ vae: AutoencoderKLWan,
202
+ scheduler: UniPCMultistepScheduler,
203
+ image_processor: CLIPImageProcessor,
204
+ image_encoder: CLIPVisionModel,
205
+ transformer: WanAnimateTransformer3DModel,
206
+ ):
207
+ super().__init__()
208
+
209
+ self.register_modules(
210
+ vae=vae,
211
+ text_encoder=text_encoder,
212
+ tokenizer=tokenizer,
213
+ image_encoder=image_encoder,
214
+ transformer=transformer,
215
+ scheduler=scheduler,
216
+ image_processor=image_processor,
217
+ )
218
+
219
+ self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if getattr(self, "vae", None) else 4
220
+ self.vae_scale_factor_spatial = self.vae.config.scale_factor_spatial if getattr(self, "vae", None) else 8
221
+ self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial)
222
+ self.video_processor_for_mask = VideoProcessor(
223
+ vae_scale_factor=self.vae_scale_factor_spatial, do_normalize=False, do_convert_grayscale=True
224
+ )
225
+ # In case self.transformer is None (e.g. for some pipeline tests)
226
+ spatial_patch_size = self.transformer.config.patch_size[-2:] if self.transformer is not None else (2, 2)
227
+ self.vae_image_processor = WanAnimateImageProcessor(
228
+ vae_scale_factor=self.vae_scale_factor_spatial,
229
+ spatial_patch_size=spatial_patch_size,
230
+ resample="bilinear",
231
+ fill_color=0,
232
+ )
233
+ self.image_processor = image_processor
234
+
235
+ def _get_t5_prompt_embeds(
236
+ self,
237
+ prompt: str | list[str] = None,
238
+ num_videos_per_prompt: int = 1,
239
+ max_sequence_length: int = 512,
240
+ device: torch.device | None = None,
241
+ dtype: torch.dtype | None = None,
242
+ ):
243
+ device = device or self._execution_device
244
+ dtype = dtype or self.text_encoder.dtype
245
+
246
+ prompt = [prompt] if isinstance(prompt, str) else prompt
247
+ prompt = [prompt_clean(u) for u in prompt]
248
+ batch_size = len(prompt)
249
+
250
+ text_inputs = self.tokenizer(
251
+ prompt,
252
+ padding="max_length",
253
+ max_length=max_sequence_length,
254
+ truncation=True,
255
+ add_special_tokens=True,
256
+ return_attention_mask=True,
257
+ return_tensors="pt",
258
+ )
259
+ text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask
260
+ seq_lens = mask.gt(0).sum(dim=1).long()
261
+
262
+ prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state
263
+ prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
264
+ prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)]
265
+ prompt_embeds = torch.stack(
266
+ [torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0
267
+ )
268
+
269
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
270
+ _, seq_len, _ = prompt_embeds.shape
271
+ prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1)
272
+ prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
273
+
274
+ return prompt_embeds
275
+
276
+ # Copied from diffusers.pipelines.wan.pipeline_wan_i2v.WanImageToVideoPipeline.encode_image
277
+ def encode_image(
278
+ self,
279
+ image: PipelineImageInput,
280
+ device: torch.device | None = None,
281
+ ):
282
+ device = device or self._execution_device
283
+ image = self.image_processor(images=image, return_tensors="pt").to(device)
284
+ image_embeds = self.image_encoder(**image, output_hidden_states=True)
285
+ return image_embeds.hidden_states[-2]
286
+
287
+ # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline.encode_prompt
288
+ def encode_prompt(
289
+ self,
290
+ prompt: str | list[str],
291
+ negative_prompt: str | list[str] | None = None,
292
+ do_classifier_free_guidance: bool = True,
293
+ num_videos_per_prompt: int = 1,
294
+ prompt_embeds: torch.Tensor | None = None,
295
+ negative_prompt_embeds: torch.Tensor | None = None,
296
+ max_sequence_length: int = 226,
297
+ device: torch.device | None = None,
298
+ dtype: torch.dtype | None = None,
299
+ ):
300
+ r"""
301
+ Encodes the prompt into text encoder hidden states.
302
+
303
+ Args:
304
+ prompt (`str` or `list[str]`, *optional*):
305
+ prompt to be encoded
306
+ negative_prompt (`str` or `list[str]`, *optional*):
307
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
308
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
309
+ less than `1`).
310
+ do_classifier_free_guidance (`bool`, *optional*, defaults to `True`):
311
+ Whether to use classifier free guidance or not.
312
+ num_videos_per_prompt (`int`, *optional*, defaults to 1):
313
+ Number of videos that should be generated per prompt. torch device to place the resulting embeddings on
314
+ prompt_embeds (`torch.Tensor`, *optional*):
315
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
316
+ provided, text embeddings will be generated from `prompt` input argument.
317
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
318
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
319
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
320
+ argument.
321
+ device: (`torch.device`, *optional*):
322
+ torch device
323
+ dtype: (`torch.dtype`, *optional*):
324
+ torch dtype
325
+ """
326
+ device = device or self._execution_device
327
+
328
+ prompt = [prompt] if isinstance(prompt, str) else prompt
329
+ if prompt is not None:
330
+ batch_size = len(prompt)
331
+ else:
332
+ batch_size = prompt_embeds.shape[0]
333
+
334
+ if prompt_embeds is None:
335
+ prompt_embeds = self._get_t5_prompt_embeds(
336
+ prompt=prompt,
337
+ num_videos_per_prompt=num_videos_per_prompt,
338
+ max_sequence_length=max_sequence_length,
339
+ device=device,
340
+ dtype=dtype,
341
+ )
342
+
343
+ if do_classifier_free_guidance and negative_prompt_embeds is None:
344
+ negative_prompt = negative_prompt or ""
345
+ negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
346
+
347
+ if prompt is not None and type(prompt) is not type(negative_prompt):
348
+ raise TypeError(
349
+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
350
+ f" {type(prompt)}."
351
+ )
352
+ elif batch_size != len(negative_prompt):
353
+ raise ValueError(
354
+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
355
+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
356
+ " the batch size of `prompt`."
357
+ )
358
+
359
+ negative_prompt_embeds = self._get_t5_prompt_embeds(
360
+ prompt=negative_prompt,
361
+ num_videos_per_prompt=num_videos_per_prompt,
362
+ max_sequence_length=max_sequence_length,
363
+ device=device,
364
+ dtype=dtype,
365
+ )
366
+
367
+ return prompt_embeds, negative_prompt_embeds
368
+
369
+ def check_inputs(
370
+ self,
371
+ prompt,
372
+ negative_prompt,
373
+ image,
374
+ pose_video,
375
+ face_video,
376
+ background_video,
377
+ mask_video,
378
+ height,
379
+ width,
380
+ prompt_embeds=None,
381
+ negative_prompt_embeds=None,
382
+ image_embeds=None,
383
+ callback_on_step_end_tensor_inputs=None,
384
+ mode=None,
385
+ prev_segment_conditioning_frames=None,
386
+ ):
387
+ if image is not None and image_embeds is not None:
388
+ raise ValueError(
389
+ f"Cannot forward both `image`: {image} and `image_embeds`: {image_embeds}. Please make sure to"
390
+ " only forward one of the two."
391
+ )
392
+ if image is None and image_embeds is None:
393
+ raise ValueError(
394
+ "Provide either `image` or `prompt_embeds`. Cannot leave both `image` and `image_embeds` undefined."
395
+ )
396
+ if image is not None and not isinstance(image, torch.Tensor) and not isinstance(image, PIL.Image.Image):
397
+ raise ValueError(f"`image` has to be of type `torch.Tensor` or `PIL.Image.Image` but is {type(image)}")
398
+ if pose_video is None:
399
+ raise ValueError("Provide `pose_video`. Cannot leave `pose_video` undefined.")
400
+ if face_video is None:
401
+ raise ValueError("Provide `face_video`. Cannot leave `face_video` undefined.")
402
+ if not isinstance(pose_video, list) or not isinstance(face_video, list):
403
+ raise ValueError("`pose_video` and `face_video` must be lists of PIL images.")
404
+ if len(pose_video) == 0 or len(face_video) == 0:
405
+ raise ValueError("`pose_video` and `face_video` must contain at least one frame.")
406
+ if mode == "replace" and (background_video is None or mask_video is None):
407
+ raise ValueError(
408
+ "Provide `background_video` and `mask_video`. Cannot leave both `background_video` and `mask_video`"
409
+ " undefined when mode is `replace`."
410
+ )
411
+ if mode == "replace" and (not isinstance(background_video, list) or not isinstance(mask_video, list)):
412
+ raise ValueError("`background_video` and `mask_video` must be lists of PIL images when mode is `replace`.")
413
+
414
+ if height % 16 != 0 or width % 16 != 0:
415
+ raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.")
416
+
417
+ if callback_on_step_end_tensor_inputs is not None and not all(
418
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
419
+ ):
420
+ raise ValueError(
421
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found"
422
+ f" {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
423
+ )
424
+
425
+ if prompt is not None and prompt_embeds is not None:
426
+ raise ValueError(
427
+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
428
+ " only forward one of the two."
429
+ )
430
+ elif negative_prompt is not None and negative_prompt_embeds is not None:
431
+ raise ValueError(
432
+ f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`: {negative_prompt_embeds}. Please make sure to"
433
+ " only forward one of the two."
434
+ )
435
+ elif prompt is None and prompt_embeds is None:
436
+ raise ValueError(
437
+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
438
+ )
439
+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
440
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
441
+ elif negative_prompt is not None and (
442
+ not isinstance(negative_prompt, str) and not isinstance(negative_prompt, list)
443
+ ):
444
+ raise ValueError(f"`negative_prompt` has to be of type `str` or `list` but is {type(negative_prompt)}")
445
+
446
+ if mode is not None and (not isinstance(mode, str) or mode not in ("animate", "replace")):
447
+ raise ValueError(
448
+ f"`mode` has to be of type `str` and in ('animate', 'replace') but its type is {type(mode)} and value is {mode}"
449
+ )
450
+
451
+ if prev_segment_conditioning_frames is not None and (
452
+ not isinstance(prev_segment_conditioning_frames, int) or prev_segment_conditioning_frames not in (1, 5)
453
+ ):
454
+ raise ValueError(
455
+ f"`prev_segment_conditioning_frames` has to be of type `int` and 1 or 5 but its type is"
456
+ f" {type(prev_segment_conditioning_frames)} and value is {prev_segment_conditioning_frames}"
457
+ )
458
+
459
+ def get_i2v_mask(
460
+ self,
461
+ batch_size: int,
462
+ latent_t: int,
463
+ latent_h: int,
464
+ latent_w: int,
465
+ mask_len: int = 1,
466
+ mask_pixel_values: torch.Tensor | None = None,
467
+ dtype: torch.dtype | None = None,
468
+ device: str | torch.device = "cuda",
469
+ ) -> torch.Tensor:
470
+ # mask_pixel_values shape (if supplied): [B, C = 1, T, latent_h, latent_w]
471
+ if mask_pixel_values is None:
472
+ mask_lat_size = torch.zeros(
473
+ batch_size, 1, (latent_t - 1) * 4 + 1, latent_h, latent_w, dtype=dtype, device=device
474
+ )
475
+ else:
476
+ mask_lat_size = mask_pixel_values.clone().to(device=device, dtype=dtype)
477
+ mask_lat_size[:, :, :mask_len] = 1
478
+ first_frame_mask = mask_lat_size[:, :, 0:1]
479
+ # Repeat first frame mask self.vae_scale_factor_temporal (= 4) times in the frame dimension
480
+ first_frame_mask = torch.repeat_interleave(first_frame_mask, dim=2, repeats=self.vae_scale_factor_temporal)
481
+ mask_lat_size = torch.concat([first_frame_mask, mask_lat_size[:, :, 1:]], dim=2)
482
+ mask_lat_size = mask_lat_size.view(
483
+ batch_size, -1, self.vae_scale_factor_temporal, latent_h, latent_w
484
+ ).transpose(1, 2) # [B, C = 1, 4 * T_lat, H_lat, W_lat] --> [B, C = 4, T_lat, H_lat, W_lat]
485
+
486
+ return mask_lat_size
487
+
488
+ def prepare_reference_image_latents(
489
+ self,
490
+ image: torch.Tensor,
491
+ batch_size: int = 1,
492
+ sample_mode: int = "argmax",
493
+ generator: torch.Generator | list[torch.Generator] | None = None,
494
+ dtype: torch.dtype | None = None,
495
+ device: torch.device | None = None,
496
+ ) -> torch.Tensor:
497
+ # image shape: (B, C, H, W) or (B, C, T, H, W)
498
+ dtype = dtype or self.vae.dtype
499
+ if image.ndim == 4:
500
+ # Add a singleton frame dimension after the channels dimension
501
+ image = image.unsqueeze(2)
502
+
503
+ _, _, _, height, width = image.shape
504
+ latent_height = height // self.vae_scale_factor_spatial
505
+ latent_width = width // self.vae_scale_factor_spatial
506
+
507
+ # Encode image to latents using VAE
508
+ image = image.to(device=device, dtype=dtype)
509
+ if isinstance(generator, list):
510
+ # Like in prepare_latents, assume len(generator) == batch_size
511
+ ref_image_latents = [
512
+ retrieve_latents(self.vae.encode(image), generator=g, sample_mode=sample_mode) for g in generator
513
+ ]
514
+ ref_image_latents = torch.cat(ref_image_latents)
515
+ else:
516
+ ref_image_latents = retrieve_latents(self.vae.encode(image), generator, sample_mode)
517
+ # Standardize latents in preparation for Wan VAE encode
518
+ latents_mean = (
519
+ torch.tensor(self.vae.config.latents_mean)
520
+ .view(1, self.vae.config.z_dim, 1, 1, 1)
521
+ .to(ref_image_latents.device, ref_image_latents.dtype)
522
+ )
523
+ latents_recip_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
524
+ ref_image_latents.device, ref_image_latents.dtype
525
+ )
526
+ ref_image_latents = (ref_image_latents - latents_mean) * latents_recip_std
527
+ # Handle the case where we supply one image and one generator, but batch_size > 1 (e.g. generating multiple
528
+ # videos per prompt)
529
+ if ref_image_latents.shape[0] == 1 and batch_size > 1:
530
+ ref_image_latents = ref_image_latents.expand(batch_size, -1, -1, -1, -1)
531
+
532
+ # Prepare I2V mask in latent space and prepend to the reference image latents along channel dim
533
+ reference_image_mask = self.get_i2v_mask(batch_size, 1, latent_height, latent_width, 1, None, dtype, device)
534
+ reference_image_latents = torch.cat([reference_image_mask, ref_image_latents], dim=1)
535
+
536
+ return reference_image_latents
537
+
538
+ def prepare_prev_segment_cond_latents(
539
+ self,
540
+ prev_segment_cond_video: torch.Tensor | None = None,
541
+ background_video: torch.Tensor | None = None,
542
+ mask_video: torch.Tensor | None = None,
543
+ batch_size: int = 1,
544
+ segment_frame_length: int = 77,
545
+ start_frame: int = 0,
546
+ height: int = 720,
547
+ width: int = 1280,
548
+ prev_segment_cond_frames: int = 1,
549
+ task: str = "animate",
550
+ interpolation_mode: str = "bicubic",
551
+ sample_mode: str = "argmax",
552
+ generator: torch.Generator | list[torch.Generator] | None = None,
553
+ dtype: torch.dtype | None = None,
554
+ device: torch.device | None = None,
555
+ ) -> torch.Tensor:
556
+ # prev_segment_cond_video shape: (B, C, T, H, W) in pixel space if supplied
557
+ # background_video shape: (B, C, T, H, W) (same as prev_segment_cond_video shape)
558
+ # mask_video shape: (B, 1, T, H, W) (same as prev_segment_cond_video, but with only 1 channel)
559
+ dtype = dtype or self.vae.dtype
560
+ if prev_segment_cond_video is None:
561
+ if task == "replace":
562
+ prev_segment_cond_video = background_video[:, :, :prev_segment_cond_frames].to(dtype)
563
+ else:
564
+ cond_frames_shape = (batch_size, 3, prev_segment_cond_frames, height, width) # In pixel space
565
+ prev_segment_cond_video = torch.zeros(cond_frames_shape, dtype=dtype, device=device)
566
+
567
+ data_batch_size, channels, _, segment_height, segment_width = prev_segment_cond_video.shape
568
+ num_latent_frames = (segment_frame_length - 1) // self.vae_scale_factor_temporal + 1
569
+ latent_height = height // self.vae_scale_factor_spatial
570
+ latent_width = width // self.vae_scale_factor_spatial
571
+ if segment_height != height or segment_width != width:
572
+ print(
573
+ f"Interpolating prev segment cond video from ({segment_width}, {segment_height}) to ({width}, {height})"
574
+ )
575
+ # Perform a 4D (spatial) rather than a 5D (spatiotemporal) reshape, following the original code
576
+ prev_segment_cond_video = prev_segment_cond_video.transpose(1, 2).flatten(0, 1) # [B * T, C, H, W]
577
+ prev_segment_cond_video = F.interpolate(
578
+ prev_segment_cond_video, size=(height, width), mode=interpolation_mode
579
+ )
580
+ prev_segment_cond_video = prev_segment_cond_video.unflatten(0, (batch_size, -1)).transpose(1, 2)
581
+
582
+ # Fill the remaining part of the cond video segment with zeros (if animating) or the background video (if
583
+ # replacing).
584
+ if task == "replace":
585
+ remaining_segment = background_video[:, :, prev_segment_cond_frames:].to(dtype)
586
+ else:
587
+ remaining_segment_frames = segment_frame_length - prev_segment_cond_frames
588
+ remaining_segment = torch.zeros(
589
+ batch_size, channels, remaining_segment_frames, height, width, dtype=dtype, device=device
590
+ )
591
+
592
+ # Prepend the conditioning frames from the previous segment to the remaining segment video in the frame dim
593
+ prev_segment_cond_video = prev_segment_cond_video.to(dtype=dtype)
594
+ full_segment_cond_video = torch.cat([prev_segment_cond_video, remaining_segment], dim=2)
595
+
596
+ if isinstance(generator, list):
597
+ if data_batch_size == len(generator):
598
+ prev_segment_cond_latents = [
599
+ retrieve_latents(self.vae.encode(full_segment_cond_video[i].unsqueeze(0)), g, sample_mode)
600
+ for i, g in enumerate(generator)
601
+ ]
602
+ elif data_batch_size == 1:
603
+ # Like prepare_latents, assume len(generator) == batch_size
604
+ prev_segment_cond_latents = [
605
+ retrieve_latents(self.vae.encode(full_segment_cond_video), g, sample_mode) for g in generator
606
+ ]
607
+ else:
608
+ raise ValueError(
609
+ f"The batch size of the prev segment video should be either {len(generator)} or 1 but is"
610
+ f" {data_batch_size}"
611
+ )
612
+ prev_segment_cond_latents = torch.cat(prev_segment_cond_latents)
613
+ else:
614
+ prev_segment_cond_latents = retrieve_latents(
615
+ self.vae.encode(full_segment_cond_video), generator, sample_mode
616
+ )
617
+ # Standardize latents in preparation for Wan VAE encode
618
+ latents_mean = (
619
+ torch.tensor(self.vae.config.latents_mean)
620
+ .view(1, self.vae.config.z_dim, 1, 1, 1)
621
+ .to(prev_segment_cond_latents.device, prev_segment_cond_latents.dtype)
622
+ )
623
+ latents_recip_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
624
+ prev_segment_cond_latents.device, prev_segment_cond_latents.dtype
625
+ )
626
+ prev_segment_cond_latents = (prev_segment_cond_latents - latents_mean) * latents_recip_std
627
+
628
+ # Prepare I2V mask
629
+ if task == "replace":
630
+ mask_video = 1 - mask_video
631
+ mask_video = mask_video.permute(0, 2, 1, 3, 4)
632
+ mask_video = mask_video.flatten(0, 1)
633
+ mask_video = F.interpolate(mask_video, size=(latent_height, latent_width), mode="nearest")
634
+ mask_pixel_values = mask_video.unflatten(0, (batch_size, -1))
635
+ mask_pixel_values = mask_pixel_values.permute(0, 2, 1, 3, 4) # output shape: [B, C = 1, T, H_lat, W_lat]
636
+ else:
637
+ mask_pixel_values = None
638
+ prev_segment_cond_mask = self.get_i2v_mask(
639
+ batch_size,
640
+ num_latent_frames,
641
+ latent_height,
642
+ latent_width,
643
+ mask_len=prev_segment_cond_frames if start_frame > 0 else 0,
644
+ mask_pixel_values=mask_pixel_values,
645
+ dtype=dtype,
646
+ device=device,
647
+ )
648
+
649
+ # Prepend cond I2V mask to prev segment cond latents along channel dimension
650
+ prev_segment_cond_latents = torch.cat([prev_segment_cond_mask, prev_segment_cond_latents], dim=1)
651
+ return prev_segment_cond_latents
652
+
653
+ def prepare_pose_latents(
654
+ self,
655
+ pose_video: torch.Tensor,
656
+ batch_size: int = 1,
657
+ sample_mode: int = "argmax",
658
+ generator: torch.Generator | list[torch.Generator] | None = None,
659
+ dtype: torch.dtype | None = None,
660
+ device: torch.device | None = None,
661
+ ) -> torch.Tensor:
662
+ # pose_video shape: (B, C, T, H, W)
663
+ pose_video = pose_video.to(device=device, dtype=dtype if dtype is not None else self.vae.dtype)
664
+ if isinstance(generator, list):
665
+ pose_latents = [
666
+ retrieve_latents(self.vae.encode(pose_video), generator=g, sample_mode=sample_mode) for g in generator
667
+ ]
668
+ pose_latents = torch.cat(pose_latents)
669
+ else:
670
+ pose_latents = retrieve_latents(self.vae.encode(pose_video), generator, sample_mode)
671
+ # Standardize latents in preparation for Wan VAE encode
672
+ latents_mean = (
673
+ torch.tensor(self.vae.config.latents_mean)
674
+ .view(1, self.vae.config.z_dim, 1, 1, 1)
675
+ .to(pose_latents.device, pose_latents.dtype)
676
+ )
677
+ latents_recip_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
678
+ pose_latents.device, pose_latents.dtype
679
+ )
680
+ pose_latents = (pose_latents - latents_mean) * latents_recip_std
681
+ if pose_latents.shape[0] == 1 and batch_size > 1:
682
+ pose_latents = pose_latents.expand(batch_size, -1, -1, -1, -1)
683
+ return pose_latents
684
+
685
+ def prepare_latents(
686
+ self,
687
+ batch_size: int,
688
+ num_channels_latents: int = 16,
689
+ height: int = 720,
690
+ width: int = 1280,
691
+ num_frames: int = 77,
692
+ dtype: torch.dtype | None = None,
693
+ device: torch.device | None = None,
694
+ generator: torch.Generator | list[torch.Generator] | None = None,
695
+ latents: torch.Tensor | None = None,
696
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
697
+ num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1
698
+ latent_height = height // self.vae_scale_factor_spatial
699
+ latent_width = width // self.vae_scale_factor_spatial
700
+
701
+ shape = (batch_size, num_channels_latents, num_latent_frames + 1, latent_height, latent_width)
702
+ if isinstance(generator, list) and len(generator) != batch_size:
703
+ raise ValueError(
704
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
705
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
706
+ )
707
+
708
+ if latents is None:
709
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
710
+ else:
711
+ latents = latents.to(device=device, dtype=dtype)
712
+
713
+ return latents
714
+
715
+ def pad_video_frames(self, frames: list[Any], num_target_frames: int) -> list[Any]:
716
+ """
717
+ Pads an array-like video `frames` to `num_target_frames` using a "reflect"-like strategy. The frame dimension
718
+ is assumed to be the first dimension. In the 1D case, we can visualize this strategy as follows:
719
+
720
+ pad_video_frames([1, 2, 3, 4, 5], 10) -> [1, 2, 3, 4, 5, 4, 3, 2, 1, 2]
721
+ """
722
+ idx = 0
723
+ flip = False
724
+ target_frames = []
725
+ while len(target_frames) < num_target_frames:
726
+ target_frames.append(deepcopy(frames[idx]))
727
+ if flip:
728
+ idx -= 1
729
+ else:
730
+ idx += 1
731
+ if idx == 0 or idx == len(frames) - 1:
732
+ flip = not flip
733
+
734
+ return target_frames
735
+
736
+ @property
737
+ def guidance_scale(self):
738
+ return self._guidance_scale
739
+
740
+ @property
741
+ def do_classifier_free_guidance(self):
742
+ return self._guidance_scale > 1
743
+
744
+ @property
745
+ def num_timesteps(self):
746
+ return self._num_timesteps
747
+
748
+ @property
749
+ def current_timestep(self):
750
+ return self._current_timestep
751
+
752
+ @property
753
+ def interrupt(self):
754
+ return self._interrupt
755
+
756
+ @property
757
+ def attention_kwargs(self):
758
+ return self._attention_kwargs
759
+
760
+ @torch.no_grad()
761
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
762
+ def __call__(
763
+ self,
764
+ image: PipelineImageInput,
765
+ pose_video: list[PIL.Image.Image],
766
+ face_video: list[PIL.Image.Image],
767
+ background_video: list[PIL.Image.Image] | None = None,
768
+ mask_video: list[PIL.Image.Image] | None = None,
769
+ prompt: str | list[str] = None,
770
+ negative_prompt: str | list[str] = None,
771
+ height: int = 720,
772
+ width: int = 1280,
773
+ segment_frame_length: int = 77,
774
+ num_inference_steps: int = 20,
775
+ mode: str = "animate",
776
+ prev_segment_conditioning_frames: int = 1,
777
+ motion_encode_batch_size: int | None = None,
778
+ guidance_scale: float = 1.0,
779
+ num_videos_per_prompt: int | None = 1,
780
+ generator: torch.Generator | list[torch.Generator] | None = None,
781
+ latents: torch.Tensor | None = None,
782
+ prompt_embeds: torch.Tensor | None = None,
783
+ negative_prompt_embeds: torch.Tensor | None = None,
784
+ image_embeds: torch.Tensor | None = None,
785
+ output_type: str | None = "np",
786
+ return_dict: bool = True,
787
+ attention_kwargs: dict[str, Any] | None = None,
788
+ callback_on_step_end: Callable[[int, int, None], PipelineCallback | MultiPipelineCallbacks] | None = None,
789
+ callback_on_step_end_tensor_inputs: list[str] = ["latents"],
790
+ max_sequence_length: int = 512,
791
+ ):
792
+ r"""
793
+ The call function to the pipeline for generation.
794
+
795
+ Args:
796
+ image (`PipelineImageInput`):
797
+ The input character image to condition the generation on. Must be an image, a list of images or a
798
+ `torch.Tensor`.
799
+ pose_video (`list[PIL.Image.Image]`):
800
+ The input pose video to condition the generation on. Must be a list of PIL images.
801
+ face_video (`list[PIL.Image.Image]`):
802
+ The input face video to condition the generation on. Must be a list of PIL images.
803
+ background_video (`list[PIL.Image.Image]`, *optional*):
804
+ When mode is `"replace"`, the input background video to condition the generation on. Must be a list of
805
+ PIL images.
806
+ mask_video (`list[PIL.Image.Image]`, *optional*):
807
+ When mode is `"replace"`, the input mask video to condition the generation on. Must be a list of PIL
808
+ images.
809
+ prompt (`str` or `list[str]`, *optional*):
810
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
811
+ instead.
812
+ negative_prompt (`str` or `list[str]`, *optional*):
813
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
814
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
815
+ less than `1`).
816
+ mode (`str`, defaults to `"animation"`):
817
+ The mode of the generation. Choose between `"animate"` and `"replace"`.
818
+ prev_segment_conditioning_frames (`int`, defaults to `1`):
819
+ The number of frames from the previous video segment to be used for temporal guidance. Recommended to
820
+ be 1 or 5. In general, should be 4N + 1, where N is a non-negative integer.
821
+ motion_encode_batch_size (`int`, *optional*):
822
+ The batch size for batched encoding of the face video via the motion encoder. This allows trading off
823
+ inference speed for lower memory usage by setting a smaller batch size. Will default to
824
+ `self.transformer.config.motion_encoder_batch_size` if not set.
825
+ height (`int`, defaults to `720`):
826
+ The height of the generated video.
827
+ width (`int`, defaults to `1280`):
828
+ The width of the generated video.
829
+ segment_frame_length (`int`, defaults to `77`):
830
+ The number of frames in each generated video segment. The total frames of video generated will be equal
831
+ to the number of frames in `pose_video`; we will generate the video in segments until we have hit this
832
+ length. In general, should be 4N + 1, where N is a non-negative integer.
833
+ num_inference_steps (`int`, defaults to `20`):
834
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
835
+ expense of slower inference.
836
+ guidance_scale (`float`, defaults to `1.0`):
837
+ Guidance scale as defined in [Classifier-Free Diffusion
838
+ Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2.
839
+ of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
840
+ `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to
841
+ the text `prompt`, usually at the expense of lower image quality. By default, CFG is not used in Wan
842
+ Animate inference.
843
+ num_videos_per_prompt (`int`, *optional*, defaults to 1):
844
+ The number of images to generate per prompt.
845
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
846
+ A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make
847
+ generation deterministic.
848
+ latents (`torch.Tensor`, *optional*):
849
+ Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image
850
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
851
+ tensor is generated by sampling using the supplied random `generator`.
852
+ prompt_embeds (`torch.Tensor`, *optional*):
853
+ Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not
854
+ provided, text embeddings are generated from the `prompt` input argument.
855
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
856
+ Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not
857
+ provided, text embeddings are generated from the `negative_prompt` input argument.
858
+ image_embeds (`torch.Tensor`, *optional*):
859
+ Pre-generated image embeddings. Can be used to easily tweak image inputs (weighting). If not provided,
860
+ image embeddings are generated from the `image` input argument.
861
+ output_type (`str`, *optional*, defaults to `"np"`):
862
+ The output format of the generated image. Choose between `PIL.Image` or `np.array`.
863
+ return_dict (`bool`, *optional*, defaults to `True`):
864
+ Whether or not to return a [`WanPipelineOutput`] instead of a plain tuple.
865
+ attention_kwargs (`dict`, *optional*):
866
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
867
+ `self.processor` in
868
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
869
+ callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*):
870
+ A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of
871
+ each denoising step during the inference. with the following arguments: `callback_on_step_end(self:
872
+ DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a
873
+ list of all tensors as specified by `callback_on_step_end_tensor_inputs`.
874
+ callback_on_step_end_tensor_inputs (`List`, *optional*):
875
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
876
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
877
+ `._callback_tensor_inputs` attribute of your pipeline class.
878
+ max_sequence_length (`int`, defaults to `512`):
879
+ The maximum sequence length of the text encoder. If the prompt is longer than this, it will be
880
+ truncated. If the prompt is shorter, it will be padded to this length.
881
+
882
+ Examples:
883
+
884
+ Returns:
885
+ [`~WanPipelineOutput`] or `tuple`:
886
+ If `return_dict` is `True`, [`WanPipelineOutput`] is returned, otherwise a `tuple` is returned where
887
+ the first element is a list with the generated images and the second element is a list of `bool`s
888
+ indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content.
889
+ """
890
+
891
+ if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
892
+ callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
893
+
894
+ # 1. Check inputs. Raise error if not correct
895
+ self.check_inputs(
896
+ prompt,
897
+ negative_prompt,
898
+ image,
899
+ pose_video,
900
+ face_video,
901
+ background_video,
902
+ mask_video,
903
+ height,
904
+ width,
905
+ prompt_embeds,
906
+ negative_prompt_embeds,
907
+ image_embeds,
908
+ callback_on_step_end_tensor_inputs,
909
+ mode,
910
+ prev_segment_conditioning_frames,
911
+ )
912
+
913
+ if segment_frame_length % self.vae_scale_factor_temporal != 1:
914
+ logger.warning(
915
+ f"`segment_frame_length - 1` has to be divisible by {self.vae_scale_factor_temporal}. Rounding to the"
916
+ f" nearest number."
917
+ )
918
+ segment_frame_length = (
919
+ segment_frame_length // self.vae_scale_factor_temporal * self.vae_scale_factor_temporal + 1
920
+ )
921
+ segment_frame_length = max(segment_frame_length, 1)
922
+
923
+ self._guidance_scale = guidance_scale
924
+ self._attention_kwargs = attention_kwargs
925
+ self._current_timestep = None
926
+ self._interrupt = False
927
+
928
+ device = self._execution_device
929
+
930
+ # 2. Define call parameters
931
+ if prompt is not None and isinstance(prompt, str):
932
+ batch_size = 1
933
+ elif prompt is not None and isinstance(prompt, list):
934
+ batch_size = len(prompt)
935
+ else:
936
+ batch_size = prompt_embeds.shape[0]
937
+
938
+ # As we generate in segments of `segment_frame_length`, set the target frame length to be the least multiple
939
+ # of the effective segment length greater than or equal to the length of `pose_video`.
940
+ cond_video_frames = len(pose_video)
941
+ effective_segment_length = segment_frame_length - prev_segment_conditioning_frames
942
+ last_segment_frames = (cond_video_frames - prev_segment_conditioning_frames) % effective_segment_length
943
+ if last_segment_frames == 0:
944
+ num_padding_frames = 0
945
+ else:
946
+ num_padding_frames = effective_segment_length - last_segment_frames
947
+ num_target_frames = cond_video_frames + num_padding_frames
948
+ num_segments = num_target_frames // effective_segment_length
949
+
950
+ # 3. Encode input prompt
951
+ prompt_embeds, negative_prompt_embeds = self.encode_prompt(
952
+ prompt=prompt,
953
+ negative_prompt=negative_prompt,
954
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
955
+ num_videos_per_prompt=num_videos_per_prompt,
956
+ prompt_embeds=prompt_embeds,
957
+ negative_prompt_embeds=negative_prompt_embeds,
958
+ max_sequence_length=max_sequence_length,
959
+ device=device,
960
+ )
961
+
962
+ transformer_dtype = self.transformer.dtype
963
+ prompt_embeds = prompt_embeds.to(transformer_dtype)
964
+ if negative_prompt_embeds is not None:
965
+ negative_prompt_embeds = negative_prompt_embeds.to(transformer_dtype)
966
+
967
+ # 4. Preprocess and encode the reference (character) image
968
+ image_height, image_width = self.video_processor.get_default_height_width(image)
969
+ if image_height != height or image_width != width:
970
+ logger.warning(f"Reshaping reference image from ({image_width}, {image_height}) to ({width}, {height})")
971
+ image_pixels = self.vae_image_processor.preprocess(image, height=height, width=width, resize_mode="fill").to(
972
+ device, dtype=torch.float32
973
+ )
974
+
975
+ # Get CLIP features from the reference image
976
+ if image_embeds is None:
977
+ image_embeds = self.encode_image(image, device)
978
+ image_embeds = image_embeds.repeat(batch_size * num_videos_per_prompt, 1, 1)
979
+ image_embeds = image_embeds.to(transformer_dtype)
980
+
981
+ # 5. Encode conditioning videos (pose, face)
982
+ pose_video = self.pad_video_frames(pose_video, num_target_frames)
983
+ face_video = self.pad_video_frames(face_video, num_target_frames)
984
+
985
+ # TODO: also support np.ndarray input (e.g. from decord like the original implementation?)
986
+ pose_video_width, pose_video_height = pose_video[0].size
987
+ if pose_video_height != height or pose_video_width != width:
988
+ logger.warning(
989
+ f"Reshaping pose video from ({pose_video_width}, {pose_video_height}) to ({width}, {height})"
990
+ )
991
+ pose_video = self.video_processor.preprocess_video(pose_video, height=height, width=width).to(
992
+ device, dtype=torch.float32
993
+ )
994
+
995
+ face_video_width, face_video_height = face_video[0].size
996
+ expected_face_size = self.transformer.config.motion_encoder_size
997
+ if face_video_width != expected_face_size or face_video_height != expected_face_size:
998
+ logger.warning(
999
+ f"Reshaping face video from ({face_video_width}, {face_video_height}) to ({expected_face_size},"
1000
+ f" {expected_face_size})"
1001
+ )
1002
+ face_video = self.video_processor.preprocess_video(
1003
+ face_video, height=expected_face_size, width=expected_face_size
1004
+ ).to(device, dtype=torch.float32)
1005
+
1006
+ if mode == "replace":
1007
+ background_video = self.pad_video_frames(background_video, num_target_frames)
1008
+ mask_video = self.pad_video_frames(mask_video, num_target_frames)
1009
+
1010
+ background_video = self.video_processor.preprocess_video(background_video, height=height, width=width).to(
1011
+ device, dtype=torch.float32
1012
+ )
1013
+ mask_video = self.video_processor_for_mask.preprocess_video(mask_video, height=height, width=width).to(
1014
+ device, dtype=torch.float32
1015
+ )
1016
+
1017
+ # 6. Prepare timesteps
1018
+ self.scheduler.set_timesteps(num_inference_steps, device=device)
1019
+ timesteps = self.scheduler.timesteps
1020
+
1021
+ # 7. Prepare latent variables which stay constant for all inference segments
1022
+ num_channels_latents = self.vae.config.z_dim
1023
+
1024
+ # Get VAE-encoded latents of the reference (character) image
1025
+ reference_image_latents = self.prepare_reference_image_latents(
1026
+ image_pixels, batch_size * num_videos_per_prompt, generator=generator, device=device
1027
+ )
1028
+
1029
+ # 8. Loop over video inference segments
1030
+ start = 0
1031
+ end = segment_frame_length # Data space frames, not latent frames
1032
+ all_out_frames = []
1033
+ out_frames = None
1034
+
1035
+ for _ in range(num_segments):
1036
+ assert start + prev_segment_conditioning_frames < cond_video_frames
1037
+
1038
+ # Sample noisy latents from prior for the current inference segment
1039
+ latents = self.prepare_latents(
1040
+ batch_size * num_videos_per_prompt,
1041
+ num_channels_latents=num_channels_latents,
1042
+ height=height,
1043
+ width=width,
1044
+ num_frames=segment_frame_length,
1045
+ dtype=torch.float32,
1046
+ device=device,
1047
+ generator=generator,
1048
+ latents=latents if start == 0 else None, # Only use pre-calculated latents for first segment
1049
+ )
1050
+
1051
+ pose_video_segment = pose_video[:, :, start:end]
1052
+ face_video_segment = face_video[:, :, start:end]
1053
+
1054
+ face_video_segment = face_video_segment.expand(batch_size * num_videos_per_prompt, -1, -1, -1, -1)
1055
+ face_video_segment = face_video_segment.to(dtype=transformer_dtype)
1056
+
1057
+ if start > 0:
1058
+ prev_segment_cond_video = out_frames[:, :, -prev_segment_conditioning_frames:].clone().detach()
1059
+ else:
1060
+ prev_segment_cond_video = None
1061
+
1062
+ if mode == "replace":
1063
+ background_video_segment = background_video[:, :, start:end]
1064
+ mask_video_segment = mask_video[:, :, start:end]
1065
+
1066
+ background_video_segment = background_video_segment.expand(
1067
+ batch_size * num_videos_per_prompt, -1, -1, -1, -1
1068
+ )
1069
+ mask_video_segment = mask_video_segment.expand(batch_size * num_videos_per_prompt, -1, -1, -1, -1)
1070
+ else:
1071
+ background_video_segment = None
1072
+ mask_video_segment = None
1073
+
1074
+ pose_latents = self.prepare_pose_latents(
1075
+ pose_video_segment, batch_size * num_videos_per_prompt, generator=generator, device=device
1076
+ )
1077
+ pose_latents = pose_latents.to(dtype=transformer_dtype)
1078
+
1079
+ prev_segment_cond_latents = self.prepare_prev_segment_cond_latents(
1080
+ prev_segment_cond_video,
1081
+ background_video=background_video_segment,
1082
+ mask_video=mask_video_segment,
1083
+ batch_size=batch_size * num_videos_per_prompt,
1084
+ segment_frame_length=segment_frame_length,
1085
+ start_frame=start,
1086
+ height=height,
1087
+ width=width,
1088
+ prev_segment_cond_frames=prev_segment_conditioning_frames,
1089
+ task=mode,
1090
+ generator=generator,
1091
+ device=device,
1092
+ )
1093
+
1094
+ # Concatenate the reference latents in the frame dimension
1095
+ reference_latents = torch.cat([reference_image_latents, prev_segment_cond_latents], dim=2)
1096
+
1097
+ # 8.1 Denoising loop
1098
+ num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
1099
+ self._num_timesteps = len(timesteps)
1100
+
1101
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
1102
+ for i, t in enumerate(timesteps):
1103
+ if self.interrupt:
1104
+ continue
1105
+
1106
+ self._current_timestep = t
1107
+
1108
+ # Concatenate the reference image + prev segment conditioning in the channel dim
1109
+ latent_model_input = torch.cat([latents, reference_latents], dim=1).to(transformer_dtype)
1110
+ timestep = t.expand(latents.shape[0])
1111
+
1112
+ with self.transformer.cache_context("cond"):
1113
+ noise_pred = self.transformer(
1114
+ hidden_states=latent_model_input,
1115
+ timestep=timestep,
1116
+ encoder_hidden_states=prompt_embeds,
1117
+ encoder_hidden_states_image=image_embeds,
1118
+ pose_hidden_states=pose_latents,
1119
+ face_pixel_values=face_video_segment,
1120
+ motion_encode_batch_size=motion_encode_batch_size,
1121
+ attention_kwargs=attention_kwargs,
1122
+ return_dict=False,
1123
+ )[0]
1124
+
1125
+ if self.do_classifier_free_guidance:
1126
+ # Blank out face for unconditional guidance (set all pixels to -1)
1127
+ face_pixel_values_uncond = face_video_segment * 0 - 1
1128
+ with self.transformer.cache_context("uncond"):
1129
+ noise_uncond = self.transformer(
1130
+ hidden_states=latent_model_input,
1131
+ timestep=timestep,
1132
+ encoder_hidden_states=negative_prompt_embeds,
1133
+ encoder_hidden_states_image=image_embeds,
1134
+ pose_hidden_states=pose_latents,
1135
+ face_pixel_values=face_pixel_values_uncond,
1136
+ motion_encode_batch_size=motion_encode_batch_size,
1137
+ attention_kwargs=attention_kwargs,
1138
+ return_dict=False,
1139
+ )[0]
1140
+ noise_pred = noise_uncond + guidance_scale * (noise_pred - noise_uncond)
1141
+
1142
+ # compute the previous noisy sample x_t -> x_t-1
1143
+ latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
1144
+
1145
+ if callback_on_step_end is not None:
1146
+ callback_kwargs = {}
1147
+ for k in callback_on_step_end_tensor_inputs:
1148
+ callback_kwargs[k] = locals()[k]
1149
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
1150
+
1151
+ latents = callback_outputs.pop("latents", latents)
1152
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
1153
+ negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
1154
+
1155
+ # call the callback, if provided
1156
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
1157
+ progress_bar.update()
1158
+
1159
+ if XLA_AVAILABLE:
1160
+ xm.mark_step()
1161
+
1162
+ latents = latents.to(self.vae.dtype)
1163
+ # Destandardize latents in preparation for Wan VAE decoding
1164
+ latents_mean = (
1165
+ torch.tensor(self.vae.config.latents_mean)
1166
+ .view(1, self.vae.config.z_dim, 1, 1, 1)
1167
+ .to(latents.device, latents.dtype)
1168
+ )
1169
+ latents_recip_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(
1170
+ 1, self.vae.config.z_dim, 1, 1, 1
1171
+ ).to(latents.device, latents.dtype)
1172
+ latents = latents / latents_recip_std + latents_mean
1173
+ # Skip the first latent frame (used for conditioning)
1174
+ out_frames = self.vae.decode(latents[:, :, 1:], return_dict=False)[0]
1175
+
1176
+ if start > 0:
1177
+ out_frames = out_frames[:, :, prev_segment_conditioning_frames:]
1178
+ all_out_frames.append(out_frames)
1179
+
1180
+ start += effective_segment_length
1181
+ end += effective_segment_length
1182
+
1183
+ # Reset scheduler timesteps / state for next denoising loop
1184
+ self.scheduler.set_timesteps(num_inference_steps, device=device)
1185
+ timesteps = self.scheduler.timesteps
1186
+
1187
+ self._current_timestep = None
1188
+ assert start + prev_segment_conditioning_frames >= cond_video_frames
1189
+
1190
+ if not output_type == "latent":
1191
+ video = torch.cat(all_out_frames, dim=2)[:, :, :cond_video_frames]
1192
+ video = self.video_processor.postprocess_video(video, output_type=output_type)
1193
+ else:
1194
+ video = latents
1195
+
1196
+ # Offload all models
1197
+ self.maybe_free_model_hooks()
1198
+
1199
+ if not return_dict:
1200
+ return (video,)
1201
+
1202
+ return WanPipelineOutput(frames=video)
western_ai/europe/diffusers/src/diffusers/pipelines/wan/pipeline_wan_i2v.py ADDED
@@ -0,0 +1,841 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 The Wan Team and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import html
16
+ from typing import Any, Callable
17
+
18
+ import PIL
19
+ import regex as re
20
+ import torch
21
+ from transformers import AutoTokenizer, CLIPImageProcessor, CLIPVisionModel, UMT5EncoderModel
22
+
23
+ from ...callbacks import MultiPipelineCallbacks, PipelineCallback
24
+ from ...image_processor import PipelineImageInput
25
+ from ...loaders import WanLoraLoaderMixin
26
+ from ...models import AutoencoderKLWan, WanTransformer3DModel
27
+ from ...schedulers import FlowMatchEulerDiscreteScheduler
28
+ from ...utils import is_ftfy_available, is_torch_xla_available, logging, replace_example_docstring
29
+ from ...utils.torch_utils import randn_tensor
30
+ from ...video_processor import VideoProcessor
31
+ from ..pipeline_utils import DiffusionPipeline
32
+ from .pipeline_output import WanPipelineOutput
33
+
34
+
35
+ if is_torch_xla_available():
36
+ import torch_xla.core.xla_model as xm
37
+
38
+ XLA_AVAILABLE = True
39
+ else:
40
+ XLA_AVAILABLE = False
41
+
42
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
43
+
44
+ if is_ftfy_available():
45
+ import ftfy
46
+
47
+ EXAMPLE_DOC_STRING = """
48
+ Examples:
49
+ ```python
50
+ >>> import torch
51
+ >>> import numpy as np
52
+ >>> from diffusers import AutoencoderKLWan, WanImageToVideoPipeline
53
+ >>> from diffusers.utils import export_to_video, load_image
54
+ >>> from transformers import CLIPVisionModel
55
+
56
+ >>> # Available models: Wan-AI/Wan2.1-I2V-14B-480P-Diffusers, Wan-AI/Wan2.1-I2V-14B-720P-Diffusers
57
+ >>> model_id = "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers"
58
+ >>> image_encoder = CLIPVisionModel.from_pretrained(
59
+ ... model_id, subfolder="image_encoder", torch_dtype=torch.float32
60
+ ... )
61
+ >>> vae = AutoencoderKLWan.from_pretrained(model_id, subfolder="vae", torch_dtype=torch.float32)
62
+ >>> pipe = WanImageToVideoPipeline.from_pretrained(
63
+ ... model_id, vae=vae, image_encoder=image_encoder, torch_dtype=torch.bfloat16
64
+ ... )
65
+ >>> pipe.to("cuda")
66
+
67
+ >>> image = load_image(
68
+ ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg"
69
+ ... )
70
+ >>> max_area = 480 * 832
71
+ >>> aspect_ratio = image.height / image.width
72
+ >>> mod_value = pipe.vae_scale_factor_spatial * pipe.transformer.config.patch_size[1]
73
+ >>> height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
74
+ >>> width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
75
+ >>> image = image.resize((width, height))
76
+ >>> prompt = (
77
+ ... "An astronaut hatching from an egg, on the surface of the moon, the darkness and depth of space realised in "
78
+ ... "the background. High quality, ultrarealistic detail and breath-taking movie-like camera shot."
79
+ ... )
80
+ >>> negative_prompt = "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards"
81
+
82
+ >>> output = pipe(
83
+ ... image=image,
84
+ ... prompt=prompt,
85
+ ... negative_prompt=negative_prompt,
86
+ ... height=height,
87
+ ... width=width,
88
+ ... num_frames=81,
89
+ ... guidance_scale=5.0,
90
+ ... ).frames[0]
91
+ >>> export_to_video(output, "output.mp4", fps=16)
92
+ ```
93
+ """
94
+
95
+
96
+ def basic_clean(text):
97
+ text = ftfy.fix_text(text)
98
+ text = html.unescape(html.unescape(text))
99
+ return text.strip()
100
+
101
+
102
+ def whitespace_clean(text):
103
+ text = re.sub(r"\s+", " ", text)
104
+ text = text.strip()
105
+ return text
106
+
107
+
108
+ def prompt_clean(text):
109
+ text = whitespace_clean(basic_clean(text))
110
+ return text
111
+
112
+
113
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
114
+ def retrieve_latents(
115
+ encoder_output: torch.Tensor, generator: torch.Generator | None = None, sample_mode: str = "sample"
116
+ ):
117
+ if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
118
+ return encoder_output.latent_dist.sample(generator)
119
+ elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
120
+ return encoder_output.latent_dist.mode()
121
+ elif hasattr(encoder_output, "latents"):
122
+ return encoder_output.latents
123
+ else:
124
+ raise AttributeError("Could not access latents of provided encoder_output")
125
+
126
+
127
+ class WanImageToVideoPipeline(DiffusionPipeline, WanLoraLoaderMixin):
128
+ r"""
129
+ Pipeline for image-to-video generation using Wan.
130
+
131
+ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods
132
+ implemented for all pipelines (downloading, saving, running on a particular device, etc.).
133
+
134
+ Args:
135
+ tokenizer ([`T5Tokenizer`]):
136
+ Tokenizer from [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5Tokenizer),
137
+ specifically the [google/umt5-xxl](https://huggingface.co/google/umt5-xxl) variant.
138
+ text_encoder ([`T5EncoderModel`]):
139
+ [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5EncoderModel), specifically
140
+ the [google/umt5-xxl](https://huggingface.co/google/umt5-xxl) variant.
141
+ image_encoder ([`CLIPVisionModel`]):
142
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPVisionModel), specifically
143
+ the
144
+ [clip-vit-huge-patch14](https://github.com/mlfoundations/open_clip/blob/main/docs/PRETRAINED.md#vit-h14-xlm-roberta-large)
145
+ variant.
146
+ transformer ([`WanTransformer3DModel`]):
147
+ Conditional Transformer to denoise the input latents.
148
+ scheduler ([`UniPCMultistepScheduler`]):
149
+ A scheduler to be used in combination with `transformer` to denoise the encoded image latents.
150
+ vae ([`AutoencoderKLWan`]):
151
+ Variational Auto-Encoder (VAE) Model to encode and decode videos to and from latent representations.
152
+ transformer_2 ([`WanTransformer3DModel`], *optional*):
153
+ Conditional Transformer to denoise the input latents during the low-noise stage. In two-stage denoising,
154
+ `transformer` handles high-noise stages and `transformer_2` handles low-noise stages. If not provided, only
155
+ `transformer` is used.
156
+ boundary_ratio (`float`, *optional*, defaults to `None`):
157
+ Ratio of total timesteps to use as the boundary for switching between transformers in two-stage denoising.
158
+ The actual boundary timestep is calculated as `boundary_ratio * num_train_timesteps`. When provided,
159
+ `transformer` handles timesteps >= boundary_timestep and `transformer_2` handles timesteps <
160
+ boundary_timestep. If `None`, only `transformer` is used for the entire denoising process.
161
+ """
162
+
163
+ model_cpu_offload_seq = "text_encoder->image_encoder->transformer->transformer_2->vae"
164
+ _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"]
165
+ _optional_components = ["transformer", "transformer_2", "image_encoder", "image_processor"]
166
+
167
+ def __init__(
168
+ self,
169
+ tokenizer: AutoTokenizer,
170
+ text_encoder: UMT5EncoderModel,
171
+ vae: AutoencoderKLWan,
172
+ scheduler: FlowMatchEulerDiscreteScheduler,
173
+ image_processor: CLIPImageProcessor = None,
174
+ image_encoder: CLIPVisionModel = None,
175
+ transformer: WanTransformer3DModel = None,
176
+ transformer_2: WanTransformer3DModel = None,
177
+ boundary_ratio: float | None = None,
178
+ expand_timesteps: bool = False,
179
+ ):
180
+ super().__init__()
181
+
182
+ self.register_modules(
183
+ vae=vae,
184
+ text_encoder=text_encoder,
185
+ tokenizer=tokenizer,
186
+ image_encoder=image_encoder,
187
+ transformer=transformer,
188
+ scheduler=scheduler,
189
+ image_processor=image_processor,
190
+ transformer_2=transformer_2,
191
+ )
192
+ self.register_to_config(boundary_ratio=boundary_ratio, expand_timesteps=expand_timesteps)
193
+
194
+ self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if getattr(self, "vae", None) else 4
195
+ self.vae_scale_factor_spatial = self.vae.config.scale_factor_spatial if getattr(self, "vae", None) else 8
196
+ self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial)
197
+ self.image_processor = image_processor
198
+
199
+ def _get_t5_prompt_embeds(
200
+ self,
201
+ prompt: str | list[str] = None,
202
+ num_videos_per_prompt: int = 1,
203
+ max_sequence_length: int = 512,
204
+ device: torch.device | None = None,
205
+ dtype: torch.dtype | None = None,
206
+ ):
207
+ device = device or self._execution_device
208
+ dtype = dtype or self.text_encoder.dtype
209
+
210
+ prompt = [prompt] if isinstance(prompt, str) else prompt
211
+ prompt = [prompt_clean(u) for u in prompt]
212
+ batch_size = len(prompt)
213
+
214
+ text_inputs = self.tokenizer(
215
+ prompt,
216
+ padding="max_length",
217
+ max_length=max_sequence_length,
218
+ truncation=True,
219
+ add_special_tokens=True,
220
+ return_attention_mask=True,
221
+ return_tensors="pt",
222
+ )
223
+ text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask
224
+ seq_lens = mask.gt(0).sum(dim=1).long()
225
+
226
+ prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state
227
+ prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
228
+ prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)]
229
+ prompt_embeds = torch.stack(
230
+ [torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0
231
+ )
232
+
233
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
234
+ _, seq_len, _ = prompt_embeds.shape
235
+ prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1)
236
+ prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
237
+
238
+ return prompt_embeds
239
+
240
+ def encode_image(
241
+ self,
242
+ image: PipelineImageInput,
243
+ device: torch.device | None = None,
244
+ ):
245
+ device = device or self._execution_device
246
+ image = self.image_processor(images=image, return_tensors="pt").to(device)
247
+ image_embeds = self.image_encoder(**image, output_hidden_states=True)
248
+ return image_embeds.hidden_states[-2]
249
+
250
+ # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline.encode_prompt
251
+ def encode_prompt(
252
+ self,
253
+ prompt: str | list[str],
254
+ negative_prompt: str | list[str] | None = None,
255
+ do_classifier_free_guidance: bool = True,
256
+ num_videos_per_prompt: int = 1,
257
+ prompt_embeds: torch.Tensor | None = None,
258
+ negative_prompt_embeds: torch.Tensor | None = None,
259
+ max_sequence_length: int = 226,
260
+ device: torch.device | None = None,
261
+ dtype: torch.dtype | None = None,
262
+ ):
263
+ r"""
264
+ Encodes the prompt into text encoder hidden states.
265
+
266
+ Args:
267
+ prompt (`str` or `list[str]`, *optional*):
268
+ prompt to be encoded
269
+ negative_prompt (`str` or `list[str]`, *optional*):
270
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
271
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
272
+ less than `1`).
273
+ do_classifier_free_guidance (`bool`, *optional*, defaults to `True`):
274
+ Whether to use classifier free guidance or not.
275
+ num_videos_per_prompt (`int`, *optional*, defaults to 1):
276
+ Number of videos that should be generated per prompt. torch device to place the resulting embeddings on
277
+ prompt_embeds (`torch.Tensor`, *optional*):
278
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
279
+ provided, text embeddings will be generated from `prompt` input argument.
280
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
281
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
282
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
283
+ argument.
284
+ device: (`torch.device`, *optional*):
285
+ torch device
286
+ dtype: (`torch.dtype`, *optional*):
287
+ torch dtype
288
+ """
289
+ device = device or self._execution_device
290
+
291
+ prompt = [prompt] if isinstance(prompt, str) else prompt
292
+ if prompt is not None:
293
+ batch_size = len(prompt)
294
+ else:
295
+ batch_size = prompt_embeds.shape[0]
296
+
297
+ if prompt_embeds is None:
298
+ prompt_embeds = self._get_t5_prompt_embeds(
299
+ prompt=prompt,
300
+ num_videos_per_prompt=num_videos_per_prompt,
301
+ max_sequence_length=max_sequence_length,
302
+ device=device,
303
+ dtype=dtype,
304
+ )
305
+
306
+ if do_classifier_free_guidance and negative_prompt_embeds is None:
307
+ negative_prompt = negative_prompt or ""
308
+ negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
309
+
310
+ if prompt is not None and type(prompt) is not type(negative_prompt):
311
+ raise TypeError(
312
+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
313
+ f" {type(prompt)}."
314
+ )
315
+ elif batch_size != len(negative_prompt):
316
+ raise ValueError(
317
+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
318
+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
319
+ " the batch size of `prompt`."
320
+ )
321
+
322
+ negative_prompt_embeds = self._get_t5_prompt_embeds(
323
+ prompt=negative_prompt,
324
+ num_videos_per_prompt=num_videos_per_prompt,
325
+ max_sequence_length=max_sequence_length,
326
+ device=device,
327
+ dtype=dtype,
328
+ )
329
+
330
+ return prompt_embeds, negative_prompt_embeds
331
+
332
+ def check_inputs(
333
+ self,
334
+ prompt,
335
+ negative_prompt,
336
+ image,
337
+ height,
338
+ width,
339
+ prompt_embeds=None,
340
+ negative_prompt_embeds=None,
341
+ image_embeds=None,
342
+ callback_on_step_end_tensor_inputs=None,
343
+ guidance_scale_2=None,
344
+ ):
345
+ if image is not None and image_embeds is not None:
346
+ raise ValueError(
347
+ f"Cannot forward both `image`: {image} and `image_embeds`: {image_embeds}. Please make sure to"
348
+ " only forward one of the two."
349
+ )
350
+ if image is None and image_embeds is None:
351
+ raise ValueError(
352
+ "Provide either `image` or `prompt_embeds`. Cannot leave both `image` and `image_embeds` undefined."
353
+ )
354
+ if image is not None and not isinstance(image, torch.Tensor) and not isinstance(image, PIL.Image.Image):
355
+ raise ValueError(f"`image` has to be of type `torch.Tensor` or `PIL.Image.Image` but is {type(image)}")
356
+ if height % 16 != 0 or width % 16 != 0:
357
+ raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.")
358
+
359
+ if callback_on_step_end_tensor_inputs is not None and not all(
360
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
361
+ ):
362
+ raise ValueError(
363
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
364
+ )
365
+
366
+ if prompt is not None and prompt_embeds is not None:
367
+ raise ValueError(
368
+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
369
+ " only forward one of the two."
370
+ )
371
+ elif negative_prompt is not None and negative_prompt_embeds is not None:
372
+ raise ValueError(
373
+ f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`: {negative_prompt_embeds}. Please make sure to"
374
+ " only forward one of the two."
375
+ )
376
+ elif prompt is None and prompt_embeds is None:
377
+ raise ValueError(
378
+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
379
+ )
380
+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
381
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
382
+ elif negative_prompt is not None and (
383
+ not isinstance(negative_prompt, str) and not isinstance(negative_prompt, list)
384
+ ):
385
+ raise ValueError(f"`negative_prompt` has to be of type `str` or `list` but is {type(negative_prompt)}")
386
+
387
+ if self.config.boundary_ratio is None and guidance_scale_2 is not None:
388
+ raise ValueError("`guidance_scale_2` is only supported when the pipeline's `boundary_ratio` is not None.")
389
+
390
+ if self.config.boundary_ratio is not None and image_embeds is not None:
391
+ raise ValueError("Cannot forward `image_embeds` when the pipeline's `boundary_ratio` is not configured.")
392
+
393
+ def prepare_latents(
394
+ self,
395
+ image: PipelineImageInput,
396
+ batch_size: int,
397
+ num_channels_latents: int = 16,
398
+ height: int = 480,
399
+ width: int = 832,
400
+ num_frames: int = 81,
401
+ dtype: torch.dtype | None = None,
402
+ device: torch.device | None = None,
403
+ generator: torch.Generator | list[torch.Generator] | None = None,
404
+ latents: torch.Tensor | None = None,
405
+ last_image: torch.Tensor | None = None,
406
+ ) -> tuple[torch.Tensor, torch.Tensor]:
407
+ num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1
408
+ latent_height = height // self.vae_scale_factor_spatial
409
+ latent_width = width // self.vae_scale_factor_spatial
410
+
411
+ shape = (batch_size, num_channels_latents, num_latent_frames, latent_height, latent_width)
412
+ if isinstance(generator, list) and len(generator) != batch_size:
413
+ raise ValueError(
414
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
415
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
416
+ )
417
+
418
+ if latents is None:
419
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
420
+ else:
421
+ latents = latents.to(device=device, dtype=dtype)
422
+
423
+ image = image.unsqueeze(2) # [batch_size, channels, 1, height, width]
424
+
425
+ if self.config.expand_timesteps:
426
+ video_condition = image
427
+
428
+ elif last_image is None:
429
+ video_condition = torch.cat(
430
+ [image, image.new_zeros(image.shape[0], image.shape[1], num_frames - 1, height, width)], dim=2
431
+ )
432
+ else:
433
+ last_image = last_image.unsqueeze(2)
434
+ video_condition = torch.cat(
435
+ [image, image.new_zeros(image.shape[0], image.shape[1], num_frames - 2, height, width), last_image],
436
+ dim=2,
437
+ )
438
+ video_condition = video_condition.to(device=device, dtype=self.vae.dtype)
439
+
440
+ latents_mean = (
441
+ torch.tensor(self.vae.config.latents_mean)
442
+ .view(1, self.vae.config.z_dim, 1, 1, 1)
443
+ .to(latents.device, latents.dtype)
444
+ )
445
+ latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
446
+ latents.device, latents.dtype
447
+ )
448
+
449
+ if isinstance(generator, list):
450
+ latent_condition = [
451
+ retrieve_latents(self.vae.encode(video_condition), sample_mode="argmax") for _ in generator
452
+ ]
453
+ latent_condition = torch.cat(latent_condition)
454
+ else:
455
+ latent_condition = retrieve_latents(self.vae.encode(video_condition), sample_mode="argmax")
456
+ latent_condition = latent_condition.repeat(batch_size, 1, 1, 1, 1)
457
+
458
+ latent_condition = latent_condition.to(dtype)
459
+ latent_condition = (latent_condition - latents_mean) * latents_std
460
+
461
+ if self.config.expand_timesteps:
462
+ first_frame_mask = torch.ones(
463
+ 1, 1, num_latent_frames, latent_height, latent_width, dtype=dtype, device=device
464
+ )
465
+ first_frame_mask[:, :, 0] = 0
466
+ return latents, latent_condition, first_frame_mask
467
+
468
+ mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, latent_width)
469
+
470
+ if last_image is None:
471
+ mask_lat_size[:, :, list(range(1, num_frames))] = 0
472
+ else:
473
+ mask_lat_size[:, :, list(range(1, num_frames - 1))] = 0
474
+ first_frame_mask = mask_lat_size[:, :, 0:1]
475
+ first_frame_mask = torch.repeat_interleave(first_frame_mask, dim=2, repeats=self.vae_scale_factor_temporal)
476
+ mask_lat_size = torch.concat([first_frame_mask, mask_lat_size[:, :, 1:, :]], dim=2)
477
+ mask_lat_size = mask_lat_size.view(batch_size, -1, self.vae_scale_factor_temporal, latent_height, latent_width)
478
+ mask_lat_size = mask_lat_size.transpose(1, 2)
479
+ mask_lat_size = mask_lat_size.to(latent_condition.device)
480
+
481
+ return latents, torch.concat([mask_lat_size, latent_condition], dim=1)
482
+
483
+ @property
484
+ def guidance_scale(self):
485
+ return self._guidance_scale
486
+
487
+ @property
488
+ def do_classifier_free_guidance(self):
489
+ return self._guidance_scale > 1
490
+
491
+ @property
492
+ def num_timesteps(self):
493
+ return self._num_timesteps
494
+
495
+ @property
496
+ def current_timestep(self):
497
+ return self._current_timestep
498
+
499
+ @property
500
+ def interrupt(self):
501
+ return self._interrupt
502
+
503
+ @property
504
+ def attention_kwargs(self):
505
+ return self._attention_kwargs
506
+
507
+ @torch.no_grad()
508
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
509
+ def __call__(
510
+ self,
511
+ image: PipelineImageInput,
512
+ prompt: str | list[str] = None,
513
+ negative_prompt: str | list[str] = None,
514
+ height: int = 480,
515
+ width: int = 832,
516
+ num_frames: int = 81,
517
+ num_inference_steps: int = 50,
518
+ guidance_scale: float = 5.0,
519
+ guidance_scale_2: float | None = None,
520
+ num_videos_per_prompt: int | None = 1,
521
+ generator: torch.Generator | list[torch.Generator] | None = None,
522
+ latents: torch.Tensor | None = None,
523
+ prompt_embeds: torch.Tensor | None = None,
524
+ negative_prompt_embeds: torch.Tensor | None = None,
525
+ image_embeds: torch.Tensor | None = None,
526
+ last_image: torch.Tensor | None = None,
527
+ output_type: str | None = "np",
528
+ return_dict: bool = True,
529
+ attention_kwargs: dict[str, Any] | None = None,
530
+ callback_on_step_end: Callable[[int, int], None] | PipelineCallback | MultiPipelineCallbacks | None = None,
531
+ callback_on_step_end_tensor_inputs: list[str] = ["latents"],
532
+ max_sequence_length: int = 512,
533
+ ):
534
+ r"""
535
+ The call function to the pipeline for generation.
536
+
537
+ Args:
538
+ image (`PipelineImageInput`):
539
+ The input image to condition the generation on. Must be an image, a list of images or a `torch.Tensor`.
540
+ last_image (`torch.Tensor`, *optional*):
541
+ Optional last frame to condition the generated video on. When provided, the model interpolates between
542
+ `image` (first frame) and `last_image` (last frame).
543
+ prompt (`str` or `list[str]`, *optional*):
544
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
545
+ instead.
546
+ negative_prompt (`str` or `list[str]`, *optional*):
547
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
548
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
549
+ less than `1`).
550
+ height (`int`, defaults to `480`):
551
+ The height of the generated video.
552
+ width (`int`, defaults to `832`):
553
+ The width of the generated video.
554
+ num_frames (`int`, defaults to `81`):
555
+ The number of frames in the generated video.
556
+ num_inference_steps (`int`, defaults to `50`):
557
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
558
+ expense of slower inference.
559
+ guidance_scale (`float`, defaults to `5.0`):
560
+ Guidance scale as defined in [Classifier-Free Diffusion
561
+ Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2.
562
+ of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
563
+ `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to
564
+ the text `prompt`, usually at the expense of lower image quality.
565
+ guidance_scale_2 (`float`, *optional*, defaults to `None`):
566
+ Guidance scale for the low-noise stage transformer (`transformer_2`). If `None` and the pipeline's
567
+ `boundary_ratio` is not None, uses the same value as `guidance_scale`. Only used when `transformer_2`
568
+ and the pipeline's `boundary_ratio` are not None.
569
+ num_videos_per_prompt (`int`, *optional*, defaults to 1):
570
+ The number of images to generate per prompt.
571
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
572
+ A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make
573
+ generation deterministic.
574
+ latents (`torch.Tensor`, *optional*):
575
+ Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image
576
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
577
+ tensor is generated by sampling using the supplied random `generator`.
578
+ prompt_embeds (`torch.Tensor`, *optional*):
579
+ Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not
580
+ provided, text embeddings are generated from the `prompt` input argument.
581
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
582
+ Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not
583
+ provided, text embeddings are generated from the `negative_prompt` input argument.
584
+ image_embeds (`torch.Tensor`, *optional*):
585
+ Pre-generated image embeddings. Can be used to easily tweak image inputs (weighting). If not provided,
586
+ image embeddings are generated from the `image` input argument.
587
+ output_type (`str`, *optional*, defaults to `"np"`):
588
+ The output format of the generated image. Choose between `PIL.Image` or `np.array`.
589
+ return_dict (`bool`, *optional*, defaults to `True`):
590
+ Whether or not to return a [`WanPipelineOutput`] instead of a plain tuple.
591
+ attention_kwargs (`dict`, *optional*):
592
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
593
+ `self.processor` in
594
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
595
+ callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*):
596
+ A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of
597
+ each denoising step during the inference. with the following arguments: `callback_on_step_end(self:
598
+ DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a
599
+ list of all tensors as specified by `callback_on_step_end_tensor_inputs`.
600
+ callback_on_step_end_tensor_inputs (`list`, *optional*):
601
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
602
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
603
+ `._callback_tensor_inputs` attribute of your pipeline class.
604
+ max_sequence_length (`int`, defaults to `512`):
605
+ The maximum sequence length of the text encoder. If the prompt is longer than this, it will be
606
+ truncated. If the prompt is shorter, it will be padded to this length.
607
+
608
+ Examples:
609
+
610
+ Returns:
611
+ [`~WanPipelineOutput`] or `tuple`:
612
+ If `return_dict` is `True`, [`WanPipelineOutput`] is returned, otherwise a `tuple` is returned where
613
+ the first element is a list with the generated images and the second element is a list of `bool`s
614
+ indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content.
615
+ """
616
+
617
+ if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
618
+ callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
619
+
620
+ # 1. Check inputs. Raise error if not correct
621
+ self.check_inputs(
622
+ prompt,
623
+ negative_prompt,
624
+ image,
625
+ height,
626
+ width,
627
+ prompt_embeds,
628
+ negative_prompt_embeds,
629
+ image_embeds,
630
+ callback_on_step_end_tensor_inputs,
631
+ guidance_scale_2,
632
+ )
633
+
634
+ if num_frames % self.vae_scale_factor_temporal != 1:
635
+ logger.warning(
636
+ f"`num_frames - 1` has to be divisible by {self.vae_scale_factor_temporal}. Rounding to the nearest number."
637
+ )
638
+ num_frames = num_frames // self.vae_scale_factor_temporal * self.vae_scale_factor_temporal + 1
639
+ num_frames = max(num_frames, 1)
640
+
641
+ patch_size = (
642
+ self.transformer.config.patch_size
643
+ if self.transformer is not None
644
+ else self.transformer_2.config.patch_size
645
+ )
646
+ h_multiple_of = self.vae_scale_factor_spatial * patch_size[1]
647
+ w_multiple_of = self.vae_scale_factor_spatial * patch_size[2]
648
+ calc_height = height // h_multiple_of * h_multiple_of
649
+ calc_width = width // w_multiple_of * w_multiple_of
650
+ if height != calc_height or width != calc_width:
651
+ logger.warning(
652
+ f"`height` and `width` must be multiples of ({h_multiple_of}, {w_multiple_of}) for proper patchification. "
653
+ f"Adjusting ({height}, {width}) -> ({calc_height}, {calc_width})."
654
+ )
655
+ height, width = calc_height, calc_width
656
+
657
+ if self.config.boundary_ratio is not None and guidance_scale_2 is None:
658
+ guidance_scale_2 = guidance_scale
659
+
660
+ self._guidance_scale = guidance_scale
661
+ self._guidance_scale_2 = guidance_scale_2
662
+ self._attention_kwargs = attention_kwargs
663
+ self._current_timestep = None
664
+ self._interrupt = False
665
+
666
+ device = self._execution_device
667
+
668
+ # 2. Define call parameters
669
+ if prompt is not None and isinstance(prompt, str):
670
+ batch_size = 1
671
+ elif prompt is not None and isinstance(prompt, list):
672
+ batch_size = len(prompt)
673
+ else:
674
+ batch_size = prompt_embeds.shape[0]
675
+
676
+ # 3. Encode input prompt
677
+ prompt_embeds, negative_prompt_embeds = self.encode_prompt(
678
+ prompt=prompt,
679
+ negative_prompt=negative_prompt,
680
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
681
+ num_videos_per_prompt=num_videos_per_prompt,
682
+ prompt_embeds=prompt_embeds,
683
+ negative_prompt_embeds=negative_prompt_embeds,
684
+ max_sequence_length=max_sequence_length,
685
+ device=device,
686
+ )
687
+
688
+ # Encode image embedding
689
+ transformer_dtype = self.transformer.dtype if self.transformer is not None else self.transformer_2.dtype
690
+ prompt_embeds = prompt_embeds.to(transformer_dtype)
691
+ if negative_prompt_embeds is not None:
692
+ negative_prompt_embeds = negative_prompt_embeds.to(transformer_dtype)
693
+
694
+ # only wan 2.1 i2v transformer accepts image_embeds
695
+ if self.transformer is not None and self.transformer.config.image_dim is not None:
696
+ if image_embeds is None:
697
+ if last_image is None:
698
+ image_embeds = self.encode_image(image, device)
699
+ else:
700
+ image_embeds = self.encode_image([image, last_image], device)
701
+ image_embeds = image_embeds.repeat(batch_size, 1, 1)
702
+ image_embeds = image_embeds.to(transformer_dtype)
703
+
704
+ # 4. Prepare timesteps
705
+ self.scheduler.set_timesteps(num_inference_steps, device=device)
706
+ timesteps = self.scheduler.timesteps
707
+
708
+ # 5. Prepare latent variables
709
+ num_channels_latents = self.vae.config.z_dim
710
+ image = self.video_processor.preprocess(image, height=height, width=width).to(device, dtype=torch.float32)
711
+ if last_image is not None:
712
+ last_image = self.video_processor.preprocess(last_image, height=height, width=width).to(
713
+ device, dtype=torch.float32
714
+ )
715
+
716
+ latents_outputs = self.prepare_latents(
717
+ image,
718
+ batch_size * num_videos_per_prompt,
719
+ num_channels_latents,
720
+ height,
721
+ width,
722
+ num_frames,
723
+ torch.float32,
724
+ device,
725
+ generator,
726
+ latents,
727
+ last_image,
728
+ )
729
+ if self.config.expand_timesteps:
730
+ # wan 2.2 5b i2v use firt_frame_mask to mask timesteps
731
+ latents, condition, first_frame_mask = latents_outputs
732
+ else:
733
+ latents, condition = latents_outputs
734
+
735
+ # 6. Denoising loop
736
+ num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
737
+ self._num_timesteps = len(timesteps)
738
+
739
+ if self.config.boundary_ratio is not None:
740
+ boundary_timestep = self.config.boundary_ratio * self.scheduler.config.num_train_timesteps
741
+ else:
742
+ boundary_timestep = None
743
+
744
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
745
+ for i, t in enumerate(timesteps):
746
+ if self.interrupt:
747
+ continue
748
+
749
+ self._current_timestep = t
750
+
751
+ if boundary_timestep is None or t >= boundary_timestep:
752
+ # wan2.1 or high-noise stage in wan2.2
753
+ current_model = self.transformer
754
+ current_guidance_scale = guidance_scale
755
+ else:
756
+ # low-noise stage in wan2.2
757
+ current_model = self.transformer_2
758
+ current_guidance_scale = guidance_scale_2
759
+
760
+ if self.config.expand_timesteps:
761
+ latent_model_input = (1 - first_frame_mask) * condition + first_frame_mask * latents
762
+ latent_model_input = latent_model_input.to(transformer_dtype)
763
+
764
+ # seq_len: num_latent_frames * (latent_height // patch_size) * (latent_width // patch_size)
765
+ temp_ts = (first_frame_mask[0][0][:, ::2, ::2] * t).flatten()
766
+ # batch_size, seq_len
767
+ timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
768
+ else:
769
+ latent_model_input = torch.cat([latents, condition], dim=1).to(transformer_dtype)
770
+ timestep = t.expand(latents.shape[0])
771
+
772
+ with current_model.cache_context("cond"):
773
+ noise_pred = current_model(
774
+ hidden_states=latent_model_input,
775
+ timestep=timestep,
776
+ encoder_hidden_states=prompt_embeds,
777
+ encoder_hidden_states_image=image_embeds,
778
+ attention_kwargs=attention_kwargs,
779
+ return_dict=False,
780
+ )[0]
781
+
782
+ if self.do_classifier_free_guidance:
783
+ with current_model.cache_context("uncond"):
784
+ noise_uncond = current_model(
785
+ hidden_states=latent_model_input,
786
+ timestep=timestep,
787
+ encoder_hidden_states=negative_prompt_embeds,
788
+ encoder_hidden_states_image=image_embeds,
789
+ attention_kwargs=attention_kwargs,
790
+ return_dict=False,
791
+ )[0]
792
+ noise_pred = noise_uncond + current_guidance_scale * (noise_pred - noise_uncond)
793
+
794
+ # compute the previous noisy sample x_t -> x_t-1
795
+ latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
796
+
797
+ if callback_on_step_end is not None:
798
+ callback_kwargs = {}
799
+ for k in callback_on_step_end_tensor_inputs:
800
+ callback_kwargs[k] = locals()[k]
801
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
802
+
803
+ latents = callback_outputs.pop("latents", latents)
804
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
805
+ negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
806
+
807
+ # call the callback, if provided
808
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
809
+ progress_bar.update()
810
+
811
+ if XLA_AVAILABLE:
812
+ xm.mark_step()
813
+
814
+ self._current_timestep = None
815
+
816
+ if self.config.expand_timesteps:
817
+ latents = (1 - first_frame_mask) * condition + first_frame_mask * latents
818
+
819
+ if not output_type == "latent":
820
+ latents = latents.to(self.vae.dtype)
821
+ latents_mean = (
822
+ torch.tensor(self.vae.config.latents_mean)
823
+ .view(1, self.vae.config.z_dim, 1, 1, 1)
824
+ .to(latents.device, latents.dtype)
825
+ )
826
+ latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
827
+ latents.device, latents.dtype
828
+ )
829
+ latents = latents / latents_std + latents_mean
830
+ video = self.vae.decode(latents, return_dict=False)[0]
831
+ video = self.video_processor.postprocess_video(video, output_type=output_type)
832
+ else:
833
+ video = latents
834
+
835
+ # Offload all models
836
+ self.maybe_free_model_hooks()
837
+
838
+ if not return_dict:
839
+ return (video,)
840
+
841
+ return WanPipelineOutput(frames=video)
western_ai/europe/diffusers/src/diffusers/pipelines/wan/pipeline_wan_vace.py ADDED
@@ -0,0 +1,1046 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 The Wan Team and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import html
16
+ from typing import Any, Callable
17
+
18
+ import PIL.Image
19
+ import regex as re
20
+ import torch
21
+ from transformers import AutoTokenizer, UMT5EncoderModel
22
+
23
+ from ...callbacks import MultiPipelineCallbacks, PipelineCallback
24
+ from ...image_processor import PipelineImageInput
25
+ from ...loaders import WanLoraLoaderMixin
26
+ from ...models import AutoencoderKLWan, WanVACETransformer3DModel
27
+ from ...schedulers import FlowMatchEulerDiscreteScheduler
28
+ from ...utils import is_ftfy_available, is_torch_xla_available, logging, replace_example_docstring
29
+ from ...utils.torch_utils import randn_tensor
30
+ from ...video_processor import VideoProcessor
31
+ from ..pipeline_utils import DiffusionPipeline
32
+ from .pipeline_output import WanPipelineOutput
33
+
34
+
35
+ if is_torch_xla_available():
36
+ import torch_xla.core.xla_model as xm
37
+
38
+ XLA_AVAILABLE = True
39
+ else:
40
+ XLA_AVAILABLE = False
41
+
42
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
43
+
44
+ if is_ftfy_available():
45
+ import ftfy
46
+
47
+
48
+ EXAMPLE_DOC_STRING = """
49
+ Examples:
50
+ ```python
51
+ >>> import torch
52
+ >>> import PIL.Image
53
+ >>> from diffusers import AutoencoderKLWan, WanVACEPipeline
54
+ >>> from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
55
+ >>> from diffusers.utils import export_to_video, load_image
56
+ def prepare_video_and_mask(first_img: PIL.Image.Image, last_img: PIL.Image.Image, height: int, width: int, num_frames: int):
57
+ first_img = first_img.resize((width, height))
58
+ last_img = last_img.resize((width, height))
59
+ frames = []
60
+ frames.append(first_img)
61
+ # Ideally, this should be 127.5 to match original code, but they perform computation on numpy arrays
62
+ # whereas we are passing PIL images. If you choose to pass numpy arrays, you can set it to 127.5 to
63
+ # match the original code.
64
+ frames.extend([PIL.Image.new("RGB", (width, height), (128, 128, 128))] * (num_frames - 2))
65
+ frames.append(last_img)
66
+ mask_black = PIL.Image.new("L", (width, height), 0)
67
+ mask_white = PIL.Image.new("L", (width, height), 255)
68
+ mask = [mask_black, *[mask_white] * (num_frames - 2), mask_black]
69
+ return frames, mask
70
+
71
+ >>> # Available checkpoints: Wan-AI/Wan2.1-VACE-1.3B-diffusers, Wan-AI/Wan2.1-VACE-14B-diffusers
72
+ >>> model_id = "Wan-AI/Wan2.1-VACE-1.3B-diffusers"
73
+ >>> vae = AutoencoderKLWan.from_pretrained(model_id, subfolder="vae", torch_dtype=torch.float32)
74
+ >>> pipe = WanVACEPipeline.from_pretrained(model_id, vae=vae, torch_dtype=torch.bfloat16)
75
+ >>> flow_shift = 3.0 # 5.0 for 720P, 3.0 for 480P
76
+ >>> pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config, flow_shift=flow_shift)
77
+ >>> pipe.to("cuda")
78
+
79
+ >>> prompt = "CG animation style, a small blue bird takes off from the ground, flapping its wings. The bird's feathers are delicate, with a unique pattern on its chest. The background shows a blue sky with white clouds under bright sunshine. The camera follows the bird upward, capturing its flight and the vastness of the sky from a close-up, low-angle perspective."
80
+ >>> negative_prompt = "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards"
81
+ >>> first_frame = load_image(
82
+ ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/flf2v_input_first_frame.png"
83
+ ... )
84
+ >>> last_frame = load_image(
85
+ ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/flf2v_input_last_frame.png>>> "
86
+ ... )
87
+
88
+ >>> height = 512
89
+ >>> width = 512
90
+ >>> num_frames = 81
91
+ >>> video, mask = prepare_video_and_mask(first_frame, last_frame, height, width, num_frames)
92
+
93
+ >>> output = pipe(
94
+ ... video=video,
95
+ ... mask=mask,
96
+ ... prompt=prompt,
97
+ ... negative_prompt=negative_prompt,
98
+ ... height=height,
99
+ ... width=width,
100
+ ... num_frames=num_frames,
101
+ ... num_inference_steps=30,
102
+ ... guidance_scale=5.0,
103
+ ... generator=torch.Generator().manual_seed(42),
104
+ ... ).frames[0]
105
+ >>> export_to_video(output, "output.mp4", fps=16)
106
+ ```
107
+ """
108
+
109
+
110
+ def basic_clean(text):
111
+ text = ftfy.fix_text(text)
112
+ text = html.unescape(html.unescape(text))
113
+ return text.strip()
114
+
115
+
116
+ def whitespace_clean(text):
117
+ text = re.sub(r"\s+", " ", text)
118
+ text = text.strip()
119
+ return text
120
+
121
+
122
+ def prompt_clean(text):
123
+ text = whitespace_clean(basic_clean(text))
124
+ return text
125
+
126
+
127
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
128
+ def retrieve_latents(
129
+ encoder_output: torch.Tensor, generator: torch.Generator | None = None, sample_mode: str = "sample"
130
+ ):
131
+ if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
132
+ return encoder_output.latent_dist.sample(generator)
133
+ elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
134
+ return encoder_output.latent_dist.mode()
135
+ elif hasattr(encoder_output, "latents"):
136
+ return encoder_output.latents
137
+ else:
138
+ raise AttributeError("Could not access latents of provided encoder_output")
139
+
140
+
141
+ class WanVACEPipeline(DiffusionPipeline, WanLoraLoaderMixin):
142
+ r"""
143
+ Pipeline for controllable generation using Wan.
144
+
145
+ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods
146
+ implemented for all pipelines (downloading, saving, running on a particular device, etc.).
147
+
148
+ Args:
149
+ tokenizer ([`T5Tokenizer`]):
150
+ Tokenizer from [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5Tokenizer),
151
+ specifically the [google/umt5-xxl](https://huggingface.co/google/umt5-xxl) variant.
152
+ text_encoder ([`T5EncoderModel`]):
153
+ [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5EncoderModel), specifically
154
+ the [google/umt5-xxl](https://huggingface.co/google/umt5-xxl) variant.
155
+ vae ([`AutoencoderKLWan`]):
156
+ Variational Auto-Encoder (VAE) Model to encode and decode videos to and from latent representations.
157
+ scheduler ([`UniPCMultistepScheduler`]):
158
+ A scheduler to be used in combination with `transformer` to denoise the encoded image latents.
159
+ transformer ([`WanVACETransformer3DModel`], *optional*):
160
+ Conditional Transformer to denoise the input latents during the high-noise stage. In two-stage denoising,
161
+ `transformer` handles high-noise stages and `transformer_2` handles low-noise stages. At least one of
162
+ `transformer` or `transformer_2` must be provided.
163
+ transformer_2 ([`WanVACETransformer3DModel`], *optional*):
164
+ Conditional Transformer to denoise the input latents during the low-noise stage. In two-stage denoising,
165
+ `transformer` handles high-noise stages and `transformer_2` handles low-noise stages. At least one of
166
+ `transformer` or `transformer_2` must be provided.
167
+ boundary_ratio (`float`, *optional*, defaults to `None`):
168
+ Ratio of total timesteps to use as the boundary for switching between transformers in two-stage denoising.
169
+ The actual boundary timestep is calculated as `boundary_ratio * num_train_timesteps`. When provided,
170
+ `transformer` handles timesteps >= boundary_timestep and `transformer_2` handles timesteps <
171
+ boundary_timestep. If `None`, only the available transformer is used for the entire denoising process.
172
+ """
173
+
174
+ model_cpu_offload_seq = "text_encoder->transformer->transformer_2->vae"
175
+ _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"]
176
+ _optional_components = ["transformer", "transformer_2"]
177
+
178
+ def __init__(
179
+ self,
180
+ tokenizer: AutoTokenizer,
181
+ text_encoder: UMT5EncoderModel,
182
+ vae: AutoencoderKLWan,
183
+ scheduler: FlowMatchEulerDiscreteScheduler,
184
+ transformer: WanVACETransformer3DModel = None,
185
+ transformer_2: WanVACETransformer3DModel = None,
186
+ boundary_ratio: float | None = None,
187
+ ):
188
+ super().__init__()
189
+
190
+ self.register_modules(
191
+ vae=vae,
192
+ text_encoder=text_encoder,
193
+ tokenizer=tokenizer,
194
+ transformer=transformer,
195
+ transformer_2=transformer_2,
196
+ scheduler=scheduler,
197
+ )
198
+ self.register_to_config(boundary_ratio=boundary_ratio)
199
+ self.vae_scale_factor_temporal = 2 ** sum(self.vae.temperal_downsample) if getattr(self, "vae", None) else 4
200
+ self.vae_scale_factor_spatial = 2 ** len(self.vae.temperal_downsample) if getattr(self, "vae", None) else 8
201
+ self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial)
202
+
203
+ # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline._get_t5_prompt_embeds
204
+ def _get_t5_prompt_embeds(
205
+ self,
206
+ prompt: str | list[str] = None,
207
+ num_videos_per_prompt: int = 1,
208
+ max_sequence_length: int = 226,
209
+ device: torch.device | None = None,
210
+ dtype: torch.dtype | None = None,
211
+ ):
212
+ device = device or self._execution_device
213
+ dtype = dtype or self.text_encoder.dtype
214
+
215
+ prompt = [prompt] if isinstance(prompt, str) else prompt
216
+ prompt = [prompt_clean(u) for u in prompt]
217
+ batch_size = len(prompt)
218
+
219
+ text_inputs = self.tokenizer(
220
+ prompt,
221
+ padding="max_length",
222
+ max_length=max_sequence_length,
223
+ truncation=True,
224
+ add_special_tokens=True,
225
+ return_attention_mask=True,
226
+ return_tensors="pt",
227
+ )
228
+ text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask
229
+ seq_lens = mask.gt(0).sum(dim=1).long()
230
+
231
+ prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state
232
+ prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
233
+ prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)]
234
+ prompt_embeds = torch.stack(
235
+ [torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0
236
+ )
237
+
238
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
239
+ _, seq_len, _ = prompt_embeds.shape
240
+ prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1)
241
+ prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
242
+
243
+ return prompt_embeds
244
+
245
+ # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline.encode_prompt
246
+ def encode_prompt(
247
+ self,
248
+ prompt: str | list[str],
249
+ negative_prompt: str | list[str] | None = None,
250
+ do_classifier_free_guidance: bool = True,
251
+ num_videos_per_prompt: int = 1,
252
+ prompt_embeds: torch.Tensor | None = None,
253
+ negative_prompt_embeds: torch.Tensor | None = None,
254
+ max_sequence_length: int = 226,
255
+ device: torch.device | None = None,
256
+ dtype: torch.dtype | None = None,
257
+ ):
258
+ r"""
259
+ Encodes the prompt into text encoder hidden states.
260
+
261
+ Args:
262
+ prompt (`str` or `list[str]`, *optional*):
263
+ prompt to be encoded
264
+ negative_prompt (`str` or `list[str]`, *optional*):
265
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
266
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
267
+ less than `1`).
268
+ do_classifier_free_guidance (`bool`, *optional*, defaults to `True`):
269
+ Whether to use classifier free guidance or not.
270
+ num_videos_per_prompt (`int`, *optional*, defaults to 1):
271
+ Number of videos that should be generated per prompt. torch device to place the resulting embeddings on
272
+ prompt_embeds (`torch.Tensor`, *optional*):
273
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
274
+ provided, text embeddings will be generated from `prompt` input argument.
275
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
276
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
277
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
278
+ argument.
279
+ device: (`torch.device`, *optional*):
280
+ torch device
281
+ dtype: (`torch.dtype`, *optional*):
282
+ torch dtype
283
+ """
284
+ device = device or self._execution_device
285
+
286
+ prompt = [prompt] if isinstance(prompt, str) else prompt
287
+ if prompt is not None:
288
+ batch_size = len(prompt)
289
+ else:
290
+ batch_size = prompt_embeds.shape[0]
291
+
292
+ if prompt_embeds is None:
293
+ prompt_embeds = self._get_t5_prompt_embeds(
294
+ prompt=prompt,
295
+ num_videos_per_prompt=num_videos_per_prompt,
296
+ max_sequence_length=max_sequence_length,
297
+ device=device,
298
+ dtype=dtype,
299
+ )
300
+
301
+ if do_classifier_free_guidance and negative_prompt_embeds is None:
302
+ negative_prompt = negative_prompt or ""
303
+ negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
304
+
305
+ if prompt is not None and type(prompt) is not type(negative_prompt):
306
+ raise TypeError(
307
+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
308
+ f" {type(prompt)}."
309
+ )
310
+ elif batch_size != len(negative_prompt):
311
+ raise ValueError(
312
+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
313
+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
314
+ " the batch size of `prompt`."
315
+ )
316
+
317
+ negative_prompt_embeds = self._get_t5_prompt_embeds(
318
+ prompt=negative_prompt,
319
+ num_videos_per_prompt=num_videos_per_prompt,
320
+ max_sequence_length=max_sequence_length,
321
+ device=device,
322
+ dtype=dtype,
323
+ )
324
+
325
+ return prompt_embeds, negative_prompt_embeds
326
+
327
+ def check_inputs(
328
+ self,
329
+ prompt,
330
+ negative_prompt,
331
+ height,
332
+ width,
333
+ prompt_embeds=None,
334
+ negative_prompt_embeds=None,
335
+ callback_on_step_end_tensor_inputs=None,
336
+ video=None,
337
+ mask=None,
338
+ reference_images=None,
339
+ guidance_scale_2=None,
340
+ ):
341
+ if self.transformer is not None:
342
+ base = self.vae_scale_factor_spatial * self.transformer.config.patch_size[1]
343
+ elif self.transformer_2 is not None:
344
+ base = self.vae_scale_factor_spatial * self.transformer_2.config.patch_size[1]
345
+ else:
346
+ raise ValueError(
347
+ "`transformer` or `transformer_2` component must be set in order to run inference with this pipeline"
348
+ )
349
+
350
+ if height % base != 0 or width % base != 0:
351
+ raise ValueError(f"`height` and `width` have to be divisible by {base} but are {height} and {width}.")
352
+
353
+ if callback_on_step_end_tensor_inputs is not None and not all(
354
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
355
+ ):
356
+ raise ValueError(
357
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
358
+ )
359
+ if self.config.boundary_ratio is None and guidance_scale_2 is not None:
360
+ raise ValueError("`guidance_scale_2` is only supported when the pipeline's `boundary_ratio` is not None.")
361
+
362
+ if prompt is not None and prompt_embeds is not None:
363
+ raise ValueError(
364
+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
365
+ " only forward one of the two."
366
+ )
367
+ elif negative_prompt is not None and negative_prompt_embeds is not None:
368
+ raise ValueError(
369
+ f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`: {negative_prompt_embeds}. Please make sure to"
370
+ " only forward one of the two."
371
+ )
372
+ elif prompt is None and prompt_embeds is None:
373
+ raise ValueError(
374
+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
375
+ )
376
+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
377
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
378
+ elif negative_prompt is not None and (
379
+ not isinstance(negative_prompt, str) and not isinstance(negative_prompt, list)
380
+ ):
381
+ raise ValueError(f"`negative_prompt` has to be of type `str` or `list` but is {type(negative_prompt)}")
382
+
383
+ if video is not None:
384
+ if mask is not None:
385
+ if len(video) != len(mask):
386
+ raise ValueError(
387
+ f"Length of `video` {len(video)} and `mask` {len(mask)} do not match. Please make sure that"
388
+ " they have the same length."
389
+ )
390
+ if reference_images is not None:
391
+ is_pil_image = isinstance(reference_images, PIL.Image.Image)
392
+ is_list_of_pil_images = isinstance(reference_images, list) and all(
393
+ isinstance(ref_img, PIL.Image.Image) for ref_img in reference_images
394
+ )
395
+ is_list_of_list_of_pil_images = isinstance(reference_images, list) and all(
396
+ isinstance(ref_img, list) and all(isinstance(ref_img_, PIL.Image.Image) for ref_img_ in ref_img)
397
+ for ref_img in reference_images
398
+ )
399
+ if not (is_pil_image or is_list_of_pil_images or is_list_of_list_of_pil_images):
400
+ raise ValueError(
401
+ "`reference_images` has to be of type `PIL.Image.Image` or `list` of `PIL.Image.Image`, or "
402
+ "`list` of `list` of `PIL.Image.Image`, but is {type(reference_images)}"
403
+ )
404
+ if is_list_of_list_of_pil_images and len(reference_images) != 1:
405
+ raise ValueError(
406
+ "The pipeline only supports generating one video at a time at the moment. When passing a list "
407
+ "of list of reference images, where the outer list corresponds to the batch size and the inner "
408
+ "list corresponds to list of conditioning images per video, please make sure to only pass "
409
+ "one inner list of reference images (i.e., `[[<image1>, <image2>, ...]]`"
410
+ )
411
+ elif mask is not None:
412
+ raise ValueError("`mask` can only be passed if `video` is passed as well.")
413
+
414
+ def preprocess_conditions(
415
+ self,
416
+ video: list[PipelineImageInput] | None = None,
417
+ mask: list[PipelineImageInput] | None = None,
418
+ reference_images: PIL.Image.Image | list[PIL.Image.Image] | list[list[PIL.Image.Image]] | None = None,
419
+ batch_size: int = 1,
420
+ height: int = 480,
421
+ width: int = 832,
422
+ num_frames: int = 81,
423
+ dtype: torch.dtype | None = None,
424
+ device: torch.device | None = None,
425
+ ):
426
+ if video is not None:
427
+ base = self.vae_scale_factor_spatial * (
428
+ self.transformer.config.patch_size[1]
429
+ if self.transformer is not None
430
+ else self.transformer_2.config.patch_size[1]
431
+ )
432
+ video_height, video_width = self.video_processor.get_default_height_width(video[0])
433
+
434
+ if video_height * video_width > height * width:
435
+ scale = min(width / video_width, height / video_height)
436
+ video_height, video_width = int(video_height * scale), int(video_width * scale)
437
+
438
+ if video_height % base != 0 or video_width % base != 0:
439
+ logger.warning(
440
+ f"Video height and width should be divisible by {base}, but got {video_height} and {video_width}. "
441
+ )
442
+ video_height = (video_height // base) * base
443
+ video_width = (video_width // base) * base
444
+
445
+ assert video_height * video_width <= height * width
446
+
447
+ video = self.video_processor.preprocess_video(video, video_height, video_width)
448
+ image_size = (video_height, video_width) # Use the height/width of video (with possible rescaling)
449
+ else:
450
+ video = torch.zeros(batch_size, 3, num_frames, height, width, dtype=dtype, device=device)
451
+ image_size = (height, width) # Use the height/width provider by user
452
+
453
+ if mask is not None:
454
+ mask = self.video_processor.preprocess_video(mask, image_size[0], image_size[1])
455
+ mask = torch.clamp((mask + 1) / 2, min=0, max=1)
456
+ else:
457
+ mask = torch.ones_like(video)
458
+
459
+ video = video.to(dtype=dtype, device=device)
460
+ mask = mask.to(dtype=dtype, device=device)
461
+
462
+ # Make a list of list of images where the outer list corresponds to video batch size and the inner list
463
+ # corresponds to list of conditioning images per video
464
+ if reference_images is None or isinstance(reference_images, PIL.Image.Image):
465
+ reference_images = [[reference_images] for _ in range(video.shape[0])]
466
+ elif isinstance(reference_images, (list, tuple)) and isinstance(next(iter(reference_images)), PIL.Image.Image):
467
+ reference_images = [reference_images]
468
+ elif (
469
+ isinstance(reference_images, (list, tuple))
470
+ and isinstance(next(iter(reference_images)), list)
471
+ and isinstance(next(iter(reference_images[0])), PIL.Image.Image)
472
+ ):
473
+ reference_images = reference_images
474
+ else:
475
+ raise ValueError(
476
+ "`reference_images` has to be of type `PIL.Image.Image` or `list` of `PIL.Image.Image`, or "
477
+ "`list` of `list` of `PIL.Image.Image`, but is {type(reference_images)}"
478
+ )
479
+
480
+ if video.shape[0] != len(reference_images):
481
+ raise ValueError(
482
+ f"Batch size of `video` {video.shape[0]} and length of `reference_images` {len(reference_images)} does not match."
483
+ )
484
+
485
+ ref_images_lengths = [len(reference_images_batch) for reference_images_batch in reference_images]
486
+ if any(l != ref_images_lengths[0] for l in ref_images_lengths):
487
+ raise ValueError(
488
+ f"All batches of `reference_images` should have the same length, but got {ref_images_lengths}. Support for this "
489
+ "may be added in the future."
490
+ )
491
+
492
+ reference_images_preprocessed = []
493
+ for i, reference_images_batch in enumerate(reference_images):
494
+ preprocessed_images = []
495
+ for j, image in enumerate(reference_images_batch):
496
+ if image is None:
497
+ continue
498
+ image = self.video_processor.preprocess(image, None, None)
499
+ img_height, img_width = image.shape[-2:]
500
+ scale = min(image_size[0] / img_height, image_size[1] / img_width)
501
+ new_height, new_width = int(img_height * scale), int(img_width * scale)
502
+ resized_image = torch.nn.functional.interpolate(
503
+ image, size=(new_height, new_width), mode="bilinear", align_corners=False
504
+ ).squeeze(0) # [C, H, W]
505
+ top = (image_size[0] - new_height) // 2
506
+ left = (image_size[1] - new_width) // 2
507
+ canvas = torch.ones(3, *image_size, device=device, dtype=dtype)
508
+ canvas[:, top : top + new_height, left : left + new_width] = resized_image
509
+ preprocessed_images.append(canvas)
510
+ reference_images_preprocessed.append(preprocessed_images)
511
+
512
+ return video, mask, reference_images_preprocessed
513
+
514
+ def prepare_video_latents(
515
+ self,
516
+ video: torch.Tensor,
517
+ mask: torch.Tensor,
518
+ reference_images: list[list[torch.Tensor]] | None = None,
519
+ generator: torch.Generator | list[torch.Generator] | None = None,
520
+ device: torch.device | None = None,
521
+ ) -> torch.Tensor:
522
+ device = device or self._execution_device
523
+
524
+ if isinstance(generator, list):
525
+ # TODO: support this
526
+ raise ValueError("Passing a list of generators is not yet supported. This may be supported in the future.")
527
+
528
+ if reference_images is None:
529
+ # For each batch of video, we set no re
530
+ # ference image (as one or more can be passed by user)
531
+ reference_images = [[None] for _ in range(video.shape[0])]
532
+ else:
533
+ if video.shape[0] != len(reference_images):
534
+ raise ValueError(
535
+ f"Batch size of `video` {video.shape[0]} and length of `reference_images` {len(reference_images)} does not match."
536
+ )
537
+
538
+ if video.shape[0] != 1:
539
+ # TODO: support this
540
+ raise ValueError(
541
+ "Generating with more than one video is not yet supported. This may be supported in the future."
542
+ )
543
+
544
+ vae_dtype = self.vae.dtype
545
+ video = video.to(dtype=vae_dtype)
546
+
547
+ latents_mean = torch.tensor(self.vae.config.latents_mean, device=device, dtype=torch.float32).view(
548
+ 1, self.vae.config.z_dim, 1, 1, 1
549
+ )
550
+ latents_std = 1.0 / torch.tensor(self.vae.config.latents_std, device=device, dtype=torch.float32).view(
551
+ 1, self.vae.config.z_dim, 1, 1, 1
552
+ )
553
+
554
+ if mask is None:
555
+ latents = retrieve_latents(self.vae.encode(video), generator, sample_mode="argmax").unbind(0)
556
+ latents = ((latents.float() - latents_mean) * latents_std).to(vae_dtype)
557
+ else:
558
+ mask = torch.where(mask > 0.5, 1.0, 0.0).to(dtype=vae_dtype)
559
+ inactive = video * (1 - mask)
560
+ reactive = video * mask
561
+ inactive = retrieve_latents(self.vae.encode(inactive), generator, sample_mode="argmax")
562
+ reactive = retrieve_latents(self.vae.encode(reactive), generator, sample_mode="argmax")
563
+ inactive = ((inactive.float() - latents_mean) * latents_std).to(vae_dtype)
564
+ reactive = ((reactive.float() - latents_mean) * latents_std).to(vae_dtype)
565
+ latents = torch.cat([inactive, reactive], dim=1)
566
+
567
+ latent_list = []
568
+ for latent, reference_images_batch in zip(latents, reference_images):
569
+ for reference_image in reference_images_batch:
570
+ assert reference_image.ndim == 3
571
+ reference_image = reference_image.to(dtype=vae_dtype)
572
+ reference_image = reference_image[None, :, None, :, :] # [1, C, 1, H, W]
573
+ reference_latent = retrieve_latents(self.vae.encode(reference_image), generator, sample_mode="argmax")
574
+ reference_latent = ((reference_latent.float() - latents_mean) * latents_std).to(vae_dtype)
575
+ reference_latent = reference_latent.squeeze(0) # [C, 1, H, W]
576
+ reference_latent = torch.cat([reference_latent, torch.zeros_like(reference_latent)], dim=0)
577
+ latent = torch.cat([reference_latent.squeeze(0), latent], dim=1)
578
+ latent_list.append(latent)
579
+ return torch.stack(latent_list)
580
+
581
+ def prepare_masks(
582
+ self,
583
+ mask: torch.Tensor,
584
+ reference_images: list[torch.Tensor] | None = None,
585
+ generator: torch.Generator | list[torch.Generator] | None = None,
586
+ ) -> torch.Tensor:
587
+ if isinstance(generator, list):
588
+ # TODO: support this
589
+ raise ValueError("Passing a list of generators is not yet supported. This may be supported in the future.")
590
+
591
+ if reference_images is None:
592
+ # For each batch of video, we set no reference image (as one or more can be passed by user)
593
+ reference_images = [[None] for _ in range(mask.shape[0])]
594
+ else:
595
+ if mask.shape[0] != len(reference_images):
596
+ raise ValueError(
597
+ f"Batch size of `mask` {mask.shape[0]} and length of `reference_images` {len(reference_images)} does not match."
598
+ )
599
+
600
+ if mask.shape[0] != 1:
601
+ # TODO: support this
602
+ raise ValueError(
603
+ "Generating with more than one video is not yet supported. This may be supported in the future."
604
+ )
605
+
606
+ transformer_patch_size = (
607
+ self.transformer.config.patch_size[1]
608
+ if self.transformer is not None
609
+ else self.transformer_2.config.patch_size[1]
610
+ )
611
+
612
+ mask_list = []
613
+ for mask_, reference_images_batch in zip(mask, reference_images):
614
+ num_channels, num_frames, height, width = mask_.shape
615
+ new_num_frames = (num_frames + self.vae_scale_factor_temporal - 1) // self.vae_scale_factor_temporal
616
+ new_height = height // (self.vae_scale_factor_spatial * transformer_patch_size) * transformer_patch_size
617
+ new_width = width // (self.vae_scale_factor_spatial * transformer_patch_size) * transformer_patch_size
618
+ mask_ = mask_[0, :, :, :]
619
+ mask_ = mask_.view(
620
+ num_frames, new_height, self.vae_scale_factor_spatial, new_width, self.vae_scale_factor_spatial
621
+ )
622
+ mask_ = mask_.permute(2, 4, 0, 1, 3).flatten(0, 1) # [8x8, num_frames, new_height, new_width]
623
+ mask_ = torch.nn.functional.interpolate(
624
+ mask_.unsqueeze(0), size=(new_num_frames, new_height, new_width), mode="nearest-exact"
625
+ ).squeeze(0)
626
+ num_ref_images = len(reference_images_batch)
627
+ if num_ref_images > 0:
628
+ mask_padding = torch.zeros_like(mask_[:, :num_ref_images, :, :])
629
+ mask_ = torch.cat([mask_padding, mask_], dim=1)
630
+ mask_list.append(mask_)
631
+ return torch.stack(mask_list)
632
+
633
+ def prepare_latents(
634
+ self,
635
+ batch_size: int,
636
+ num_channels_latents: int = 16,
637
+ height: int = 480,
638
+ width: int = 832,
639
+ num_frames: int = 81,
640
+ dtype: torch.dtype | None = None,
641
+ device: torch.device | None = None,
642
+ generator: torch.Generator | list[torch.Generator] | None = None,
643
+ latents: torch.Tensor | None = None,
644
+ ) -> torch.Tensor:
645
+ if latents is not None:
646
+ return latents.to(device=device, dtype=dtype)
647
+
648
+ num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1
649
+ shape = (
650
+ batch_size,
651
+ num_channels_latents,
652
+ num_latent_frames,
653
+ int(height) // self.vae_scale_factor_spatial,
654
+ int(width) // self.vae_scale_factor_spatial,
655
+ )
656
+ if isinstance(generator, list) and len(generator) != batch_size:
657
+ raise ValueError(
658
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
659
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
660
+ )
661
+
662
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
663
+ return latents
664
+
665
+ @property
666
+ def guidance_scale(self):
667
+ return self._guidance_scale
668
+
669
+ @property
670
+ def do_classifier_free_guidance(self):
671
+ return self._guidance_scale > 1.0
672
+
673
+ @property
674
+ def num_timesteps(self):
675
+ return self._num_timesteps
676
+
677
+ @property
678
+ def current_timestep(self):
679
+ return self._current_timestep
680
+
681
+ @property
682
+ def interrupt(self):
683
+ return self._interrupt
684
+
685
+ @property
686
+ def attention_kwargs(self):
687
+ return self._attention_kwargs
688
+
689
+ @torch.no_grad()
690
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
691
+ def __call__(
692
+ self,
693
+ prompt: str | list[str] = None,
694
+ negative_prompt: str | list[str] = None,
695
+ video: list[PipelineImageInput] | None = None,
696
+ mask: list[PipelineImageInput] | None = None,
697
+ reference_images: list[PipelineImageInput] | None = None,
698
+ conditioning_scale: float | list[float] | torch.Tensor = 1.0,
699
+ height: int = 480,
700
+ width: int = 832,
701
+ num_frames: int = 81,
702
+ num_inference_steps: int = 50,
703
+ guidance_scale: float = 5.0,
704
+ guidance_scale_2: float | None = None,
705
+ num_videos_per_prompt: int | None = 1,
706
+ generator: torch.Generator | list[torch.Generator] | None = None,
707
+ latents: torch.Tensor | None = None,
708
+ prompt_embeds: torch.Tensor | None = None,
709
+ negative_prompt_embeds: torch.Tensor | None = None,
710
+ output_type: str | None = "np",
711
+ return_dict: bool = True,
712
+ attention_kwargs: dict[str, Any] | None = None,
713
+ callback_on_step_end: Callable[[int, int], None] | PipelineCallback | MultiPipelineCallbacks | None = None,
714
+ callback_on_step_end_tensor_inputs: list[str] = ["latents"],
715
+ max_sequence_length: int = 512,
716
+ ):
717
+ r"""
718
+ The call function to the pipeline for generation.
719
+
720
+ Args:
721
+ prompt (`str` or `list[str]`, *optional*):
722
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`
723
+ instead.
724
+ negative_prompt (`str` or `list[str]`, *optional*):
725
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
726
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
727
+ less than `1`).
728
+ video (`list[PIL.Image.Image]`, *optional*):
729
+ The input video or videos to be used as a starting point for the generation. The video should be a list
730
+ of PIL images, a numpy array, or a torch tensor. Currently, the pipeline only supports generating one
731
+ video at a time.
732
+ mask (`list[PIL.Image.Image]`, *optional*):
733
+ The input mask defines which video regions to condition on and which to generate. Black areas in the
734
+ mask indicate conditioning regions, while white areas indicate regions for generation. The mask should
735
+ be a list of PIL images, a numpy array, or a torch tensor. Currently supports generating a single video
736
+ at a time.
737
+ reference_images (`list[PIL.Image.Image]`, *optional*):
738
+ A list of one or more reference images as extra conditioning for the generation. For example, if you
739
+ are trying to inpaint a video to change the character, you can pass reference images of the new
740
+ character here. Refer to the Diffusers [examples](https://github.com/huggingface/diffusers/pull/11582)
741
+ and original [user
742
+ guide](https://github.com/ali-vilab/VACE/blob/0897c6d055d7d9ea9e191dce763006664d9780f8/UserGuide.md)
743
+ for a full list of supported tasks and use cases.
744
+ conditioning_scale (`float`, `list[float]`, `torch.Tensor`, defaults to `1.0`):
745
+ The conditioning scale to be applied when adding the control conditioning latent stream to the
746
+ denoising latent stream in each control layer of the model. If a float is provided, it will be applied
747
+ uniformly to all layers. If a list or tensor is provided, it should have the same length as the number
748
+ of control layers in the model (`len(transformer.config.vace_layers)`).
749
+ height (`int`, defaults to `480`):
750
+ The height in pixels of the generated image.
751
+ width (`int`, defaults to `832`):
752
+ The width in pixels of the generated image.
753
+ num_frames (`int`, defaults to `81`):
754
+ The number of frames in the generated video.
755
+ num_inference_steps (`int`, defaults to `50`):
756
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
757
+ expense of slower inference.
758
+ guidance_scale (`float`, defaults to `5.0`):
759
+ Guidance scale as defined in [Classifier-Free Diffusion
760
+ Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2.
761
+ of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
762
+ `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to
763
+ the text `prompt`, usually at the expense of lower image quality.
764
+ guidance_scale_2 (`float`, *optional*, defaults to `None`):
765
+ Guidance scale for the low-noise stage transformer (`transformer_2`). If `None` and the pipeline's
766
+ `boundary_ratio` is not None, uses the same value as `guidance_scale`. Only used when `transformer_2`
767
+ and the pipeline's `boundary_ratio` are not None.
768
+ num_videos_per_prompt (`int`, *optional*, defaults to 1):
769
+ The number of images to generate per prompt.
770
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
771
+ A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make
772
+ generation deterministic.
773
+ latents (`torch.Tensor`, *optional*):
774
+ Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image
775
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
776
+ tensor is generated by sampling using the supplied random `generator`.
777
+ prompt_embeds (`torch.Tensor`, *optional*):
778
+ Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not
779
+ provided, text embeddings are generated from the `prompt` input argument.
780
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
781
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If
782
+ not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.
783
+ output_type (`str`, *optional*, defaults to `"np"`):
784
+ The output format of the generated image. Choose between `PIL.Image` or `np.array`.
785
+ return_dict (`bool`, *optional*, defaults to `True`):
786
+ Whether or not to return a [`WanPipelineOutput`] instead of a plain tuple.
787
+ attention_kwargs (`dict`, *optional*):
788
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
789
+ `self.processor` in
790
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
791
+ callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*):
792
+ A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of
793
+ each denoising step during the inference. with the following arguments: `callback_on_step_end(self:
794
+ DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a
795
+ list of all tensors as specified by `callback_on_step_end_tensor_inputs`.
796
+ callback_on_step_end_tensor_inputs (`list`, *optional*):
797
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
798
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
799
+ `._callback_tensor_inputs` attribute of your pipeline class.
800
+ max_sequence_length (`int`, defaults to `512`):
801
+ The maximum sequence length of the text encoder. If the prompt is longer than this, it will be
802
+ truncated. If the prompt is shorter, it will be padded to this length.
803
+
804
+ Examples:
805
+
806
+ Returns:
807
+ [`~WanPipelineOutput`] or `tuple`:
808
+ If `return_dict` is `True`, [`WanPipelineOutput`] is returned, otherwise a `tuple` is returned where
809
+ the first element is a list with the generated images and the second element is a list of `bool`s
810
+ indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content.
811
+ """
812
+
813
+ if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
814
+ callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
815
+
816
+ # Simplification of implementation for now
817
+ if prompt is not None and not isinstance(prompt, str):
818
+ raise ValueError("Passing a list of prompts is not yet supported. This may be supported in the future.")
819
+ if num_videos_per_prompt != 1:
820
+ raise ValueError(
821
+ "Generating multiple videos per prompt is not yet supported. This may be supported in the future."
822
+ )
823
+
824
+ # 1. Check inputs. Raise error if not correct
825
+ self.check_inputs(
826
+ prompt,
827
+ negative_prompt,
828
+ height,
829
+ width,
830
+ prompt_embeds,
831
+ negative_prompt_embeds,
832
+ callback_on_step_end_tensor_inputs,
833
+ video,
834
+ mask,
835
+ reference_images,
836
+ guidance_scale_2,
837
+ )
838
+
839
+ if num_frames % self.vae_scale_factor_temporal != 1:
840
+ logger.warning(
841
+ f"`num_frames - 1` has to be divisible by {self.vae_scale_factor_temporal}. Rounding to the nearest number."
842
+ )
843
+ num_frames = num_frames // self.vae_scale_factor_temporal * self.vae_scale_factor_temporal + 1
844
+ num_frames = max(num_frames, 1)
845
+
846
+ if self.config.boundary_ratio is not None and guidance_scale_2 is None:
847
+ guidance_scale_2 = guidance_scale
848
+
849
+ self._guidance_scale = guidance_scale
850
+ self._guidance_scale_2 = guidance_scale_2
851
+ self._attention_kwargs = attention_kwargs
852
+ self._current_timestep = None
853
+ self._interrupt = False
854
+
855
+ device = self._execution_device
856
+
857
+ # 2. Define call parameters
858
+ if prompt is not None and isinstance(prompt, str):
859
+ batch_size = 1
860
+ elif prompt is not None and isinstance(prompt, list):
861
+ batch_size = len(prompt)
862
+ else:
863
+ batch_size = prompt_embeds.shape[0]
864
+
865
+ vae_dtype = self.vae.dtype
866
+ transformer_dtype = self.transformer.dtype if self.transformer is not None else self.transformer_2.dtype
867
+
868
+ vace_layers = (
869
+ self.transformer.config.vace_layers
870
+ if self.transformer is not None
871
+ else self.transformer_2.config.vace_layers
872
+ )
873
+ if isinstance(conditioning_scale, (int, float)):
874
+ conditioning_scale = [conditioning_scale] * len(vace_layers)
875
+ if isinstance(conditioning_scale, list):
876
+ if len(conditioning_scale) != len(vace_layers):
877
+ raise ValueError(
878
+ f"Length of `conditioning_scale` {len(conditioning_scale)} does not match number of layers {len(vace_layers)}."
879
+ )
880
+ conditioning_scale = torch.tensor(conditioning_scale)
881
+ if isinstance(conditioning_scale, torch.Tensor):
882
+ if conditioning_scale.size(0) != len(vace_layers):
883
+ raise ValueError(
884
+ f"Length of `conditioning_scale` {conditioning_scale.size(0)} does not match number of layers {len(vace_layers)}."
885
+ )
886
+ conditioning_scale = conditioning_scale.to(device=device, dtype=transformer_dtype)
887
+
888
+ # 3. Encode input prompt
889
+ prompt_embeds, negative_prompt_embeds = self.encode_prompt(
890
+ prompt=prompt,
891
+ negative_prompt=negative_prompt,
892
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
893
+ num_videos_per_prompt=num_videos_per_prompt,
894
+ prompt_embeds=prompt_embeds,
895
+ negative_prompt_embeds=negative_prompt_embeds,
896
+ max_sequence_length=max_sequence_length,
897
+ device=device,
898
+ )
899
+
900
+ prompt_embeds = prompt_embeds.to(transformer_dtype)
901
+ if negative_prompt_embeds is not None:
902
+ negative_prompt_embeds = negative_prompt_embeds.to(transformer_dtype)
903
+
904
+ # 4. Prepare timesteps
905
+ self.scheduler.set_timesteps(num_inference_steps, device=device)
906
+ timesteps = self.scheduler.timesteps
907
+
908
+ # 5. Prepare latent variables
909
+ video, mask, reference_images = self.preprocess_conditions(
910
+ video,
911
+ mask,
912
+ reference_images,
913
+ batch_size,
914
+ height,
915
+ width,
916
+ num_frames,
917
+ torch.float32,
918
+ device,
919
+ )
920
+ num_reference_images = len(reference_images[0])
921
+
922
+ conditioning_latents = self.prepare_video_latents(video, mask, reference_images, generator, device)
923
+ mask = self.prepare_masks(mask, reference_images, generator)
924
+ conditioning_latents = torch.cat([conditioning_latents, mask], dim=1)
925
+ conditioning_latents = conditioning_latents.to(transformer_dtype)
926
+
927
+ num_channels_latents = (
928
+ self.transformer.config.in_channels
929
+ if self.transformer is not None
930
+ else self.transformer_2.config.in_channels
931
+ )
932
+ latents = self.prepare_latents(
933
+ batch_size * num_videos_per_prompt,
934
+ num_channels_latents,
935
+ height,
936
+ width,
937
+ num_frames + num_reference_images * self.vae_scale_factor_temporal,
938
+ torch.float32,
939
+ device,
940
+ generator,
941
+ latents,
942
+ )
943
+
944
+ if conditioning_latents.shape[2] != latents.shape[2]:
945
+ logger.warning(
946
+ "The number of frames in the conditioning latents does not match the number of frames to be generated. Generation quality may be affected."
947
+ )
948
+
949
+ # 6. Denoising loop
950
+ num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
951
+ self._num_timesteps = len(timesteps)
952
+
953
+ if self.config.boundary_ratio is not None:
954
+ boundary_timestep = self.config.boundary_ratio * self.scheduler.config.num_train_timesteps
955
+ else:
956
+ boundary_timestep = None
957
+
958
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
959
+ for i, t in enumerate(timesteps):
960
+ if self.interrupt:
961
+ continue
962
+
963
+ self._current_timestep = t
964
+
965
+ if boundary_timestep is None or t >= boundary_timestep:
966
+ # wan2.1 or high-noise stage in wan2.2
967
+ current_model = self.transformer
968
+ current_guidance_scale = guidance_scale
969
+ else:
970
+ # low-noise stage in wan2.2
971
+ current_model = self.transformer_2
972
+ current_guidance_scale = guidance_scale_2
973
+
974
+ latent_model_input = latents.to(transformer_dtype)
975
+ timestep = t.expand(latents.shape[0])
976
+
977
+ with current_model.cache_context("cond"):
978
+ noise_pred = current_model(
979
+ hidden_states=latent_model_input,
980
+ timestep=timestep,
981
+ encoder_hidden_states=prompt_embeds,
982
+ control_hidden_states=conditioning_latents,
983
+ control_hidden_states_scale=conditioning_scale,
984
+ attention_kwargs=attention_kwargs,
985
+ return_dict=False,
986
+ )[0]
987
+
988
+ if self.do_classifier_free_guidance:
989
+ with current_model.cache_context("uncond"):
990
+ noise_uncond = current_model(
991
+ hidden_states=latent_model_input,
992
+ timestep=timestep,
993
+ encoder_hidden_states=negative_prompt_embeds,
994
+ control_hidden_states=conditioning_latents,
995
+ control_hidden_states_scale=conditioning_scale,
996
+ attention_kwargs=attention_kwargs,
997
+ return_dict=False,
998
+ )[0]
999
+ noise_pred = noise_uncond + current_guidance_scale * (noise_pred - noise_uncond)
1000
+
1001
+ # compute the previous noisy sample x_t -> x_t-1
1002
+ latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
1003
+
1004
+ if callback_on_step_end is not None:
1005
+ callback_kwargs = {}
1006
+ for k in callback_on_step_end_tensor_inputs:
1007
+ callback_kwargs[k] = locals()[k]
1008
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
1009
+
1010
+ latents = callback_outputs.pop("latents", latents)
1011
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
1012
+ negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
1013
+
1014
+ # call the callback, if provided
1015
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
1016
+ progress_bar.update()
1017
+
1018
+ if XLA_AVAILABLE:
1019
+ xm.mark_step()
1020
+
1021
+ self._current_timestep = None
1022
+
1023
+ if not output_type == "latent":
1024
+ latents = latents[:, :, num_reference_images:]
1025
+ latents = latents.to(vae_dtype)
1026
+ latents_mean = (
1027
+ torch.tensor(self.vae.config.latents_mean)
1028
+ .view(1, self.vae.config.z_dim, 1, 1, 1)
1029
+ .to(latents.device, latents.dtype)
1030
+ )
1031
+ latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
1032
+ latents.device, latents.dtype
1033
+ )
1034
+ latents = latents / latents_std + latents_mean
1035
+ video = self.vae.decode(latents, return_dict=False)[0]
1036
+ video = self.video_processor.postprocess_video(video, output_type=output_type)
1037
+ else:
1038
+ video = latents
1039
+
1040
+ # Offload all models
1041
+ self.maybe_free_model_hooks()
1042
+
1043
+ if not return_dict:
1044
+ return (video,)
1045
+
1046
+ return WanPipelineOutput(frames=video)
western_ai/europe/diffusers/src/diffusers/pipelines/wan/pipeline_wan_video2video.py ADDED
@@ -0,0 +1,741 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 The Wan Team and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import html
16
+ import inspect
17
+ from typing import Any, Callable
18
+
19
+ import regex as re
20
+ import torch
21
+ from PIL import Image
22
+ from transformers import AutoTokenizer, UMT5EncoderModel
23
+
24
+ from ...callbacks import MultiPipelineCallbacks, PipelineCallback
25
+ from ...loaders import WanLoraLoaderMixin
26
+ from ...models import AutoencoderKLWan, WanTransformer3DModel
27
+ from ...schedulers import FlowMatchEulerDiscreteScheduler
28
+ from ...utils import is_ftfy_available, is_torch_xla_available, logging, replace_example_docstring
29
+ from ...utils.torch_utils import randn_tensor
30
+ from ...video_processor import VideoProcessor
31
+ from ..pipeline_utils import DiffusionPipeline
32
+ from .pipeline_output import WanPipelineOutput
33
+
34
+
35
+ if is_torch_xla_available():
36
+ import torch_xla.core.xla_model as xm
37
+
38
+ XLA_AVAILABLE = True
39
+ else:
40
+ XLA_AVAILABLE = False
41
+
42
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
43
+
44
+ if is_ftfy_available():
45
+ import ftfy
46
+
47
+
48
+ EXAMPLE_DOC_STRING = """
49
+ Examples:
50
+ ```python
51
+ >>> import torch
52
+ >>> from diffusers.utils import export_to_video, load_video
53
+ >>> from diffusers import AutoencoderKLWan, WanVideoToVideoPipeline
54
+ >>> from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
55
+
56
+ >>> # Available models: Wan-AI/Wan2.1-T2V-14B-Diffusers, Wan-AI/Wan2.1-T2V-1.3B-Diffusers
57
+ >>> model_id = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers"
58
+ >>> vae = AutoencoderKLWan.from_pretrained(model_id, subfolder="vae", torch_dtype=torch.float32)
59
+ >>> pipe = WanVideoToVideoPipeline.from_pretrained(model_id, vae=vae, torch_dtype=torch.bfloat16)
60
+ >>> flow_shift = 3.0 # 5.0 for 720P, 3.0 for 480P
61
+ >>> pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config, flow_shift=flow_shift)
62
+ >>> pipe.to("cuda")
63
+
64
+ >>> prompt = "A robot standing on a mountain top. The sun is setting in the background"
65
+ >>> negative_prompt = "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards"
66
+ >>> video = load_video(
67
+ ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/hiker.mp4"
68
+ ... )
69
+ >>> output = pipe(
70
+ ... video=video,
71
+ ... prompt=prompt,
72
+ ... negative_prompt=negative_prompt,
73
+ ... height=480,
74
+ ... width=720,
75
+ ... guidance_scale=5.0,
76
+ ... strength=0.7,
77
+ ... ).frames[0]
78
+ >>> export_to_video(output, "output.mp4", fps=16)
79
+ ```
80
+ """
81
+
82
+
83
+ def basic_clean(text):
84
+ text = ftfy.fix_text(text)
85
+ text = html.unescape(html.unescape(text))
86
+ return text.strip()
87
+
88
+
89
+ def whitespace_clean(text):
90
+ text = re.sub(r"\s+", " ", text)
91
+ text = text.strip()
92
+ return text
93
+
94
+
95
+ def prompt_clean(text):
96
+ text = whitespace_clean(basic_clean(text))
97
+ return text
98
+
99
+
100
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
101
+ def retrieve_timesteps(
102
+ scheduler,
103
+ num_inference_steps: int | None = None,
104
+ device: str | torch.device | None = None,
105
+ timesteps: list[int] | None = None,
106
+ sigmas: list[float] | None = None,
107
+ **kwargs,
108
+ ):
109
+ r"""
110
+ Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
111
+ custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
112
+
113
+ Args:
114
+ scheduler (`SchedulerMixin`):
115
+ The scheduler to get timesteps from.
116
+ num_inference_steps (`int`):
117
+ The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
118
+ must be `None`.
119
+ device (`str` or `torch.device`, *optional*):
120
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
121
+ timesteps (`list[int]`, *optional*):
122
+ Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
123
+ `num_inference_steps` and `sigmas` must be `None`.
124
+ sigmas (`list[float]`, *optional*):
125
+ Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
126
+ `num_inference_steps` and `timesteps` must be `None`.
127
+
128
+ Returns:
129
+ `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
130
+ second element is the number of inference steps.
131
+ """
132
+ if timesteps is not None and sigmas is not None:
133
+ raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
134
+ if timesteps is not None:
135
+ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
136
+ if not accepts_timesteps:
137
+ raise ValueError(
138
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
139
+ f" timestep schedules. Please check whether you are using the correct scheduler."
140
+ )
141
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
142
+ timesteps = scheduler.timesteps
143
+ num_inference_steps = len(timesteps)
144
+ elif sigmas is not None:
145
+ accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
146
+ if not accept_sigmas:
147
+ raise ValueError(
148
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
149
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
150
+ )
151
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
152
+ timesteps = scheduler.timesteps
153
+ num_inference_steps = len(timesteps)
154
+ else:
155
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
156
+ timesteps = scheduler.timesteps
157
+ return timesteps, num_inference_steps
158
+
159
+
160
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
161
+ def retrieve_latents(
162
+ encoder_output: torch.Tensor, generator: torch.Generator | None = None, sample_mode: str = "sample"
163
+ ):
164
+ if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
165
+ return encoder_output.latent_dist.sample(generator)
166
+ elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
167
+ return encoder_output.latent_dist.mode()
168
+ elif hasattr(encoder_output, "latents"):
169
+ return encoder_output.latents
170
+ else:
171
+ raise AttributeError("Could not access latents of provided encoder_output")
172
+
173
+
174
+ class WanVideoToVideoPipeline(DiffusionPipeline, WanLoraLoaderMixin):
175
+ r"""
176
+ Pipeline for video-to-video generation using Wan.
177
+
178
+ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods
179
+ implemented for all pipelines (downloading, saving, running on a particular device, etc.).
180
+
181
+ Args:
182
+ tokenizer ([`T5Tokenizer`]):
183
+ Tokenizer from [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5Tokenizer),
184
+ specifically the [google/umt5-xxl](https://huggingface.co/google/umt5-xxl) variant.
185
+ text_encoder ([`T5EncoderModel`]):
186
+ [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5EncoderModel), specifically
187
+ the [google/umt5-xxl](https://huggingface.co/google/umt5-xxl) variant.
188
+ transformer ([`WanTransformer3DModel`]):
189
+ Conditional Transformer to denoise the input latents.
190
+ scheduler ([`UniPCMultistepScheduler`]):
191
+ A scheduler to be used in combination with `transformer` to denoise the encoded image latents.
192
+ vae ([`AutoencoderKLWan`]):
193
+ Variational Auto-Encoder (VAE) Model to encode and decode videos to and from latent representations.
194
+ """
195
+
196
+ model_cpu_offload_seq = "text_encoder->transformer->vae"
197
+ _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"]
198
+
199
+ def __init__(
200
+ self,
201
+ tokenizer: AutoTokenizer,
202
+ text_encoder: UMT5EncoderModel,
203
+ transformer: WanTransformer3DModel,
204
+ vae: AutoencoderKLWan,
205
+ scheduler: FlowMatchEulerDiscreteScheduler,
206
+ ):
207
+ super().__init__()
208
+
209
+ self.register_modules(
210
+ vae=vae,
211
+ text_encoder=text_encoder,
212
+ tokenizer=tokenizer,
213
+ transformer=transformer,
214
+ scheduler=scheduler,
215
+ )
216
+
217
+ self.vae_scale_factor_temporal = 2 ** sum(self.vae.temperal_downsample) if getattr(self, "vae", None) else 4
218
+ self.vae_scale_factor_spatial = 2 ** len(self.vae.temperal_downsample) if getattr(self, "vae", None) else 8
219
+ self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial)
220
+
221
+ # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline._get_t5_prompt_embeds
222
+ def _get_t5_prompt_embeds(
223
+ self,
224
+ prompt: str | list[str] = None,
225
+ num_videos_per_prompt: int = 1,
226
+ max_sequence_length: int = 226,
227
+ device: torch.device | None = None,
228
+ dtype: torch.dtype | None = None,
229
+ ):
230
+ device = device or self._execution_device
231
+ dtype = dtype or self.text_encoder.dtype
232
+
233
+ prompt = [prompt] if isinstance(prompt, str) else prompt
234
+ prompt = [prompt_clean(u) for u in prompt]
235
+ batch_size = len(prompt)
236
+
237
+ text_inputs = self.tokenizer(
238
+ prompt,
239
+ padding="max_length",
240
+ max_length=max_sequence_length,
241
+ truncation=True,
242
+ add_special_tokens=True,
243
+ return_attention_mask=True,
244
+ return_tensors="pt",
245
+ )
246
+ text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask
247
+ seq_lens = mask.gt(0).sum(dim=1).long()
248
+
249
+ prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state
250
+ prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
251
+ prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)]
252
+ prompt_embeds = torch.stack(
253
+ [torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0
254
+ )
255
+
256
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
257
+ _, seq_len, _ = prompt_embeds.shape
258
+ prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1)
259
+ prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
260
+
261
+ return prompt_embeds
262
+
263
+ # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline.encode_prompt
264
+ def encode_prompt(
265
+ self,
266
+ prompt: str | list[str],
267
+ negative_prompt: str | list[str] | None = None,
268
+ do_classifier_free_guidance: bool = True,
269
+ num_videos_per_prompt: int = 1,
270
+ prompt_embeds: torch.Tensor | None = None,
271
+ negative_prompt_embeds: torch.Tensor | None = None,
272
+ max_sequence_length: int = 226,
273
+ device: torch.device | None = None,
274
+ dtype: torch.dtype | None = None,
275
+ ):
276
+ r"""
277
+ Encodes the prompt into text encoder hidden states.
278
+
279
+ Args:
280
+ prompt (`str` or `list[str]`, *optional*):
281
+ prompt to be encoded
282
+ negative_prompt (`str` or `list[str]`, *optional*):
283
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
284
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
285
+ less than `1`).
286
+ do_classifier_free_guidance (`bool`, *optional*, defaults to `True`):
287
+ Whether to use classifier free guidance or not.
288
+ num_videos_per_prompt (`int`, *optional*, defaults to 1):
289
+ Number of videos that should be generated per prompt. torch device to place the resulting embeddings on
290
+ prompt_embeds (`torch.Tensor`, *optional*):
291
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
292
+ provided, text embeddings will be generated from `prompt` input argument.
293
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
294
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
295
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
296
+ argument.
297
+ device: (`torch.device`, *optional*):
298
+ torch device
299
+ dtype: (`torch.dtype`, *optional*):
300
+ torch dtype
301
+ """
302
+ device = device or self._execution_device
303
+
304
+ prompt = [prompt] if isinstance(prompt, str) else prompt
305
+ if prompt is not None:
306
+ batch_size = len(prompt)
307
+ else:
308
+ batch_size = prompt_embeds.shape[0]
309
+
310
+ if prompt_embeds is None:
311
+ prompt_embeds = self._get_t5_prompt_embeds(
312
+ prompt=prompt,
313
+ num_videos_per_prompt=num_videos_per_prompt,
314
+ max_sequence_length=max_sequence_length,
315
+ device=device,
316
+ dtype=dtype,
317
+ )
318
+
319
+ if do_classifier_free_guidance and negative_prompt_embeds is None:
320
+ negative_prompt = negative_prompt or ""
321
+ negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
322
+
323
+ if prompt is not None and type(prompt) is not type(negative_prompt):
324
+ raise TypeError(
325
+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
326
+ f" {type(prompt)}."
327
+ )
328
+ elif batch_size != len(negative_prompt):
329
+ raise ValueError(
330
+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
331
+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
332
+ " the batch size of `prompt`."
333
+ )
334
+
335
+ negative_prompt_embeds = self._get_t5_prompt_embeds(
336
+ prompt=negative_prompt,
337
+ num_videos_per_prompt=num_videos_per_prompt,
338
+ max_sequence_length=max_sequence_length,
339
+ device=device,
340
+ dtype=dtype,
341
+ )
342
+
343
+ return prompt_embeds, negative_prompt_embeds
344
+
345
+ def check_inputs(
346
+ self,
347
+ prompt,
348
+ negative_prompt,
349
+ height,
350
+ width,
351
+ video=None,
352
+ latents=None,
353
+ prompt_embeds=None,
354
+ negative_prompt_embeds=None,
355
+ callback_on_step_end_tensor_inputs=None,
356
+ ):
357
+ if height % 16 != 0 or width % 16 != 0:
358
+ raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.")
359
+
360
+ if callback_on_step_end_tensor_inputs is not None and not all(
361
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
362
+ ):
363
+ raise ValueError(
364
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
365
+ )
366
+
367
+ if prompt is not None and prompt_embeds is not None:
368
+ raise ValueError(
369
+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
370
+ " only forward one of the two."
371
+ )
372
+ elif negative_prompt is not None and negative_prompt_embeds is not None:
373
+ raise ValueError(
374
+ f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`: {negative_prompt_embeds}. Please make sure to"
375
+ " only forward one of the two."
376
+ )
377
+ elif prompt is None and prompt_embeds is None:
378
+ raise ValueError(
379
+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
380
+ )
381
+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
382
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
383
+ elif negative_prompt is not None and (
384
+ not isinstance(negative_prompt, str) and not isinstance(negative_prompt, list)
385
+ ):
386
+ raise ValueError(f"`negative_prompt` has to be of type `str` or `list` but is {type(negative_prompt)}")
387
+
388
+ if video is not None and latents is not None:
389
+ raise ValueError("Only one of `video` or `latents` should be provided")
390
+
391
+ def prepare_latents(
392
+ self,
393
+ video: torch.Tensor | None = None,
394
+ batch_size: int = 1,
395
+ num_channels_latents: int = 16,
396
+ height: int = 480,
397
+ width: int = 832,
398
+ dtype: torch.dtype | None = None,
399
+ device: torch.device | None = None,
400
+ generator: torch.Generator | None = None,
401
+ latents: torch.Tensor | None = None,
402
+ timestep: torch.Tensor | None = None,
403
+ ):
404
+ if isinstance(generator, list) and len(generator) != batch_size:
405
+ raise ValueError(
406
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
407
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
408
+ )
409
+
410
+ num_latent_frames = (
411
+ (video.size(2) - 1) // self.vae_scale_factor_temporal + 1 if latents is None else latents.size(1)
412
+ )
413
+ shape = (
414
+ batch_size,
415
+ num_channels_latents,
416
+ num_latent_frames,
417
+ height // self.vae_scale_factor_spatial,
418
+ width // self.vae_scale_factor_spatial,
419
+ )
420
+
421
+ if latents is None:
422
+ init_latents = [retrieve_latents(self.vae.encode(vid.unsqueeze(0)), sample_mode="argmax") for vid in video]
423
+
424
+ init_latents = torch.cat(init_latents, dim=0).to(dtype)
425
+
426
+ latents_mean = (
427
+ torch.tensor(self.vae.config.latents_mean).view(1, self.vae.config.z_dim, 1, 1, 1).to(device, dtype)
428
+ )
429
+ latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
430
+ device, dtype
431
+ )
432
+
433
+ init_latents = (init_latents - latents_mean) * latents_std
434
+
435
+ noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
436
+ if hasattr(self.scheduler, "add_noise"):
437
+ latents = self.scheduler.add_noise(init_latents, noise, timestep)
438
+ else:
439
+ latents = self.scheduler.scale_noise(init_latents, timestep, noise)
440
+ else:
441
+ latents = latents.to(device)
442
+
443
+ return latents
444
+
445
+ # Copied from diffusers.pipelines.animatediff.pipeline_animatediff_video2video.AnimateDiffVideoToVideoPipeline.get_timesteps
446
+ def get_timesteps(self, num_inference_steps, timesteps, strength, device):
447
+ # get the original timestep using init_timestep
448
+ init_timestep = min(int(num_inference_steps * strength), num_inference_steps)
449
+
450
+ t_start = max(num_inference_steps - init_timestep, 0)
451
+ timesteps = timesteps[t_start * self.scheduler.order :]
452
+
453
+ return timesteps, num_inference_steps - t_start
454
+
455
+ @property
456
+ def guidance_scale(self):
457
+ return self._guidance_scale
458
+
459
+ @property
460
+ def do_classifier_free_guidance(self):
461
+ return self._guidance_scale > 1.0
462
+
463
+ @property
464
+ def num_timesteps(self):
465
+ return self._num_timesteps
466
+
467
+ @property
468
+ def current_timestep(self):
469
+ return self._current_timestep
470
+
471
+ @property
472
+ def interrupt(self):
473
+ return self._interrupt
474
+
475
+ @property
476
+ def attention_kwargs(self):
477
+ return self._attention_kwargs
478
+
479
+ @torch.no_grad()
480
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
481
+ def __call__(
482
+ self,
483
+ video: list[Image.Image] = None,
484
+ prompt: str | list[str] = None,
485
+ negative_prompt: str | list[str] = None,
486
+ height: int = 480,
487
+ width: int = 832,
488
+ num_inference_steps: int = 50,
489
+ timesteps: list[int] | None = None,
490
+ guidance_scale: float = 5.0,
491
+ strength: float = 0.8,
492
+ num_videos_per_prompt: int | None = 1,
493
+ generator: torch.Generator | list[torch.Generator] | None = None,
494
+ latents: torch.Tensor | None = None,
495
+ prompt_embeds: torch.Tensor | None = None,
496
+ negative_prompt_embeds: torch.Tensor | None = None,
497
+ output_type: str | None = "np",
498
+ return_dict: bool = True,
499
+ attention_kwargs: dict[str, Any] | None = None,
500
+ callback_on_step_end: Callable[[int, int], None] | PipelineCallback | MultiPipelineCallbacks | None = None,
501
+ callback_on_step_end_tensor_inputs: list[str] = ["latents"],
502
+ max_sequence_length: int = 512,
503
+ ):
504
+ r"""
505
+ The call function to the pipeline for generation.
506
+
507
+ Args:
508
+ video (`list[PIL.Image.Image]`):
509
+ The input video used as the starting point for video-to-video generation. The video should be provided
510
+ as a list of PIL images, a numpy array, or a torch tensor.
511
+ prompt (`str` or `list[str]`, *optional*):
512
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`
513
+ instead.
514
+ negative_prompt (`str` or `list[str]`, *optional*):
515
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
516
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
517
+ less than `1`).
518
+ height (`int`, defaults to `480`):
519
+ The height in pixels of the generated image.
520
+ width (`int`, defaults to `832`):
521
+ The width in pixels of the generated image.
522
+ num_inference_steps (`int`, defaults to `50`):
523
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
524
+ expense of slower inference.
525
+ timesteps (`list[int]`, *optional*):
526
+ Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument
527
+ in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is
528
+ passed will be used. Must be in descending order.
529
+ guidance_scale (`float`, defaults to `5.0`):
530
+ Guidance scale as defined in [Classifier-Free Diffusion
531
+ Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2.
532
+ of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
533
+ `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to
534
+ the text `prompt`, usually at the expense of lower image quality.
535
+ strength (`float`, defaults to `0.8`):
536
+ Higher strength leads to more differences between original image and generated video.
537
+ num_videos_per_prompt (`int`, *optional*, defaults to 1):
538
+ The number of images to generate per prompt.
539
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
540
+ A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make
541
+ generation deterministic.
542
+ latents (`torch.Tensor`, *optional*):
543
+ Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image
544
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
545
+ tensor is generated by sampling using the supplied random `generator`.
546
+ prompt_embeds (`torch.Tensor`, *optional*):
547
+ Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not
548
+ provided, text embeddings are generated from the `prompt` input argument.
549
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
550
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If
551
+ not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.
552
+ output_type (`str`, *optional*, defaults to `"np"`):
553
+ The output format of the generated image. Choose between `PIL.Image` or `np.array`.
554
+ return_dict (`bool`, *optional*, defaults to `True`):
555
+ Whether or not to return a [`WanPipelineOutput`] instead of a plain tuple.
556
+ attention_kwargs (`dict`, *optional*):
557
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
558
+ `self.processor` in
559
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
560
+ callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*):
561
+ A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of
562
+ each denoising step during the inference. with the following arguments: `callback_on_step_end(self:
563
+ DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a
564
+ list of all tensors as specified by `callback_on_step_end_tensor_inputs`.
565
+ callback_on_step_end_tensor_inputs (`list`, *optional*):
566
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
567
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
568
+ `._callback_tensor_inputs` attribute of your pipeline class.
569
+ max_sequence_length (`int`, defaults to `512`):
570
+ The maximum sequence length of the text encoder. If the prompt is longer than this, it will be
571
+ truncated. If the prompt is shorter, it will be padded to this length.
572
+
573
+ Examples:
574
+
575
+ Returns:
576
+ [`~WanPipelineOutput`] or `tuple`:
577
+ If `return_dict` is `True`, [`WanPipelineOutput`] is returned, otherwise a `tuple` is returned where
578
+ the first element is a list with the generated images and the second element is a list of `bool`s
579
+ indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content.
580
+ """
581
+
582
+ if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
583
+ callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
584
+
585
+ height = height or self.transformer.config.sample_height * self.vae_scale_factor_spatial
586
+ width = width or self.transformer.config.sample_width * self.vae_scale_factor_spatial
587
+ num_videos_per_prompt = 1
588
+
589
+ # 1. Check inputs. Raise error if not correct
590
+ self.check_inputs(
591
+ prompt,
592
+ negative_prompt,
593
+ height,
594
+ width,
595
+ video,
596
+ latents,
597
+ prompt_embeds,
598
+ negative_prompt_embeds,
599
+ callback_on_step_end_tensor_inputs,
600
+ )
601
+
602
+ self._guidance_scale = guidance_scale
603
+ self._attention_kwargs = attention_kwargs
604
+ self._current_timestep = None
605
+ self._interrupt = False
606
+
607
+ device = self._execution_device
608
+
609
+ # 2. Define call parameters
610
+ if prompt is not None and isinstance(prompt, str):
611
+ batch_size = 1
612
+ elif prompt is not None and isinstance(prompt, list):
613
+ batch_size = len(prompt)
614
+ else:
615
+ batch_size = prompt_embeds.shape[0]
616
+
617
+ # 3. Encode input prompt
618
+ prompt_embeds, negative_prompt_embeds = self.encode_prompt(
619
+ prompt=prompt,
620
+ negative_prompt=negative_prompt,
621
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
622
+ num_videos_per_prompt=num_videos_per_prompt,
623
+ prompt_embeds=prompt_embeds,
624
+ negative_prompt_embeds=negative_prompt_embeds,
625
+ max_sequence_length=max_sequence_length,
626
+ device=device,
627
+ )
628
+
629
+ transformer_dtype = self.transformer.dtype
630
+ prompt_embeds = prompt_embeds.to(transformer_dtype)
631
+ if negative_prompt_embeds is not None:
632
+ negative_prompt_embeds = negative_prompt_embeds.to(transformer_dtype)
633
+
634
+ # 4. Prepare timesteps
635
+ if XLA_AVAILABLE:
636
+ timestep_device = "cpu"
637
+ else:
638
+ timestep_device = device
639
+ timesteps, num_inference_steps = retrieve_timesteps(
640
+ self.scheduler, num_inference_steps, timestep_device, timesteps
641
+ )
642
+ timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, timesteps, strength, device)
643
+ latent_timestep = timesteps[:1].repeat(batch_size * num_videos_per_prompt)
644
+ self._num_timesteps = len(timesteps)
645
+
646
+ if latents is None:
647
+ video = self.video_processor.preprocess_video(video, height=height, width=width).to(
648
+ device, dtype=torch.float32
649
+ )
650
+
651
+ # 5. Prepare latent variables
652
+ num_channels_latents = self.transformer.config.in_channels
653
+ latents = self.prepare_latents(
654
+ video,
655
+ batch_size * num_videos_per_prompt,
656
+ num_channels_latents,
657
+ height,
658
+ width,
659
+ torch.float32,
660
+ device,
661
+ generator,
662
+ latents,
663
+ latent_timestep,
664
+ )
665
+
666
+ # 6. Denoising loop
667
+ num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
668
+ self._num_timesteps = len(timesteps)
669
+
670
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
671
+ for i, t in enumerate(timesteps):
672
+ if self.interrupt:
673
+ continue
674
+
675
+ self._current_timestep = t
676
+ latent_model_input = latents.to(transformer_dtype)
677
+ timestep = t.expand(latents.shape[0])
678
+
679
+ noise_pred = self.transformer(
680
+ hidden_states=latent_model_input,
681
+ timestep=timestep,
682
+ encoder_hidden_states=prompt_embeds,
683
+ attention_kwargs=attention_kwargs,
684
+ return_dict=False,
685
+ )[0]
686
+
687
+ if self.do_classifier_free_guidance:
688
+ noise_uncond = self.transformer(
689
+ hidden_states=latent_model_input,
690
+ timestep=timestep,
691
+ encoder_hidden_states=negative_prompt_embeds,
692
+ attention_kwargs=attention_kwargs,
693
+ return_dict=False,
694
+ )[0]
695
+ noise_pred = noise_uncond + guidance_scale * (noise_pred - noise_uncond)
696
+
697
+ # compute the previous noisy sample x_t -> x_t-1
698
+ latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
699
+
700
+ if callback_on_step_end is not None:
701
+ callback_kwargs = {}
702
+ for k in callback_on_step_end_tensor_inputs:
703
+ callback_kwargs[k] = locals()[k]
704
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
705
+
706
+ latents = callback_outputs.pop("latents", latents)
707
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
708
+ negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
709
+
710
+ # call the callback, if provided
711
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
712
+ progress_bar.update()
713
+
714
+ if XLA_AVAILABLE:
715
+ xm.mark_step()
716
+
717
+ self._current_timestep = None
718
+
719
+ if not output_type == "latent":
720
+ latents = latents.to(self.vae.dtype)
721
+ latents_mean = (
722
+ torch.tensor(self.vae.config.latents_mean)
723
+ .view(1, self.vae.config.z_dim, 1, 1, 1)
724
+ .to(latents.device, latents.dtype)
725
+ )
726
+ latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
727
+ latents.device, latents.dtype
728
+ )
729
+ latents = latents / latents_std + latents_mean
730
+ video = self.vae.decode(latents, return_dict=False)[0]
731
+ video = self.video_processor.postprocess_video(video, output_type=output_type)
732
+ else:
733
+ video = latents
734
+
735
+ # Offload all models
736
+ self.maybe_free_model_hooks()
737
+
738
+ if not return_dict:
739
+ return (video,)
740
+
741
+ return WanPipelineOutput(frames=video)
western_ai/europe/diffusers/src/diffusers/pipelines/z_image/__init__.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import TYPE_CHECKING
2
+
3
+ from ...utils import (
4
+ DIFFUSERS_SLOW_IMPORT,
5
+ OptionalDependencyNotAvailable,
6
+ _LazyModule,
7
+ get_objects_from_module,
8
+ is_torch_available,
9
+ is_transformers_available,
10
+ )
11
+
12
+
13
+ _dummy_objects = {}
14
+ _import_structure = {}
15
+
16
+ try:
17
+ if not (is_transformers_available() and is_torch_available()):
18
+ raise OptionalDependencyNotAvailable()
19
+ except OptionalDependencyNotAvailable:
20
+ from ...utils import dummy_torch_and_transformers_objects # noqa: F403
21
+
22
+ _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects))
23
+ else:
24
+ _import_structure["pipeline_output"] = ["ZImagePipelineOutput"]
25
+ _import_structure["pipeline_z_image"] = ["ZImagePipeline"]
26
+ _import_structure["pipeline_z_image_controlnet"] = ["ZImageControlNetPipeline"]
27
+ _import_structure["pipeline_z_image_controlnet_inpaint"] = ["ZImageControlNetInpaintPipeline"]
28
+ _import_structure["pipeline_z_image_img2img"] = ["ZImageImg2ImgPipeline"]
29
+ _import_structure["pipeline_z_image_inpaint"] = ["ZImageInpaintPipeline"]
30
+ _import_structure["pipeline_z_image_omni"] = ["ZImageOmniPipeline"]
31
+
32
+
33
+ if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT:
34
+ try:
35
+ if not (is_transformers_available() and is_torch_available()):
36
+ raise OptionalDependencyNotAvailable()
37
+
38
+ except OptionalDependencyNotAvailable:
39
+ from ...utils.dummy_torch_and_transformers_objects import *
40
+ else:
41
+ from .pipeline_output import ZImagePipelineOutput
42
+ from .pipeline_z_image import ZImagePipeline
43
+ from .pipeline_z_image_controlnet import ZImageControlNetPipeline
44
+ from .pipeline_z_image_controlnet_inpaint import ZImageControlNetInpaintPipeline
45
+ from .pipeline_z_image_img2img import ZImageImg2ImgPipeline
46
+ from .pipeline_z_image_inpaint import ZImageInpaintPipeline
47
+ from .pipeline_z_image_omni import ZImageOmniPipeline
48
+ else:
49
+ import sys
50
+
51
+ sys.modules[__name__] = _LazyModule(
52
+ __name__,
53
+ globals()["__file__"],
54
+ _import_structure,
55
+ module_spec=__spec__,
56
+ )
57
+
58
+ for name, value in _dummy_objects.items():
59
+ setattr(sys.modules[__name__], name, value)
western_ai/europe/diffusers/src/diffusers/pipelines/z_image/pipeline_output.py ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 Alibaba Z-Image Team and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from dataclasses import dataclass
16
+
17
+ import numpy as np
18
+ import PIL.Image
19
+
20
+ from diffusers.utils import BaseOutput
21
+
22
+
23
+ @dataclass
24
+ class ZImagePipelineOutput(BaseOutput):
25
+ """
26
+ Output class for Z-Image pipelines.
27
+
28
+ Args:
29
+ images (`list[PIL.Image.Image]` or `np.ndarray`)
30
+ List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width,
31
+ num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline.
32
+ """
33
+
34
+ images: list[PIL.Image.Image, np.ndarray]
western_ai/europe/diffusers/src/diffusers/pipelines/z_image/pipeline_z_image.py ADDED
@@ -0,0 +1,600 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 Alibaba Z-Image Team and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from typing import Any, Callable
17
+
18
+ import torch
19
+ from transformers import AutoTokenizer, PreTrainedModel
20
+
21
+ from ...image_processor import VaeImageProcessor
22
+ from ...loaders import FromSingleFileMixin, ZImageLoraLoaderMixin
23
+ from ...models.autoencoders import AutoencoderKL
24
+ from ...models.transformers import ZImageTransformer2DModel
25
+ from ...pipelines.pipeline_utils import DiffusionPipeline
26
+ from ...schedulers import FlowMatchEulerDiscreteScheduler
27
+ from ...utils import logging, replace_example_docstring
28
+ from ...utils.torch_utils import randn_tensor
29
+ from .pipeline_output import ZImagePipelineOutput
30
+
31
+
32
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
33
+
34
+ EXAMPLE_DOC_STRING = """
35
+ Examples:
36
+ ```py
37
+ >>> import torch
38
+ >>> from diffusers import ZImagePipeline
39
+
40
+ >>> pipe = ZImagePipeline.from_pretrained("Z-a-o/Z-Image-Turbo", torch_dtype=torch.bfloat16)
41
+ >>> pipe.to("cuda")
42
+
43
+ >>> # Optionally, set the attention backend to flash-attn 2 or 3, default is SDPA in PyTorch.
44
+ >>> # (1) Use flash attention 2
45
+ >>> # pipe.transformer.set_attention_backend("flash")
46
+ >>> # (2) Use flash attention 3
47
+ >>> # pipe.transformer.set_attention_backend("_flash_3")
48
+
49
+ >>> prompt = "一幅为名为“造相「Z-IMAGE-TURBO」”的项目设计的创意海报。画面巧妙地将文字概念视觉化:一辆复古蒸汽小火车化身为巨大的拉链头,正拉开厚厚的冬日积雪,展露出一个生机盎然的春天。"
50
+ >>> image = pipe(
51
+ ... prompt,
52
+ ... height=1024,
53
+ ... width=1024,
54
+ ... num_inference_steps=8,
55
+ ... guidance_scale=0.0,
56
+ ... generator=torch.Generator("cuda").manual_seed(42),
57
+ ... ).images[0]
58
+ >>> image.save("zimage.png")
59
+ ```
60
+ """
61
+
62
+
63
+ # Copied from diffusers.pipelines.flux.pipeline_flux.calculate_shift
64
+ def calculate_shift(
65
+ image_seq_len,
66
+ base_seq_len: int = 256,
67
+ max_seq_len: int = 4096,
68
+ base_shift: float = 0.5,
69
+ max_shift: float = 1.15,
70
+ ):
71
+ m = (max_shift - base_shift) / (max_seq_len - base_seq_len)
72
+ b = base_shift - m * base_seq_len
73
+ mu = image_seq_len * m + b
74
+ return mu
75
+
76
+
77
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
78
+ def retrieve_timesteps(
79
+ scheduler,
80
+ num_inference_steps: int | None = None,
81
+ device: str | torch.device | None = None,
82
+ timesteps: list[int] | None = None,
83
+ sigmas: list[float] | None = None,
84
+ **kwargs,
85
+ ):
86
+ r"""
87
+ Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
88
+ custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
89
+
90
+ Args:
91
+ scheduler (`SchedulerMixin`):
92
+ The scheduler to get timesteps from.
93
+ num_inference_steps (`int`):
94
+ The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
95
+ must be `None`.
96
+ device (`str` or `torch.device`, *optional*):
97
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
98
+ timesteps (`list[int]`, *optional*):
99
+ Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
100
+ `num_inference_steps` and `sigmas` must be `None`.
101
+ sigmas (`list[float]`, *optional*):
102
+ Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
103
+ `num_inference_steps` and `timesteps` must be `None`.
104
+
105
+ Returns:
106
+ `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
107
+ second element is the number of inference steps.
108
+ """
109
+ if timesteps is not None and sigmas is not None:
110
+ raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
111
+ if timesteps is not None:
112
+ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
113
+ if not accepts_timesteps:
114
+ raise ValueError(
115
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
116
+ f" timestep schedules. Please check whether you are using the correct scheduler."
117
+ )
118
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
119
+ timesteps = scheduler.timesteps
120
+ num_inference_steps = len(timesteps)
121
+ elif sigmas is not None:
122
+ accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
123
+ if not accept_sigmas:
124
+ raise ValueError(
125
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
126
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
127
+ )
128
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
129
+ timesteps = scheduler.timesteps
130
+ num_inference_steps = len(timesteps)
131
+ else:
132
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
133
+ timesteps = scheduler.timesteps
134
+ return timesteps, num_inference_steps
135
+
136
+
137
+ def get_default_z_image_sigmas(num_inference_steps: int) -> list[float]:
138
+ return torch.linspace(1.0, 1 / num_inference_steps, num_inference_steps).tolist()
139
+
140
+
141
+ class ZImagePipeline(DiffusionPipeline, ZImageLoraLoaderMixin, FromSingleFileMixin):
142
+ model_cpu_offload_seq = "text_encoder->transformer->vae"
143
+ _optional_components = []
144
+ _callback_tensor_inputs = ["latents", "prompt_embeds"]
145
+
146
+ def __init__(
147
+ self,
148
+ scheduler: FlowMatchEulerDiscreteScheduler,
149
+ vae: AutoencoderKL,
150
+ text_encoder: PreTrainedModel,
151
+ tokenizer: AutoTokenizer,
152
+ transformer: ZImageTransformer2DModel,
153
+ ):
154
+ super().__init__()
155
+
156
+ self.register_modules(
157
+ vae=vae,
158
+ text_encoder=text_encoder,
159
+ tokenizer=tokenizer,
160
+ scheduler=scheduler,
161
+ transformer=transformer,
162
+ )
163
+ self.vae_scale_factor = (
164
+ 2 ** (len(self.vae.config.block_out_channels) - 1) if hasattr(self, "vae") and self.vae is not None else 8
165
+ )
166
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2)
167
+
168
+ def encode_prompt(
169
+ self,
170
+ prompt: str | list[str],
171
+ device: torch.device | None = None,
172
+ do_classifier_free_guidance: bool = True,
173
+ negative_prompt: str | list[str] | None = None,
174
+ prompt_embeds: list[torch.FloatTensor] | None = None,
175
+ negative_prompt_embeds: torch.FloatTensor | None = None,
176
+ max_sequence_length: int = 512,
177
+ ):
178
+ prompt = [prompt] if isinstance(prompt, str) else prompt
179
+ prompt_embeds = self._encode_prompt(
180
+ prompt=prompt,
181
+ device=device,
182
+ prompt_embeds=prompt_embeds,
183
+ max_sequence_length=max_sequence_length,
184
+ )
185
+
186
+ if do_classifier_free_guidance:
187
+ if negative_prompt is None:
188
+ negative_prompt = ["" for _ in prompt]
189
+ else:
190
+ negative_prompt = [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
191
+ assert len(prompt) == len(negative_prompt)
192
+ negative_prompt_embeds = self._encode_prompt(
193
+ prompt=negative_prompt,
194
+ device=device,
195
+ prompt_embeds=negative_prompt_embeds,
196
+ max_sequence_length=max_sequence_length,
197
+ )
198
+ else:
199
+ negative_prompt_embeds = []
200
+ return prompt_embeds, negative_prompt_embeds
201
+
202
+ def _encode_prompt(
203
+ self,
204
+ prompt: str | list[str],
205
+ device: torch.device | None = None,
206
+ prompt_embeds: list[torch.FloatTensor] | None = None,
207
+ max_sequence_length: int = 512,
208
+ ) -> list[torch.FloatTensor]:
209
+ device = device or self._execution_device
210
+
211
+ if prompt_embeds is not None:
212
+ return prompt_embeds
213
+
214
+ if isinstance(prompt, str):
215
+ prompt = [prompt]
216
+
217
+ for i, prompt_item in enumerate(prompt):
218
+ messages = [
219
+ {"role": "user", "content": prompt_item},
220
+ ]
221
+ prompt_item = self.tokenizer.apply_chat_template(
222
+ messages,
223
+ tokenize=False,
224
+ add_generation_prompt=True,
225
+ enable_thinking=True,
226
+ )
227
+ prompt[i] = prompt_item
228
+
229
+ text_inputs = self.tokenizer(
230
+ prompt,
231
+ padding="max_length",
232
+ max_length=max_sequence_length,
233
+ truncation=True,
234
+ return_tensors="pt",
235
+ )
236
+
237
+ text_input_ids = text_inputs.input_ids.to(device)
238
+ prompt_masks = text_inputs.attention_mask.to(device).bool()
239
+
240
+ prompt_embeds = self.text_encoder(
241
+ input_ids=text_input_ids,
242
+ attention_mask=prompt_masks,
243
+ output_hidden_states=True,
244
+ ).hidden_states[-2]
245
+
246
+ embeddings_list = []
247
+
248
+ for i in range(len(prompt_embeds)):
249
+ embeddings_list.append(prompt_embeds[i][prompt_masks[i]])
250
+
251
+ return embeddings_list
252
+
253
+ def prepare_latents(
254
+ self,
255
+ batch_size,
256
+ num_channels_latents,
257
+ height,
258
+ width,
259
+ dtype,
260
+ device,
261
+ generator,
262
+ latents=None,
263
+ ):
264
+ height = 2 * (int(height) // (self.vae_scale_factor * 2))
265
+ width = 2 * (int(width) // (self.vae_scale_factor * 2))
266
+
267
+ shape = (batch_size, num_channels_latents, height, width)
268
+
269
+ if latents is None:
270
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
271
+ else:
272
+ if latents.shape != shape:
273
+ raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}")
274
+ latents = latents.to(device)
275
+ return latents
276
+
277
+ @property
278
+ def guidance_scale(self):
279
+ return self._guidance_scale
280
+
281
+ @property
282
+ def do_classifier_free_guidance(self):
283
+ return self._guidance_scale > 0
284
+
285
+ @property
286
+ def joint_attention_kwargs(self):
287
+ return self._joint_attention_kwargs
288
+
289
+ @property
290
+ def num_timesteps(self):
291
+ return self._num_timesteps
292
+
293
+ @property
294
+ def interrupt(self):
295
+ return self._interrupt
296
+
297
+ @torch.no_grad()
298
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
299
+ def __call__(
300
+ self,
301
+ prompt: str | list[str] = None,
302
+ height: int | None = None,
303
+ width: int | None = None,
304
+ num_inference_steps: int = 50,
305
+ sigmas: list[float] | None = None,
306
+ guidance_scale: float = 5.0,
307
+ cfg_normalization: bool = False,
308
+ cfg_truncation: float = 1.0,
309
+ negative_prompt: str | list[str] | None = None,
310
+ num_images_per_prompt: int | None = 1,
311
+ generator: torch.Generator | list[torch.Generator] | None = None,
312
+ latents: torch.FloatTensor | None = None,
313
+ prompt_embeds: list[torch.FloatTensor] | None = None,
314
+ negative_prompt_embeds: list[torch.FloatTensor] | None = None,
315
+ output_type: str | None = "pil",
316
+ return_dict: bool = True,
317
+ joint_attention_kwargs: dict[str, Any] | None = None,
318
+ callback_on_step_end: Callable[[int, int], None] | None = None,
319
+ callback_on_step_end_tensor_inputs: list[str] = ["latents"],
320
+ max_sequence_length: int = 512,
321
+ ):
322
+ r"""
323
+ Function invoked when calling the pipeline for generation.
324
+
325
+ Args:
326
+ prompt (`str` or `list[str]`, *optional*):
327
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
328
+ instead.
329
+ height (`int`, *optional*, defaults to 1024):
330
+ The height in pixels of the generated image.
331
+ width (`int`, *optional*, defaults to 1024):
332
+ The width in pixels of the generated image.
333
+ num_inference_steps (`int`, *optional*, defaults to 50):
334
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
335
+ expense of slower inference.
336
+ sigmas (`list[float]`, *optional*):
337
+ Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
338
+ their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
339
+ will be used.
340
+ guidance_scale (`float`, *optional*, defaults to 5.0):
341
+ Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
342
+ `guidance_scale` is defined as `w` of equation 2. of [Imagen
343
+ Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
344
+ 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
345
+ usually at the expense of lower image quality.
346
+ cfg_normalization (`bool`, *optional*, defaults to False):
347
+ Whether to apply configuration normalization.
348
+ cfg_truncation (`float`, *optional*, defaults to 1.0):
349
+ The truncation value for configuration.
350
+ negative_prompt (`str` or `list[str]`, *optional*):
351
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
352
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
353
+ less than `1`).
354
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
355
+ The number of images to generate per prompt.
356
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
357
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
358
+ to make generation deterministic.
359
+ latents (`torch.FloatTensor`, *optional*):
360
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
361
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
362
+ tensor will be generated by sampling using the supplied random `generator`.
363
+ prompt_embeds (`list[torch.FloatTensor]`, *optional*):
364
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
365
+ provided, text embeddings will be generated from `prompt` input argument.
366
+ negative_prompt_embeds (`list[torch.FloatTensor]`, *optional*):
367
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
368
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
369
+ argument.
370
+ output_type (`str`, *optional*, defaults to `"pil"`):
371
+ The output format of the generate image. Choose between
372
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
373
+ return_dict (`bool`, *optional*, defaults to `True`):
374
+ Whether or not to return a [`~pipelines.stable_diffusion.ZImagePipelineOutput`] instead of a plain
375
+ tuple.
376
+ joint_attention_kwargs (`dict`, *optional*):
377
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
378
+ `self.processor` in
379
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
380
+ callback_on_step_end (`Callable`, *optional*):
381
+ A function that calls at the end of each denoising steps during the inference. The function is called
382
+ with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
383
+ callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
384
+ `callback_on_step_end_tensor_inputs`.
385
+ callback_on_step_end_tensor_inputs (`List`, *optional*):
386
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
387
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
388
+ `._callback_tensor_inputs` attribute of your pipeline class.
389
+ max_sequence_length (`int`, *optional*, defaults to 512):
390
+ Maximum sequence length to use with the `prompt`.
391
+
392
+ Examples:
393
+
394
+ Returns:
395
+ [`~pipelines.z_image.ZImagePipelineOutput`] or `tuple`: [`~pipelines.z_image.ZImagePipelineOutput`] if
396
+ `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the
397
+ generated images.
398
+ """
399
+ height = height or 1024
400
+ width = width or 1024
401
+
402
+ vae_scale = self.vae_scale_factor * 2
403
+ if height % vae_scale != 0:
404
+ raise ValueError(
405
+ f"Height must be divisible by {vae_scale} (got {height}). "
406
+ f"Please adjust the height to a multiple of {vae_scale}."
407
+ )
408
+ if width % vae_scale != 0:
409
+ raise ValueError(
410
+ f"Width must be divisible by {vae_scale} (got {width}). "
411
+ f"Please adjust the width to a multiple of {vae_scale}."
412
+ )
413
+
414
+ device = self._execution_device
415
+
416
+ self._guidance_scale = guidance_scale
417
+ self._joint_attention_kwargs = joint_attention_kwargs
418
+ self._interrupt = False
419
+ self._cfg_normalization = cfg_normalization
420
+ self._cfg_truncation = cfg_truncation
421
+ # 2. Define call parameters
422
+ if prompt is not None and isinstance(prompt, str):
423
+ batch_size = 1
424
+ elif prompt is not None and isinstance(prompt, list):
425
+ batch_size = len(prompt)
426
+ else:
427
+ batch_size = len(prompt_embeds)
428
+
429
+ # If prompt_embeds is provided and prompt is None, skip encoding
430
+ if prompt_embeds is not None and prompt is None:
431
+ if self.do_classifier_free_guidance and negative_prompt_embeds is None:
432
+ raise ValueError(
433
+ "When `prompt_embeds` is provided without `prompt`, "
434
+ "`negative_prompt_embeds` must also be provided for classifier-free guidance."
435
+ )
436
+ else:
437
+ (
438
+ prompt_embeds,
439
+ negative_prompt_embeds,
440
+ ) = self.encode_prompt(
441
+ prompt=prompt,
442
+ negative_prompt=negative_prompt,
443
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
444
+ prompt_embeds=prompt_embeds,
445
+ negative_prompt_embeds=negative_prompt_embeds,
446
+ device=device,
447
+ max_sequence_length=max_sequence_length,
448
+ )
449
+
450
+ # 4. Prepare latent variables
451
+ num_channels_latents = self.transformer.in_channels
452
+
453
+ latents = self.prepare_latents(
454
+ batch_size * num_images_per_prompt,
455
+ num_channels_latents,
456
+ height,
457
+ width,
458
+ torch.float32,
459
+ device,
460
+ generator,
461
+ latents,
462
+ )
463
+
464
+ # Repeat prompt_embeds for num_images_per_prompt
465
+ if num_images_per_prompt > 1:
466
+ prompt_embeds = [pe for pe in prompt_embeds for _ in range(num_images_per_prompt)]
467
+ if self.do_classifier_free_guidance and negative_prompt_embeds:
468
+ negative_prompt_embeds = [npe for npe in negative_prompt_embeds for _ in range(num_images_per_prompt)]
469
+
470
+ actual_batch_size = batch_size * num_images_per_prompt
471
+ image_seq_len = (latents.shape[2] // 2) * (latents.shape[3] // 2)
472
+
473
+ # 5. Prepare timesteps
474
+ mu = calculate_shift(
475
+ image_seq_len,
476
+ self.scheduler.config.get("base_image_seq_len", 256),
477
+ self.scheduler.config.get("max_image_seq_len", 4096),
478
+ self.scheduler.config.get("base_shift", 0.5),
479
+ self.scheduler.config.get("max_shift", 1.15),
480
+ )
481
+ if sigmas is None:
482
+ sigmas = get_default_z_image_sigmas(num_inference_steps)
483
+ scheduler_kwargs = {"mu": mu}
484
+ timesteps, num_inference_steps = retrieve_timesteps(
485
+ self.scheduler,
486
+ num_inference_steps,
487
+ device,
488
+ sigmas=sigmas,
489
+ **scheduler_kwargs,
490
+ )
491
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
492
+ self._num_timesteps = len(timesteps)
493
+
494
+ # We set the index here to remove DtoH sync, helpful especially during compilation.
495
+ # Check out more details here: https://github.com/huggingface/diffusers/pull/11696
496
+ self.scheduler.set_begin_index(0)
497
+
498
+ if self.do_classifier_free_guidance and self._cfg_truncation is not None and float(self._cfg_truncation) <= 1:
499
+ _precomputed_t_norms = ((1000 - timesteps.float()) / 1000).tolist()
500
+ else:
501
+ _precomputed_t_norms = None
502
+
503
+ # 6. Denoising loop
504
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
505
+ for i, t in enumerate(timesteps):
506
+ if self.interrupt:
507
+ continue
508
+
509
+ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
510
+ timestep = t.expand(latents.shape[0])
511
+ timestep = (1000 - timestep) / 1000
512
+ current_guidance_scale = self.guidance_scale
513
+ if _precomputed_t_norms is not None:
514
+ if _precomputed_t_norms[i] > self._cfg_truncation:
515
+ current_guidance_scale = 0.0
516
+
517
+ # Run CFG only if configured AND scale is non-zero
518
+ apply_cfg = self.do_classifier_free_guidance and current_guidance_scale > 0
519
+
520
+ if apply_cfg:
521
+ latents_typed = latents.to(self.transformer.dtype)
522
+ latent_model_input = latents_typed.repeat(2, 1, 1, 1)
523
+ prompt_embeds_model_input = prompt_embeds + negative_prompt_embeds
524
+ timestep_model_input = timestep.repeat(2)
525
+ else:
526
+ latent_model_input = latents.to(self.transformer.dtype)
527
+ prompt_embeds_model_input = prompt_embeds
528
+ timestep_model_input = timestep
529
+
530
+ latent_model_input = latent_model_input.unsqueeze(2)
531
+ latent_model_input_list = list(latent_model_input.unbind(dim=0))
532
+
533
+ model_out_list = self.transformer(
534
+ latent_model_input_list, timestep_model_input, prompt_embeds_model_input, return_dict=False
535
+ )[0]
536
+
537
+ if apply_cfg:
538
+ # Perform CFG
539
+ pos_out = model_out_list[:actual_batch_size]
540
+ neg_out = model_out_list[actual_batch_size:]
541
+
542
+ noise_pred = []
543
+ for j in range(actual_batch_size):
544
+ pos = pos_out[j].float()
545
+ neg = neg_out[j].float()
546
+
547
+ pred = pos + current_guidance_scale * (pos - neg)
548
+
549
+ # Renormalization
550
+ if self._cfg_normalization and float(self._cfg_normalization) > 0.0:
551
+ ori_pos_norm = torch.linalg.vector_norm(pos)
552
+ new_pos_norm = torch.linalg.vector_norm(pred)
553
+ max_new_norm = ori_pos_norm * float(self._cfg_normalization)
554
+ if new_pos_norm > max_new_norm:
555
+ pred = pred * (max_new_norm / new_pos_norm)
556
+
557
+ noise_pred.append(pred)
558
+
559
+ noise_pred = torch.stack(noise_pred, dim=0)
560
+ else:
561
+ noise_pred = torch.stack([t.float() for t in model_out_list], dim=0)
562
+
563
+ noise_pred = noise_pred.squeeze(2)
564
+ noise_pred = -noise_pred
565
+
566
+ # compute the previous noisy sample x_t -> x_t-1
567
+ latents = self.scheduler.step(noise_pred.to(torch.float32), t, latents, return_dict=False)[0]
568
+ assert latents.dtype == torch.float32
569
+
570
+ if callback_on_step_end is not None:
571
+ callback_kwargs = {}
572
+ for k in callback_on_step_end_tensor_inputs:
573
+ callback_kwargs[k] = locals()[k]
574
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
575
+
576
+ latents = callback_outputs.pop("latents", latents)
577
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
578
+ negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
579
+
580
+ # call the callback, if provided
581
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
582
+ progress_bar.update()
583
+
584
+ if output_type == "latent":
585
+ image = latents
586
+
587
+ else:
588
+ latents = latents.to(self.vae.dtype)
589
+ latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor
590
+
591
+ image = self.vae.decode(latents, return_dict=False)[0]
592
+ image = self.image_processor.postprocess(image, output_type=output_type)
593
+
594
+ # Offload all models
595
+ self.maybe_free_model_hooks()
596
+
597
+ if not return_dict:
598
+ return (image,)
599
+
600
+ return ZImagePipelineOutput(images=image)
western_ai/europe/diffusers/src/diffusers/pipelines/z_image/pipeline_z_image_controlnet.py ADDED
@@ -0,0 +1,740 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 Alibaba Z-Image Team and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from typing import Any, Callable
17
+
18
+ import torch
19
+ from transformers import AutoTokenizer, PreTrainedModel
20
+
21
+ from ...image_processor import PipelineImageInput, VaeImageProcessor
22
+ from ...loaders import FromSingleFileMixin, ZImageLoraLoaderMixin
23
+ from ...models.autoencoders import AutoencoderKL
24
+ from ...models.controlnets import ZImageControlNetModel
25
+ from ...models.transformers import ZImageTransformer2DModel
26
+ from ...pipelines.pipeline_utils import DiffusionPipeline
27
+ from ...schedulers import FlowMatchEulerDiscreteScheduler
28
+ from ...utils import logging, replace_example_docstring
29
+ from ...utils.torch_utils import randn_tensor
30
+ from .pipeline_output import ZImagePipelineOutput
31
+
32
+
33
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
34
+
35
+ EXAMPLE_DOC_STRING = """
36
+ Examples:
37
+ ```py
38
+ >>> import torch
39
+ >>> from diffusers import ZImageControlNetPipeline
40
+ >>> from diffusers import ZImageControlNetModel
41
+ >>> from diffusers.utils import load_image
42
+ >>> from huggingface_hub import hf_hub_download
43
+
44
+ >>> controlnet = ZImageControlNetModel.from_single_file(
45
+ ... hf_hub_download(
46
+ ... "alibaba-pai/Z-Image-Turbo-Fun-Controlnet-Union",
47
+ ... filename="Z-Image-Turbo-Fun-Controlnet-Union.safetensors",
48
+ ... ),
49
+ ... torch_dtype=torch.bfloat16,
50
+ ... )
51
+
52
+ >>> # 2.1
53
+ >>> # controlnet = ZImageControlNetModel.from_single_file(
54
+ >>> # hf_hub_download(
55
+ >>> # "alibaba-pai/Z-Image-Turbo-Fun-Controlnet-Union-2.0",
56
+ >>> # filename="Z-Image-Turbo-Fun-Controlnet-Union-2.1.safetensors",
57
+ >>> # ),
58
+ >>> # torch_dtype=torch.bfloat16,
59
+ >>> # )
60
+
61
+ >>> # 2.0
62
+ >>> # controlnet = ZImageControlNetModel.from_single_file(
63
+ >>> # hf_hub_download(
64
+ >>> # "alibaba-pai/Z-Image-Turbo-Fun-Controlnet-Union-2.0",
65
+ >>> # filename="Z-Image-Turbo-Fun-Controlnet-Union-2.0.safetensors",
66
+ >>> # ),
67
+ >>> # torch_dtype=torch.bfloat16,
68
+ >>> # )
69
+
70
+ >>> pipe = ZImageControlNetPipeline.from_pretrained(
71
+ ... "Tongyi-MAI/Z-Image-Turbo", controlnet=controlnet, torch_dtype=torch.bfloat16
72
+ ... )
73
+ >>> pipe.to("cuda")
74
+
75
+ >>> # Optionally, set the attention backend to flash-attn 2 or 3, default is SDPA in PyTorch.
76
+ >>> # (1) Use flash attention 2
77
+ >>> # pipe.transformer.set_attention_backend("flash")
78
+ >>> # (2) Use flash attention 3
79
+ >>> # pipe.transformer.set_attention_backend("_flash_3")
80
+
81
+ >>> control_image = load_image(
82
+ ... "https://huggingface.co/alibaba-pai/Z-Image-Turbo-Fun-Controlnet-Union/resolve/main/asset/pose.jpg?download=true"
83
+ ... )
84
+ >>> prompt = "一位年轻女子站在阳光明媚的海岸线上,白裙在轻拂的海风中微微飘动。她拥有一头鲜艳的紫色长发,在风中轻盈舞动,发间系着一个精致的黑色蝴蝶结,与身后柔和的蔚蓝天空形成鲜明对比。她面容清秀,眉目精致,透着一股甜美的青春气息;神情柔和,略带羞涩,目光静静地凝望着远方的地平线,双手自然交叠于身前,仿佛沉浸在思绪之中。在她身后,是辽阔无垠、波光粼粼的大海,阳光洒在海面上,映出温暖的金色光晕。"
85
+ >>> image = pipe(
86
+ ... prompt,
87
+ ... control_image=control_image,
88
+ ... controlnet_conditioning_scale=0.75,
89
+ ... height=1728,
90
+ ... width=992,
91
+ ... num_inference_steps=8,
92
+ ... guidance_scale=0.0,
93
+ ... generator=torch.Generator("cuda").manual_seed(43),
94
+ ... ).images[0]
95
+ >>> image.save("zimage.png")
96
+ ```
97
+ """
98
+
99
+
100
+ # Copied from diffusers.pipelines.flux.pipeline_flux.calculate_shift
101
+ def calculate_shift(
102
+ image_seq_len,
103
+ base_seq_len: int = 256,
104
+ max_seq_len: int = 4096,
105
+ base_shift: float = 0.5,
106
+ max_shift: float = 1.15,
107
+ ):
108
+ m = (max_shift - base_shift) / (max_seq_len - base_seq_len)
109
+ b = base_shift - m * base_seq_len
110
+ mu = image_seq_len * m + b
111
+ return mu
112
+
113
+
114
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
115
+ def retrieve_latents(
116
+ encoder_output: torch.Tensor, generator: torch.Generator | None = None, sample_mode: str = "sample"
117
+ ):
118
+ if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
119
+ return encoder_output.latent_dist.sample(generator)
120
+ elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
121
+ return encoder_output.latent_dist.mode()
122
+ elif hasattr(encoder_output, "latents"):
123
+ return encoder_output.latents
124
+ else:
125
+ raise AttributeError("Could not access latents of provided encoder_output")
126
+
127
+
128
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
129
+ def retrieve_timesteps(
130
+ scheduler,
131
+ num_inference_steps: int | None = None,
132
+ device: str | torch.device | None = None,
133
+ timesteps: list[int] | None = None,
134
+ sigmas: list[float] | None = None,
135
+ **kwargs,
136
+ ):
137
+ r"""
138
+ Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
139
+ custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
140
+
141
+ Args:
142
+ scheduler (`SchedulerMixin`):
143
+ The scheduler to get timesteps from.
144
+ num_inference_steps (`int`):
145
+ The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
146
+ must be `None`.
147
+ device (`str` or `torch.device`, *optional*):
148
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
149
+ timesteps (`list[int]`, *optional*):
150
+ Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
151
+ `num_inference_steps` and `sigmas` must be `None`.
152
+ sigmas (`list[float]`, *optional*):
153
+ Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
154
+ `num_inference_steps` and `timesteps` must be `None`.
155
+
156
+ Returns:
157
+ `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
158
+ second element is the number of inference steps.
159
+ """
160
+ if timesteps is not None and sigmas is not None:
161
+ raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
162
+ if timesteps is not None:
163
+ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
164
+ if not accepts_timesteps:
165
+ raise ValueError(
166
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
167
+ f" timestep schedules. Please check whether you are using the correct scheduler."
168
+ )
169
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
170
+ timesteps = scheduler.timesteps
171
+ num_inference_steps = len(timesteps)
172
+ elif sigmas is not None:
173
+ accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
174
+ if not accept_sigmas:
175
+ raise ValueError(
176
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
177
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
178
+ )
179
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
180
+ timesteps = scheduler.timesteps
181
+ num_inference_steps = len(timesteps)
182
+ else:
183
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
184
+ timesteps = scheduler.timesteps
185
+ return timesteps, num_inference_steps
186
+
187
+
188
+ # Copied from diffusers.pipelines.z_image.pipeline_z_image.get_default_z_image_sigmas
189
+ def get_default_z_image_sigmas(num_inference_steps: int) -> list[float]:
190
+ return torch.linspace(1.0, 1 / num_inference_steps, num_inference_steps).tolist()
191
+
192
+
193
+ class ZImageControlNetPipeline(DiffusionPipeline, ZImageLoraLoaderMixin, FromSingleFileMixin):
194
+ model_cpu_offload_seq = "text_encoder->transformer->vae"
195
+ _optional_components = []
196
+ _callback_tensor_inputs = ["latents", "prompt_embeds"]
197
+
198
+ def __init__(
199
+ self,
200
+ scheduler: FlowMatchEulerDiscreteScheduler,
201
+ vae: AutoencoderKL,
202
+ text_encoder: PreTrainedModel,
203
+ tokenizer: AutoTokenizer,
204
+ transformer: ZImageTransformer2DModel,
205
+ controlnet: ZImageControlNetModel,
206
+ ):
207
+ super().__init__()
208
+ controlnet = ZImageControlNetModel.from_transformer(controlnet, transformer)
209
+
210
+ self.register_modules(
211
+ vae=vae,
212
+ text_encoder=text_encoder,
213
+ tokenizer=tokenizer,
214
+ scheduler=scheduler,
215
+ transformer=transformer,
216
+ controlnet=controlnet,
217
+ )
218
+ self.vae_scale_factor = (
219
+ 2 ** (len(self.vae.config.block_out_channels) - 1) if hasattr(self, "vae") and self.vae is not None else 8
220
+ )
221
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2)
222
+
223
+ def encode_prompt(
224
+ self,
225
+ prompt: str | list[str],
226
+ device: torch.device | None = None,
227
+ do_classifier_free_guidance: bool = True,
228
+ negative_prompt: str | list[str] | None = None,
229
+ prompt_embeds: list[torch.FloatTensor] | None = None,
230
+ negative_prompt_embeds: torch.FloatTensor | None = None,
231
+ max_sequence_length: int = 512,
232
+ ):
233
+ prompt = [prompt] if isinstance(prompt, str) else prompt
234
+ prompt_embeds = self._encode_prompt(
235
+ prompt=prompt,
236
+ device=device,
237
+ prompt_embeds=prompt_embeds,
238
+ max_sequence_length=max_sequence_length,
239
+ )
240
+
241
+ if do_classifier_free_guidance:
242
+ if negative_prompt is None:
243
+ negative_prompt = ["" for _ in prompt]
244
+ else:
245
+ negative_prompt = [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
246
+ assert len(prompt) == len(negative_prompt)
247
+ negative_prompt_embeds = self._encode_prompt(
248
+ prompt=negative_prompt,
249
+ device=device,
250
+ prompt_embeds=negative_prompt_embeds,
251
+ max_sequence_length=max_sequence_length,
252
+ )
253
+ else:
254
+ negative_prompt_embeds = []
255
+ return prompt_embeds, negative_prompt_embeds
256
+
257
+ def _encode_prompt(
258
+ self,
259
+ prompt: str | list[str],
260
+ device: torch.device | None = None,
261
+ prompt_embeds: list[torch.FloatTensor] | None = None,
262
+ max_sequence_length: int = 512,
263
+ ) -> list[torch.FloatTensor]:
264
+ device = device or self._execution_device
265
+
266
+ if prompt_embeds is not None:
267
+ return prompt_embeds
268
+
269
+ if isinstance(prompt, str):
270
+ prompt = [prompt]
271
+
272
+ for i, prompt_item in enumerate(prompt):
273
+ messages = [
274
+ {"role": "user", "content": prompt_item},
275
+ ]
276
+ prompt_item = self.tokenizer.apply_chat_template(
277
+ messages,
278
+ tokenize=False,
279
+ add_generation_prompt=True,
280
+ enable_thinking=True,
281
+ )
282
+ prompt[i] = prompt_item
283
+
284
+ text_inputs = self.tokenizer(
285
+ prompt,
286
+ padding="max_length",
287
+ max_length=max_sequence_length,
288
+ truncation=True,
289
+ return_tensors="pt",
290
+ )
291
+
292
+ text_input_ids = text_inputs.input_ids.to(device)
293
+ prompt_masks = text_inputs.attention_mask.to(device).bool()
294
+
295
+ prompt_embeds = self.text_encoder(
296
+ input_ids=text_input_ids,
297
+ attention_mask=prompt_masks,
298
+ output_hidden_states=True,
299
+ ).hidden_states[-2]
300
+
301
+ embeddings_list = []
302
+
303
+ for i in range(len(prompt_embeds)):
304
+ embeddings_list.append(prompt_embeds[i][prompt_masks[i]])
305
+
306
+ return embeddings_list
307
+
308
+ def prepare_latents(
309
+ self,
310
+ batch_size,
311
+ num_channels_latents,
312
+ height,
313
+ width,
314
+ dtype,
315
+ device,
316
+ generator,
317
+ latents=None,
318
+ ):
319
+ height = 2 * (int(height) // (self.vae_scale_factor * 2))
320
+ width = 2 * (int(width) // (self.vae_scale_factor * 2))
321
+
322
+ shape = (batch_size, num_channels_latents, height, width)
323
+
324
+ if latents is None:
325
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
326
+ else:
327
+ if latents.shape != shape:
328
+ raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}")
329
+ latents = latents.to(device)
330
+ return latents
331
+
332
+ # Copied from diffusers.pipelines.controlnet_sd3.pipeline_stable_diffusion_3_controlnet.StableDiffusion3ControlNetPipeline.prepare_image
333
+ def prepare_image(
334
+ self,
335
+ image,
336
+ width,
337
+ height,
338
+ batch_size,
339
+ num_images_per_prompt,
340
+ device,
341
+ dtype,
342
+ do_classifier_free_guidance=False,
343
+ guess_mode=False,
344
+ ):
345
+ if isinstance(image, torch.Tensor):
346
+ pass
347
+ else:
348
+ image = self.image_processor.preprocess(image, height=height, width=width)
349
+
350
+ image_batch_size = image.shape[0]
351
+
352
+ if image_batch_size == 1:
353
+ repeat_by = batch_size
354
+ else:
355
+ # image batch size is the same as prompt batch size
356
+ repeat_by = num_images_per_prompt
357
+
358
+ image = image.repeat_interleave(repeat_by, dim=0)
359
+
360
+ image = image.to(device=device, dtype=dtype)
361
+
362
+ if do_classifier_free_guidance and not guess_mode:
363
+ image = torch.cat([image] * 2)
364
+
365
+ return image
366
+
367
+ @property
368
+ def guidance_scale(self):
369
+ return self._guidance_scale
370
+
371
+ @property
372
+ def do_classifier_free_guidance(self):
373
+ return self._guidance_scale > 0
374
+
375
+ @property
376
+ def joint_attention_kwargs(self):
377
+ return self._joint_attention_kwargs
378
+
379
+ @property
380
+ def num_timesteps(self):
381
+ return self._num_timesteps
382
+
383
+ @property
384
+ def interrupt(self):
385
+ return self._interrupt
386
+
387
+ @torch.no_grad()
388
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
389
+ def __call__(
390
+ self,
391
+ prompt: str | list[str] = None,
392
+ height: int | None = None,
393
+ width: int | None = None,
394
+ num_inference_steps: int = 50,
395
+ sigmas: list[float] | None = None,
396
+ guidance_scale: float = 5.0,
397
+ control_image: PipelineImageInput = None,
398
+ controlnet_conditioning_scale: float | list[float] = 0.75,
399
+ cfg_normalization: bool = False,
400
+ cfg_truncation: float = 1.0,
401
+ negative_prompt: str | list[str] | None = None,
402
+ num_images_per_prompt: int | None = 1,
403
+ generator: torch.Generator | list[torch.Generator] | None = None,
404
+ latents: torch.FloatTensor | None = None,
405
+ prompt_embeds: list[torch.FloatTensor] | None = None,
406
+ negative_prompt_embeds: list[torch.FloatTensor] | None = None,
407
+ output_type: str | None = "pil",
408
+ return_dict: bool = True,
409
+ joint_attention_kwargs: dict[str, Any] | None = None,
410
+ callback_on_step_end: Callable[[int, int], None] | None = None,
411
+ callback_on_step_end_tensor_inputs: list[str] = ["latents"],
412
+ max_sequence_length: int = 512,
413
+ ):
414
+ r"""
415
+ Function invoked when calling the pipeline for generation.
416
+
417
+ Args:
418
+ prompt (`str` or `list[str]`, *optional*):
419
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
420
+ instead.
421
+ height (`int`, *optional*, defaults to 1024):
422
+ The height in pixels of the generated image.
423
+ width (`int`, *optional*, defaults to 1024):
424
+ The width in pixels of the generated image.
425
+ num_inference_steps (`int`, *optional*, defaults to 50):
426
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
427
+ expense of slower inference.
428
+ sigmas (`list[float]`, *optional*):
429
+ Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
430
+ their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
431
+ will be used.
432
+ guidance_scale (`float`, *optional*, defaults to 5.0):
433
+ Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
434
+ `guidance_scale` is defined as `w` of equation 2. of [Imagen
435
+ Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
436
+ 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
437
+ usually at the expense of lower image quality.
438
+ control_image (`PipelineImageInput`):
439
+ The ControlNet input condition to provide guidance to the `transformer` for generation. If the type is
440
+ specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted
441
+ as an image. The dimensions of the output image defaults to `control_image`'s dimensions. If height
442
+ and/or width are passed, `control_image` is resized accordingly.
443
+ controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 0.75):
444
+ The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added
445
+ to the residual in the original `transformer`.
446
+ cfg_normalization (`bool`, *optional*, defaults to False):
447
+ Whether to apply configuration normalization.
448
+ cfg_truncation (`float`, *optional*, defaults to 1.0):
449
+ The truncation value for configuration.
450
+ negative_prompt (`str` or `list[str]`, *optional*):
451
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
452
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
453
+ less than `1`).
454
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
455
+ The number of images to generate per prompt.
456
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
457
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
458
+ to make generation deterministic.
459
+ latents (`torch.FloatTensor`, *optional*):
460
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
461
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
462
+ tensor will be generated by sampling using the supplied random `generator`.
463
+ prompt_embeds (`list[torch.FloatTensor]`, *optional*):
464
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
465
+ provided, text embeddings will be generated from `prompt` input argument.
466
+ negative_prompt_embeds (`list[torch.FloatTensor]`, *optional*):
467
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
468
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
469
+ argument.
470
+ output_type (`str`, *optional*, defaults to `"pil"`):
471
+ The output format of the generate image. Choose between
472
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
473
+ return_dict (`bool`, *optional*, defaults to `True`):
474
+ Whether or not to return a [`~pipelines.stable_diffusion.ZImagePipelineOutput`] instead of a plain
475
+ tuple.
476
+ joint_attention_kwargs (`dict`, *optional*):
477
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
478
+ `self.processor` in
479
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
480
+ callback_on_step_end (`Callable`, *optional*):
481
+ A function that calls at the end of each denoising steps during the inference. The function is called
482
+ with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
483
+ callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
484
+ `callback_on_step_end_tensor_inputs`.
485
+ callback_on_step_end_tensor_inputs (`List`, *optional*):
486
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
487
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
488
+ `._callback_tensor_inputs` attribute of your pipeline class.
489
+ max_sequence_length (`int`, *optional*, defaults to 512):
490
+ Maximum sequence length to use with the `prompt`.
491
+
492
+ Examples:
493
+
494
+ Returns:
495
+ [`~pipelines.z_image.ZImagePipelineOutput`] or `tuple`: [`~pipelines.z_image.ZImagePipelineOutput`] if
496
+ `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the
497
+ generated images.
498
+ """
499
+ height = height or 1024
500
+ width = width or 1024
501
+
502
+ vae_scale = self.vae_scale_factor * 2
503
+ if height % vae_scale != 0:
504
+ raise ValueError(
505
+ f"Height must be divisible by {vae_scale} (got {height}). "
506
+ f"Please adjust the height to a multiple of {vae_scale}."
507
+ )
508
+ if width % vae_scale != 0:
509
+ raise ValueError(
510
+ f"Width must be divisible by {vae_scale} (got {width}). "
511
+ f"Please adjust the width to a multiple of {vae_scale}."
512
+ )
513
+
514
+ device = self._execution_device
515
+
516
+ self._guidance_scale = guidance_scale
517
+ self._joint_attention_kwargs = joint_attention_kwargs
518
+ self._interrupt = False
519
+ self._cfg_normalization = cfg_normalization
520
+ self._cfg_truncation = cfg_truncation
521
+ # 2. Define call parameters
522
+ if prompt is not None and isinstance(prompt, str):
523
+ batch_size = 1
524
+ elif prompt is not None and isinstance(prompt, list):
525
+ batch_size = len(prompt)
526
+ else:
527
+ batch_size = len(prompt_embeds)
528
+
529
+ # If prompt_embeds is provided and prompt is None, skip encoding
530
+ if prompt_embeds is not None and prompt is None:
531
+ if self.do_classifier_free_guidance and negative_prompt_embeds is None:
532
+ raise ValueError(
533
+ "When `prompt_embeds` is provided without `prompt`, "
534
+ "`negative_prompt_embeds` must also be provided for classifier-free guidance."
535
+ )
536
+ else:
537
+ (
538
+ prompt_embeds,
539
+ negative_prompt_embeds,
540
+ ) = self.encode_prompt(
541
+ prompt=prompt,
542
+ negative_prompt=negative_prompt,
543
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
544
+ prompt_embeds=prompt_embeds,
545
+ negative_prompt_embeds=negative_prompt_embeds,
546
+ device=device,
547
+ max_sequence_length=max_sequence_length,
548
+ )
549
+
550
+ # 4. Prepare latent variables
551
+ num_channels_latents = self.transformer.in_channels
552
+
553
+ control_image = self.prepare_image(
554
+ image=control_image,
555
+ width=width,
556
+ height=height,
557
+ batch_size=batch_size * num_images_per_prompt,
558
+ num_images_per_prompt=num_images_per_prompt,
559
+ device=device,
560
+ dtype=self.vae.dtype,
561
+ )
562
+ height, width = control_image.shape[-2:]
563
+ control_image = retrieve_latents(self.vae.encode(control_image), generator=generator, sample_mode="argmax")
564
+ control_image = (control_image - self.vae.config.shift_factor) * self.vae.config.scaling_factor
565
+ control_image = control_image.unsqueeze(2)
566
+
567
+ if num_channels_latents != self.controlnet.config.control_in_dim:
568
+ # For model version 2.0
569
+ control_image = torch.cat(
570
+ [
571
+ control_image,
572
+ torch.zeros(
573
+ control_image.shape[0],
574
+ self.controlnet.config.control_in_dim - num_channels_latents,
575
+ *control_image.shape[2:],
576
+ ).to(device=control_image.device, dtype=control_image.dtype),
577
+ ],
578
+ dim=1,
579
+ )
580
+
581
+ latents = self.prepare_latents(
582
+ batch_size * num_images_per_prompt,
583
+ num_channels_latents,
584
+ height,
585
+ width,
586
+ torch.float32,
587
+ device,
588
+ generator,
589
+ latents,
590
+ )
591
+
592
+ # Repeat prompt_embeds for num_images_per_prompt
593
+ if num_images_per_prompt > 1:
594
+ prompt_embeds = [pe for pe in prompt_embeds for _ in range(num_images_per_prompt)]
595
+ if self.do_classifier_free_guidance and negative_prompt_embeds:
596
+ negative_prompt_embeds = [npe for npe in negative_prompt_embeds for _ in range(num_images_per_prompt)]
597
+
598
+ actual_batch_size = batch_size * num_images_per_prompt
599
+ image_seq_len = (latents.shape[2] // 2) * (latents.shape[3] // 2)
600
+
601
+ # 5. Prepare timesteps
602
+ mu = calculate_shift(
603
+ image_seq_len,
604
+ self.scheduler.config.get("base_image_seq_len", 256),
605
+ self.scheduler.config.get("max_image_seq_len", 4096),
606
+ self.scheduler.config.get("base_shift", 0.5),
607
+ self.scheduler.config.get("max_shift", 1.15),
608
+ )
609
+ if sigmas is None:
610
+ sigmas = get_default_z_image_sigmas(num_inference_steps)
611
+ scheduler_kwargs = {"mu": mu}
612
+ timesteps, num_inference_steps = retrieve_timesteps(
613
+ self.scheduler,
614
+ num_inference_steps,
615
+ device,
616
+ sigmas=sigmas,
617
+ **scheduler_kwargs,
618
+ )
619
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
620
+ self._num_timesteps = len(timesteps)
621
+
622
+ # 6. Denoising loop
623
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
624
+ for i, t in enumerate(timesteps):
625
+ if self.interrupt:
626
+ continue
627
+
628
+ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
629
+ timestep = t.expand(latents.shape[0])
630
+ timestep = (1000 - timestep) / 1000
631
+ # Normalized time for time-aware config (0 at start, 1 at end)
632
+ t_norm = timestep[0].item()
633
+
634
+ # Handle cfg truncation
635
+ current_guidance_scale = self.guidance_scale
636
+ if (
637
+ self.do_classifier_free_guidance
638
+ and self._cfg_truncation is not None
639
+ and float(self._cfg_truncation) <= 1
640
+ ):
641
+ if t_norm > self._cfg_truncation:
642
+ current_guidance_scale = 0.0
643
+
644
+ # Run CFG only if configured AND scale is non-zero
645
+ apply_cfg = self.do_classifier_free_guidance and current_guidance_scale > 0
646
+
647
+ if apply_cfg:
648
+ latents_typed = latents.to(self.transformer.dtype)
649
+ latent_model_input = latents_typed.repeat(2, 1, 1, 1)
650
+ prompt_embeds_model_input = prompt_embeds + negative_prompt_embeds
651
+ timestep_model_input = timestep.repeat(2)
652
+ control_image_input = control_image.repeat(2, 1, 1, 1, 1)
653
+ else:
654
+ latent_model_input = latents.to(self.transformer.dtype)
655
+ prompt_embeds_model_input = prompt_embeds
656
+ timestep_model_input = timestep
657
+ control_image_input = control_image
658
+
659
+ latent_model_input = latent_model_input.unsqueeze(2)
660
+ latent_model_input_list = list(latent_model_input.unbind(dim=0))
661
+
662
+ controlnet_block_samples = self.controlnet(
663
+ latent_model_input_list,
664
+ timestep_model_input,
665
+ prompt_embeds_model_input,
666
+ control_image_input,
667
+ conditioning_scale=controlnet_conditioning_scale,
668
+ )
669
+
670
+ model_out_list = self.transformer(
671
+ latent_model_input_list,
672
+ timestep_model_input,
673
+ prompt_embeds_model_input,
674
+ controlnet_block_samples=controlnet_block_samples,
675
+ )[0]
676
+
677
+ if apply_cfg:
678
+ # Perform CFG
679
+ pos_out = model_out_list[:actual_batch_size]
680
+ neg_out = model_out_list[actual_batch_size:]
681
+
682
+ noise_pred = []
683
+ for j in range(actual_batch_size):
684
+ pos = pos_out[j].float()
685
+ neg = neg_out[j].float()
686
+
687
+ pred = pos + current_guidance_scale * (pos - neg)
688
+
689
+ # Renormalization
690
+ if self._cfg_normalization and float(self._cfg_normalization) > 0.0:
691
+ ori_pos_norm = torch.linalg.vector_norm(pos)
692
+ new_pos_norm = torch.linalg.vector_norm(pred)
693
+ max_new_norm = ori_pos_norm * float(self._cfg_normalization)
694
+ if new_pos_norm > max_new_norm:
695
+ pred = pred * (max_new_norm / new_pos_norm)
696
+
697
+ noise_pred.append(pred)
698
+
699
+ noise_pred = torch.stack(noise_pred, dim=0)
700
+ else:
701
+ noise_pred = torch.stack([t.float() for t in model_out_list], dim=0)
702
+
703
+ noise_pred = noise_pred.squeeze(2)
704
+ noise_pred = -noise_pred
705
+
706
+ # compute the previous noisy sample x_t -> x_t-1
707
+ latents = self.scheduler.step(noise_pred.to(torch.float32), t, latents, return_dict=False)[0]
708
+ assert latents.dtype == torch.float32
709
+
710
+ if callback_on_step_end is not None:
711
+ callback_kwargs = {}
712
+ for k in callback_on_step_end_tensor_inputs:
713
+ callback_kwargs[k] = locals()[k]
714
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
715
+
716
+ latents = callback_outputs.pop("latents", latents)
717
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
718
+ negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
719
+
720
+ # call the callback, if provided
721
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
722
+ progress_bar.update()
723
+
724
+ if output_type == "latent":
725
+ image = latents
726
+
727
+ else:
728
+ latents = latents.to(self.vae.dtype)
729
+ latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor
730
+
731
+ image = self.vae.decode(latents, return_dict=False)[0]
732
+ image = self.image_processor.postprocess(image, output_type=output_type)
733
+
734
+ # Offload all models
735
+ self.maybe_free_model_hooks()
736
+
737
+ if not return_dict:
738
+ return (image,)
739
+
740
+ return ZImagePipelineOutput(images=image)
western_ai/europe/diffusers/src/diffusers/pipelines/z_image/pipeline_z_image_controlnet_inpaint.py ADDED
@@ -0,0 +1,767 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 Alibaba Z-Image Team and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from typing import Any, Callable
17
+
18
+ import torch
19
+ import torch.nn.functional as F
20
+ from transformers import AutoTokenizer, PreTrainedModel
21
+
22
+ from ...image_processor import PipelineImageInput, VaeImageProcessor
23
+ from ...loaders import FromSingleFileMixin, ZImageLoraLoaderMixin
24
+ from ...models.autoencoders import AutoencoderKL
25
+ from ...models.controlnets import ZImageControlNetModel
26
+ from ...models.transformers import ZImageTransformer2DModel
27
+ from ...pipelines.pipeline_utils import DiffusionPipeline
28
+ from ...schedulers import FlowMatchEulerDiscreteScheduler
29
+ from ...utils import logging, replace_example_docstring
30
+ from ...utils.torch_utils import randn_tensor
31
+ from .pipeline_output import ZImagePipelineOutput
32
+
33
+
34
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
35
+
36
+ EXAMPLE_DOC_STRING = """
37
+ Examples:
38
+ ```py
39
+ >>> import torch
40
+ >>> from diffusers import ZImageControlNetInpaintPipeline
41
+ >>> from diffusers import ZImageControlNetModel
42
+ >>> from diffusers.utils import load_image
43
+ >>> from huggingface_hub import hf_hub_download
44
+
45
+ >>> controlnet = ZImageControlNetModel.from_single_file(
46
+ ... hf_hub_download(
47
+ ... "alibaba-pai/Z-Image-Turbo-Fun-Controlnet-Union-2.0",
48
+ ... filename="Z-Image-Turbo-Fun-Controlnet-Union-2.1.safetensors",
49
+ ... ),
50
+ ... torch_dtype=torch.bfloat16,
51
+ ... )
52
+
53
+ >>> # 2.0
54
+ >>> # controlnet = ZImageControlNetModel.from_single_file(
55
+ >>> # hf_hub_download(
56
+ >>> # "alibaba-pai/Z-Image-Turbo-Fun-Controlnet-Union-2.0",
57
+ >>> # filename="Z-Image-Turbo-Fun-Controlnet-Union-2.0.safetensors",
58
+ >>> # ),
59
+ >>> # torch_dtype=torch.bfloat16,
60
+ >>> # )
61
+
62
+ >>> pipe = ZImageControlNetInpaintPipeline.from_pretrained(
63
+ ... "Tongyi-MAI/Z-Image-Turbo", controlnet=controlnet, torch_dtype=torch.bfloat16
64
+ ... )
65
+ >>> pipe.to("cuda")
66
+
67
+ >>> # Optionally, set the attention backend to flash-attn 2 or 3, default is SDPA in PyTorch.
68
+ >>> # (1) Use flash attention 2
69
+ >>> # pipe.transformer.set_attention_backend("flash")
70
+ >>> # (2) Use flash attention 3
71
+ >>> # pipe.transformer.set_attention_backend("_flash_3")
72
+
73
+ >>> image = load_image(
74
+ ... "https://huggingface.co/alibaba-pai/Z-Image-Turbo-Fun-Controlnet-Union-2.0/resolve/main/asset/inpaint.jpg?download=true"
75
+ ... )
76
+ >>> mask_image = load_image(
77
+ ... "https://huggingface.co/alibaba-pai/Z-Image-Turbo-Fun-Controlnet-Union-2.0/resolve/main/asset/mask.jpg?download=true"
78
+ ... )
79
+ >>> control_image = load_image(
80
+ ... "https://huggingface.co/alibaba-pai/Z-Image-Turbo-Fun-Controlnet-Union-2.0/resolve/main/asset/pose.jpg?download=true"
81
+ ... )
82
+ >>> prompt = "一位年轻女子站在阳光明媚的海岸线上,画面为全身竖构图,身体微微侧向右侧,左手自然下垂,右臂弯曲扶在腰间,她的手指清晰可见,站姿放松而略带羞涩。她身穿轻盈的白色连衣裙,裙摆在海风中轻轻飘动,布料半透、质感柔软。女子拥有一头鲜艳的及腰紫色长发,被海风吹起,在身侧轻盈飞舞,发间系着一个精致的黑色蝴蝶结,与发色形成对比。她面容清秀,眉目精致,肤色白皙细腻,表情温柔略显羞涩,微微低头,眼神静静望向远处的海平线,流露出甜美的青春气息与若有所思的神情。背景是辽阔无垠的海洋与蔚蓝天空,阳光从侧前方洒下,海面波光粼粼,泛着温暖的金色光晕,天空清澈明亮,云朵稀薄,整体色调清新唯美。"
83
+ >>> image = pipe(
84
+ ... prompt,
85
+ ... image=image,
86
+ ... mask_image=mask_image,
87
+ ... control_image=control_image,
88
+ ... controlnet_conditioning_scale=0.75,
89
+ ... height=1728,
90
+ ... width=992,
91
+ ... num_inference_steps=25,
92
+ ... guidance_scale=0.0,
93
+ ... generator=torch.Generator("cuda").manual_seed(43),
94
+ ... ).images[0]
95
+ >>> image.save("zimage-inpaint.png")
96
+ ```
97
+ """
98
+
99
+
100
+ # Copied from diffusers.pipelines.flux.pipeline_flux.calculate_shift
101
+ def calculate_shift(
102
+ image_seq_len,
103
+ base_seq_len: int = 256,
104
+ max_seq_len: int = 4096,
105
+ base_shift: float = 0.5,
106
+ max_shift: float = 1.15,
107
+ ):
108
+ m = (max_shift - base_shift) / (max_seq_len - base_seq_len)
109
+ b = base_shift - m * base_seq_len
110
+ mu = image_seq_len * m + b
111
+ return mu
112
+
113
+
114
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
115
+ def retrieve_latents(
116
+ encoder_output: torch.Tensor, generator: torch.Generator | None = None, sample_mode: str = "sample"
117
+ ):
118
+ if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
119
+ return encoder_output.latent_dist.sample(generator)
120
+ elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
121
+ return encoder_output.latent_dist.mode()
122
+ elif hasattr(encoder_output, "latents"):
123
+ return encoder_output.latents
124
+ else:
125
+ raise AttributeError("Could not access latents of provided encoder_output")
126
+
127
+
128
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
129
+ def retrieve_timesteps(
130
+ scheduler,
131
+ num_inference_steps: int | None = None,
132
+ device: str | torch.device | None = None,
133
+ timesteps: list[int] | None = None,
134
+ sigmas: list[float] | None = None,
135
+ **kwargs,
136
+ ):
137
+ r"""
138
+ Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
139
+ custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
140
+
141
+ Args:
142
+ scheduler (`SchedulerMixin`):
143
+ The scheduler to get timesteps from.
144
+ num_inference_steps (`int`):
145
+ The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
146
+ must be `None`.
147
+ device (`str` or `torch.device`, *optional*):
148
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
149
+ timesteps (`list[int]`, *optional*):
150
+ Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
151
+ `num_inference_steps` and `sigmas` must be `None`.
152
+ sigmas (`list[float]`, *optional*):
153
+ Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
154
+ `num_inference_steps` and `timesteps` must be `None`.
155
+
156
+ Returns:
157
+ `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
158
+ second element is the number of inference steps.
159
+ """
160
+ if timesteps is not None and sigmas is not None:
161
+ raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
162
+ if timesteps is not None:
163
+ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
164
+ if not accepts_timesteps:
165
+ raise ValueError(
166
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
167
+ f" timestep schedules. Please check whether you are using the correct scheduler."
168
+ )
169
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
170
+ timesteps = scheduler.timesteps
171
+ num_inference_steps = len(timesteps)
172
+ elif sigmas is not None:
173
+ accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
174
+ if not accept_sigmas:
175
+ raise ValueError(
176
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
177
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
178
+ )
179
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
180
+ timesteps = scheduler.timesteps
181
+ num_inference_steps = len(timesteps)
182
+ else:
183
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
184
+ timesteps = scheduler.timesteps
185
+ return timesteps, num_inference_steps
186
+
187
+
188
+ # Copied from diffusers.pipelines.z_image.pipeline_z_image.get_default_z_image_sigmas
189
+ def get_default_z_image_sigmas(num_inference_steps: int) -> list[float]:
190
+ return torch.linspace(1.0, 1 / num_inference_steps, num_inference_steps).tolist()
191
+
192
+
193
+ class ZImageControlNetInpaintPipeline(DiffusionPipeline, ZImageLoraLoaderMixin, FromSingleFileMixin):
194
+ model_cpu_offload_seq = "text_encoder->transformer->vae"
195
+ _optional_components = []
196
+ _callback_tensor_inputs = ["latents", "prompt_embeds"]
197
+
198
+ def __init__(
199
+ self,
200
+ scheduler: FlowMatchEulerDiscreteScheduler,
201
+ vae: AutoencoderKL,
202
+ text_encoder: PreTrainedModel,
203
+ tokenizer: AutoTokenizer,
204
+ transformer: ZImageTransformer2DModel,
205
+ controlnet: ZImageControlNetModel,
206
+ ):
207
+ super().__init__()
208
+ if transformer.in_channels == controlnet.config.control_in_dim:
209
+ raise ValueError(
210
+ "ZImageControlNetInpaintPipeline is not compatible with `alibaba-pai/Z-Image-Turbo-Fun-Controlnet-Union`, use `alibaba-pai/Z-Image-Turbo-Fun-Controlnet-Union-2.0`."
211
+ )
212
+ controlnet = ZImageControlNetModel.from_transformer(controlnet, transformer)
213
+
214
+ self.register_modules(
215
+ vae=vae,
216
+ text_encoder=text_encoder,
217
+ tokenizer=tokenizer,
218
+ scheduler=scheduler,
219
+ transformer=transformer,
220
+ controlnet=controlnet,
221
+ )
222
+ self.vae_scale_factor = (
223
+ 2 ** (len(self.vae.config.block_out_channels) - 1) if hasattr(self, "vae") and self.vae is not None else 8
224
+ )
225
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2)
226
+ self.mask_processor = VaeImageProcessor(
227
+ vae_scale_factor=self.vae_scale_factor, do_normalize=False, do_binarize=True, do_convert_grayscale=True
228
+ )
229
+
230
+ def encode_prompt(
231
+ self,
232
+ prompt: str | list[str],
233
+ device: torch.device | None = None,
234
+ do_classifier_free_guidance: bool = True,
235
+ negative_prompt: str | list[str] | None = None,
236
+ prompt_embeds: list[torch.FloatTensor] | None = None,
237
+ negative_prompt_embeds: torch.FloatTensor | None = None,
238
+ max_sequence_length: int = 512,
239
+ ):
240
+ prompt = [prompt] if isinstance(prompt, str) else prompt
241
+ prompt_embeds = self._encode_prompt(
242
+ prompt=prompt,
243
+ device=device,
244
+ prompt_embeds=prompt_embeds,
245
+ max_sequence_length=max_sequence_length,
246
+ )
247
+
248
+ if do_classifier_free_guidance:
249
+ if negative_prompt is None:
250
+ negative_prompt = ["" for _ in prompt]
251
+ else:
252
+ negative_prompt = [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
253
+ assert len(prompt) == len(negative_prompt)
254
+ negative_prompt_embeds = self._encode_prompt(
255
+ prompt=negative_prompt,
256
+ device=device,
257
+ prompt_embeds=negative_prompt_embeds,
258
+ max_sequence_length=max_sequence_length,
259
+ )
260
+ else:
261
+ negative_prompt_embeds = []
262
+ return prompt_embeds, negative_prompt_embeds
263
+
264
+ def _encode_prompt(
265
+ self,
266
+ prompt: str | list[str],
267
+ device: torch.device | None = None,
268
+ prompt_embeds: list[torch.FloatTensor] | None = None,
269
+ max_sequence_length: int = 512,
270
+ ) -> list[torch.FloatTensor]:
271
+ device = device or self._execution_device
272
+
273
+ if prompt_embeds is not None:
274
+ return prompt_embeds
275
+
276
+ if isinstance(prompt, str):
277
+ prompt = [prompt]
278
+
279
+ for i, prompt_item in enumerate(prompt):
280
+ messages = [
281
+ {"role": "user", "content": prompt_item},
282
+ ]
283
+ prompt_item = self.tokenizer.apply_chat_template(
284
+ messages,
285
+ tokenize=False,
286
+ add_generation_prompt=True,
287
+ enable_thinking=True,
288
+ )
289
+ prompt[i] = prompt_item
290
+
291
+ text_inputs = self.tokenizer(
292
+ prompt,
293
+ padding="max_length",
294
+ max_length=max_sequence_length,
295
+ truncation=True,
296
+ return_tensors="pt",
297
+ )
298
+
299
+ text_input_ids = text_inputs.input_ids.to(device)
300
+ prompt_masks = text_inputs.attention_mask.to(device).bool()
301
+
302
+ prompt_embeds = self.text_encoder(
303
+ input_ids=text_input_ids,
304
+ attention_mask=prompt_masks,
305
+ output_hidden_states=True,
306
+ ).hidden_states[-2]
307
+
308
+ embeddings_list = []
309
+
310
+ for i in range(len(prompt_embeds)):
311
+ embeddings_list.append(prompt_embeds[i][prompt_masks[i]])
312
+
313
+ return embeddings_list
314
+
315
+ def prepare_latents(
316
+ self,
317
+ batch_size,
318
+ num_channels_latents,
319
+ height,
320
+ width,
321
+ dtype,
322
+ device,
323
+ generator,
324
+ latents=None,
325
+ ):
326
+ height = 2 * (int(height) // (self.vae_scale_factor * 2))
327
+ width = 2 * (int(width) // (self.vae_scale_factor * 2))
328
+
329
+ shape = (batch_size, num_channels_latents, height, width)
330
+
331
+ if latents is None:
332
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
333
+ else:
334
+ if latents.shape != shape:
335
+ raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}")
336
+ latents = latents.to(device)
337
+ return latents
338
+
339
+ # Copied from diffusers.pipelines.controlnet_sd3.pipeline_stable_diffusion_3_controlnet.StableDiffusion3ControlNetPipeline.prepare_image
340
+ def prepare_image(
341
+ self,
342
+ image,
343
+ width,
344
+ height,
345
+ batch_size,
346
+ num_images_per_prompt,
347
+ device,
348
+ dtype,
349
+ do_classifier_free_guidance=False,
350
+ guess_mode=False,
351
+ ):
352
+ if isinstance(image, torch.Tensor):
353
+ pass
354
+ else:
355
+ image = self.image_processor.preprocess(image, height=height, width=width)
356
+
357
+ image_batch_size = image.shape[0]
358
+
359
+ if image_batch_size == 1:
360
+ repeat_by = batch_size
361
+ else:
362
+ # image batch size is the same as prompt batch size
363
+ repeat_by = num_images_per_prompt
364
+
365
+ image = image.repeat_interleave(repeat_by, dim=0)
366
+
367
+ image = image.to(device=device, dtype=dtype)
368
+
369
+ if do_classifier_free_guidance and not guess_mode:
370
+ image = torch.cat([image] * 2)
371
+
372
+ return image
373
+
374
+ @property
375
+ def guidance_scale(self):
376
+ return self._guidance_scale
377
+
378
+ @property
379
+ def do_classifier_free_guidance(self):
380
+ return self._guidance_scale > 0
381
+
382
+ @property
383
+ def joint_attention_kwargs(self):
384
+ return self._joint_attention_kwargs
385
+
386
+ @property
387
+ def num_timesteps(self):
388
+ return self._num_timesteps
389
+
390
+ @property
391
+ def interrupt(self):
392
+ return self._interrupt
393
+
394
+ @torch.no_grad()
395
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
396
+ def __call__(
397
+ self,
398
+ prompt: str | list[str] = None,
399
+ height: int | None = None,
400
+ width: int | None = None,
401
+ num_inference_steps: int = 50,
402
+ sigmas: list[float] | None = None,
403
+ guidance_scale: float = 5.0,
404
+ image: PipelineImageInput = None,
405
+ mask_image: PipelineImageInput = None,
406
+ control_image: PipelineImageInput = None,
407
+ controlnet_conditioning_scale: float | list[float] = 0.75,
408
+ cfg_normalization: bool = False,
409
+ cfg_truncation: float = 1.0,
410
+ negative_prompt: str | list[str] | None = None,
411
+ num_images_per_prompt: int | None = 1,
412
+ generator: torch.Generator | list[torch.Generator] | None = None,
413
+ latents: torch.FloatTensor | None = None,
414
+ prompt_embeds: list[torch.FloatTensor] | None = None,
415
+ negative_prompt_embeds: list[torch.FloatTensor] | None = None,
416
+ output_type: str | None = "pil",
417
+ return_dict: bool = True,
418
+ joint_attention_kwargs: dict[str, Any] | None = None,
419
+ callback_on_step_end: Callable[[int, int], None] | None = None,
420
+ callback_on_step_end_tensor_inputs: list[str] = ["latents"],
421
+ max_sequence_length: int = 512,
422
+ ):
423
+ r"""
424
+ Function invoked when calling the pipeline for generation.
425
+
426
+ Args:
427
+ prompt (`str` or `list[str]`, *optional*):
428
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
429
+ instead.
430
+ height (`int`, *optional*, defaults to 1024):
431
+ The height in pixels of the generated image.
432
+ width (`int`, *optional*, defaults to 1024):
433
+ The width in pixels of the generated image.
434
+ num_inference_steps (`int`, *optional*, defaults to 50):
435
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
436
+ expense of slower inference.
437
+ sigmas (`list[float]`, *optional*):
438
+ Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
439
+ their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
440
+ will be used.
441
+ guidance_scale (`float`, *optional*, defaults to 5.0):
442
+ Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
443
+ `guidance_scale` is defined as `w` of equation 2. of [Imagen
444
+ Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
445
+ 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
446
+ usually at the expense of lower image quality.
447
+ image (`PipelineImageInput`):
448
+ `Image`, numpy array or tensor representing an image batch to be inpainted (which parts of the image to
449
+ be masked out with `mask_image` and repainted according to `prompt`).
450
+ mask_image (`PipelineImageInput`):
451
+ `Image`, numpy array or tensor representing an image batch to mask `image`. White pixels in the mask
452
+ are repainted while black pixels are preserved.
453
+ control_image (`PipelineImageInput`):
454
+ The ControlNet input condition to provide guidance to the `transformer` for generation. If the type is
455
+ specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted
456
+ as an image.
457
+ controlnet_conditioning_scale (`float` or `list[float]`, *optional*, defaults to 0.75):
458
+ The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added
459
+ to the residual in the original `transformer`.
460
+ cfg_normalization (`bool`, *optional*, defaults to False):
461
+ Whether to apply configuration normalization.
462
+ cfg_truncation (`float`, *optional*, defaults to 1.0):
463
+ The truncation value for configuration.
464
+ negative_prompt (`str` or `list[str]`, *optional*):
465
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
466
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
467
+ less than `1`).
468
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
469
+ The number of images to generate per prompt.
470
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
471
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
472
+ to make generation deterministic.
473
+ latents (`torch.FloatTensor`, *optional*):
474
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
475
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
476
+ tensor will be generated by sampling using the supplied random `generator`.
477
+ prompt_embeds (`list[torch.FloatTensor]`, *optional*):
478
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
479
+ provided, text embeddings will be generated from `prompt` input argument.
480
+ negative_prompt_embeds (`list[torch.FloatTensor]`, *optional*):
481
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
482
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
483
+ argument.
484
+ output_type (`str`, *optional*, defaults to `"pil"`):
485
+ The output format of the generate image. Choose between
486
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
487
+ return_dict (`bool`, *optional*, defaults to `True`):
488
+ Whether or not to return a [`~pipelines.stable_diffusion.ZImagePipelineOutput`] instead of a plain
489
+ tuple.
490
+ joint_attention_kwargs (`dict`, *optional*):
491
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
492
+ `self.processor` in
493
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
494
+ callback_on_step_end (`Callable`, *optional*):
495
+ A function that calls at the end of each denoising steps during the inference. The function is called
496
+ with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
497
+ callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
498
+ `callback_on_step_end_tensor_inputs`.
499
+ callback_on_step_end_tensor_inputs (`List`, *optional*):
500
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
501
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
502
+ `._callback_tensor_inputs` attribute of your pipeline class.
503
+ max_sequence_length (`int`, *optional*, defaults to 512):
504
+ Maximum sequence length to use with the `prompt`.
505
+
506
+ Examples:
507
+
508
+ Returns:
509
+ [`~pipelines.z_image.ZImagePipelineOutput`] or `tuple`: [`~pipelines.z_image.ZImagePipelineOutput`] if
510
+ `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the
511
+ generated images.
512
+ """
513
+ height = height or 1024
514
+ width = width or 1024
515
+
516
+ vae_scale = self.vae_scale_factor * 2
517
+ if height % vae_scale != 0:
518
+ raise ValueError(
519
+ f"Height must be divisible by {vae_scale} (got {height}). "
520
+ f"Please adjust the height to a multiple of {vae_scale}."
521
+ )
522
+ if width % vae_scale != 0:
523
+ raise ValueError(
524
+ f"Width must be divisible by {vae_scale} (got {width}). "
525
+ f"Please adjust the width to a multiple of {vae_scale}."
526
+ )
527
+
528
+ device = self._execution_device
529
+
530
+ self._guidance_scale = guidance_scale
531
+ self._joint_attention_kwargs = joint_attention_kwargs
532
+ self._interrupt = False
533
+ self._cfg_normalization = cfg_normalization
534
+ self._cfg_truncation = cfg_truncation
535
+ # 2. Define call parameters
536
+ if prompt is not None and isinstance(prompt, str):
537
+ batch_size = 1
538
+ elif prompt is not None and isinstance(prompt, list):
539
+ batch_size = len(prompt)
540
+ else:
541
+ batch_size = len(prompt_embeds)
542
+
543
+ # If prompt_embeds is provided and prompt is None, skip encoding
544
+ if prompt_embeds is not None and prompt is None:
545
+ if self.do_classifier_free_guidance and negative_prompt_embeds is None:
546
+ raise ValueError(
547
+ "When `prompt_embeds` is provided without `prompt`, "
548
+ "`negative_prompt_embeds` must also be provided for classifier-free guidance."
549
+ )
550
+ else:
551
+ (
552
+ prompt_embeds,
553
+ negative_prompt_embeds,
554
+ ) = self.encode_prompt(
555
+ prompt=prompt,
556
+ negative_prompt=negative_prompt,
557
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
558
+ prompt_embeds=prompt_embeds,
559
+ negative_prompt_embeds=negative_prompt_embeds,
560
+ device=device,
561
+ max_sequence_length=max_sequence_length,
562
+ )
563
+
564
+ # 4. Prepare latent variables
565
+ num_channels_latents = self.transformer.in_channels
566
+
567
+ control_image = self.prepare_image(
568
+ image=control_image,
569
+ width=width,
570
+ height=height,
571
+ batch_size=batch_size * num_images_per_prompt,
572
+ num_images_per_prompt=num_images_per_prompt,
573
+ device=device,
574
+ dtype=self.vae.dtype,
575
+ )
576
+ height, width = control_image.shape[-2:]
577
+ control_image = retrieve_latents(self.vae.encode(control_image), generator=generator, sample_mode="argmax")
578
+ control_image = (control_image - self.vae.config.shift_factor) * self.vae.config.scaling_factor
579
+ control_image = control_image.unsqueeze(2)
580
+
581
+ mask_condition = self.mask_processor.preprocess(mask_image, height=height, width=width)
582
+ mask_condition = torch.tile(mask_condition, [1, 3, 1, 1]).to(
583
+ device=control_image.device, dtype=control_image.dtype
584
+ )
585
+
586
+ init_image = self.prepare_image(
587
+ image=image,
588
+ width=width,
589
+ height=height,
590
+ batch_size=batch_size * num_images_per_prompt,
591
+ num_images_per_prompt=num_images_per_prompt,
592
+ device=device,
593
+ dtype=self.vae.dtype,
594
+ )
595
+ height, width = init_image.shape[-2:]
596
+ init_image = init_image * (mask_condition < 0.5)
597
+ init_image = retrieve_latents(self.vae.encode(init_image), generator=generator, sample_mode="argmax")
598
+ init_image = (init_image - self.vae.config.shift_factor) * self.vae.config.scaling_factor
599
+ init_image = init_image.unsqueeze(2)
600
+
601
+ mask_condition = F.interpolate(1 - mask_condition[:, :1], size=init_image.size()[-2:], mode="nearest").to(
602
+ device=control_image.device, dtype=control_image.dtype
603
+ )
604
+ mask_condition = mask_condition.unsqueeze(2)
605
+
606
+ control_image = torch.cat([control_image, mask_condition, init_image], dim=1)
607
+
608
+ latents = self.prepare_latents(
609
+ batch_size * num_images_per_prompt,
610
+ num_channels_latents,
611
+ height,
612
+ width,
613
+ torch.float32,
614
+ device,
615
+ generator,
616
+ latents,
617
+ )
618
+
619
+ # Repeat prompt_embeds for num_images_per_prompt
620
+ if num_images_per_prompt > 1:
621
+ prompt_embeds = [pe for pe in prompt_embeds for _ in range(num_images_per_prompt)]
622
+ if self.do_classifier_free_guidance and negative_prompt_embeds:
623
+ negative_prompt_embeds = [npe for npe in negative_prompt_embeds for _ in range(num_images_per_prompt)]
624
+
625
+ actual_batch_size = batch_size * num_images_per_prompt
626
+ image_seq_len = (latents.shape[2] // 2) * (latents.shape[3] // 2)
627
+
628
+ # 5. Prepare timesteps
629
+ mu = calculate_shift(
630
+ image_seq_len,
631
+ self.scheduler.config.get("base_image_seq_len", 256),
632
+ self.scheduler.config.get("max_image_seq_len", 4096),
633
+ self.scheduler.config.get("base_shift", 0.5),
634
+ self.scheduler.config.get("max_shift", 1.15),
635
+ )
636
+ if sigmas is None:
637
+ sigmas = get_default_z_image_sigmas(num_inference_steps)
638
+ scheduler_kwargs = {"mu": mu}
639
+ timesteps, num_inference_steps = retrieve_timesteps(
640
+ self.scheduler,
641
+ num_inference_steps,
642
+ device,
643
+ sigmas=sigmas,
644
+ **scheduler_kwargs,
645
+ )
646
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
647
+ self._num_timesteps = len(timesteps)
648
+
649
+ # 6. Denoising loop
650
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
651
+ for i, t in enumerate(timesteps):
652
+ if self.interrupt:
653
+ continue
654
+
655
+ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
656
+ timestep = t.expand(latents.shape[0])
657
+ timestep = (1000 - timestep) / 1000
658
+ # Normalized time for time-aware config (0 at start, 1 at end)
659
+ t_norm = timestep[0].item()
660
+
661
+ # Handle cfg truncation
662
+ current_guidance_scale = self.guidance_scale
663
+ if (
664
+ self.do_classifier_free_guidance
665
+ and self._cfg_truncation is not None
666
+ and float(self._cfg_truncation) <= 1
667
+ ):
668
+ if t_norm > self._cfg_truncation:
669
+ current_guidance_scale = 0.0
670
+
671
+ # Run CFG only if configured AND scale is non-zero
672
+ apply_cfg = self.do_classifier_free_guidance and current_guidance_scale > 0
673
+
674
+ if apply_cfg:
675
+ latents_typed = latents.to(self.transformer.dtype)
676
+ latent_model_input = latents_typed.repeat(2, 1, 1, 1)
677
+ prompt_embeds_model_input = prompt_embeds + negative_prompt_embeds
678
+ timestep_model_input = timestep.repeat(2)
679
+ control_image_input = control_image.repeat(2, 1, 1, 1, 1)
680
+ else:
681
+ latent_model_input = latents.to(self.transformer.dtype)
682
+ prompt_embeds_model_input = prompt_embeds
683
+ timestep_model_input = timestep
684
+ control_image_input = control_image
685
+
686
+ latent_model_input = latent_model_input.unsqueeze(2)
687
+ latent_model_input_list = list(latent_model_input.unbind(dim=0))
688
+
689
+ controlnet_block_samples = self.controlnet(
690
+ latent_model_input_list,
691
+ timestep_model_input,
692
+ prompt_embeds_model_input,
693
+ control_image_input,
694
+ conditioning_scale=controlnet_conditioning_scale,
695
+ )
696
+
697
+ model_out_list = self.transformer(
698
+ latent_model_input_list,
699
+ timestep_model_input,
700
+ prompt_embeds_model_input,
701
+ controlnet_block_samples=controlnet_block_samples,
702
+ )[0]
703
+
704
+ if apply_cfg:
705
+ # Perform CFG
706
+ pos_out = model_out_list[:actual_batch_size]
707
+ neg_out = model_out_list[actual_batch_size:]
708
+
709
+ noise_pred = []
710
+ for j in range(actual_batch_size):
711
+ pos = pos_out[j].float()
712
+ neg = neg_out[j].float()
713
+
714
+ pred = pos + current_guidance_scale * (pos - neg)
715
+
716
+ # Renormalization
717
+ if self._cfg_normalization and float(self._cfg_normalization) > 0.0:
718
+ ori_pos_norm = torch.linalg.vector_norm(pos)
719
+ new_pos_norm = torch.linalg.vector_norm(pred)
720
+ max_new_norm = ori_pos_norm * float(self._cfg_normalization)
721
+ if new_pos_norm > max_new_norm:
722
+ pred = pred * (max_new_norm / new_pos_norm)
723
+
724
+ noise_pred.append(pred)
725
+
726
+ noise_pred = torch.stack(noise_pred, dim=0)
727
+ else:
728
+ noise_pred = torch.stack([t.float() for t in model_out_list], dim=0)
729
+
730
+ noise_pred = noise_pred.squeeze(2)
731
+ noise_pred = -noise_pred
732
+
733
+ # compute the previous noisy sample x_t -> x_t-1
734
+ latents = self.scheduler.step(noise_pred.to(torch.float32), t, latents, return_dict=False)[0]
735
+ assert latents.dtype == torch.float32
736
+
737
+ if callback_on_step_end is not None:
738
+ callback_kwargs = {}
739
+ for k in callback_on_step_end_tensor_inputs:
740
+ callback_kwargs[k] = locals()[k]
741
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
742
+
743
+ latents = callback_outputs.pop("latents", latents)
744
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
745
+ negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
746
+
747
+ # call the callback, if provided
748
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
749
+ progress_bar.update()
750
+
751
+ if output_type == "latent":
752
+ image = latents
753
+
754
+ else:
755
+ latents = latents.to(self.vae.dtype)
756
+ latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor
757
+
758
+ image = self.vae.decode(latents, return_dict=False)[0]
759
+ image = self.image_processor.postprocess(image, output_type=output_type)
760
+
761
+ # Offload all models
762
+ self.maybe_free_model_hooks()
763
+
764
+ if not return_dict:
765
+ return (image,)
766
+
767
+ return ZImagePipelineOutput(images=image)
western_ai/europe/diffusers/src/diffusers/pipelines/z_image/pipeline_z_image_img2img.py ADDED
@@ -0,0 +1,715 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 Alibaba Z-Image Team and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from typing import Any, Callable
17
+
18
+ import torch
19
+ from transformers import AutoTokenizer, PreTrainedModel
20
+
21
+ from ...image_processor import PipelineImageInput, VaeImageProcessor
22
+ from ...loaders import FromSingleFileMixin, ZImageLoraLoaderMixin
23
+ from ...models.autoencoders import AutoencoderKL
24
+ from ...models.transformers import ZImageTransformer2DModel
25
+ from ...pipelines.pipeline_utils import DiffusionPipeline
26
+ from ...schedulers import FlowMatchEulerDiscreteScheduler
27
+ from ...utils import logging, replace_example_docstring
28
+ from ...utils.torch_utils import randn_tensor
29
+ from .pipeline_output import ZImagePipelineOutput
30
+
31
+
32
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
33
+
34
+ EXAMPLE_DOC_STRING = """
35
+ Examples:
36
+ ```py
37
+ >>> import torch
38
+ >>> from diffusers import ZImageImg2ImgPipeline
39
+ >>> from diffusers.utils import load_image
40
+
41
+ >>> pipe = ZImageImg2ImgPipeline.from_pretrained("Z-a-o/Z-Image-Turbo", torch_dtype=torch.bfloat16)
42
+ >>> pipe.to("cuda")
43
+
44
+ >>> url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg"
45
+ >>> init_image = load_image(url).resize((1024, 1024))
46
+
47
+ >>> prompt = "A fantasy landscape with mountains and a river, detailed, vibrant colors"
48
+ >>> image = pipe(
49
+ ... prompt,
50
+ ... image=init_image,
51
+ ... strength=0.6,
52
+ ... num_inference_steps=8,
53
+ ... guidance_scale=0.0,
54
+ ... generator=torch.Generator("cuda").manual_seed(42),
55
+ ... ).images[0]
56
+ >>> image.save("zimage_img2img.png")
57
+ ```
58
+ """
59
+
60
+
61
+ # Copied from diffusers.pipelines.flux.pipeline_flux.calculate_shift
62
+ def calculate_shift(
63
+ image_seq_len,
64
+ base_seq_len: int = 256,
65
+ max_seq_len: int = 4096,
66
+ base_shift: float = 0.5,
67
+ max_shift: float = 1.15,
68
+ ):
69
+ m = (max_shift - base_shift) / (max_seq_len - base_seq_len)
70
+ b = base_shift - m * base_seq_len
71
+ mu = image_seq_len * m + b
72
+ return mu
73
+
74
+
75
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
76
+ def retrieve_latents(
77
+ encoder_output: torch.Tensor, generator: torch.Generator | None = None, sample_mode: str = "sample"
78
+ ):
79
+ if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
80
+ return encoder_output.latent_dist.sample(generator)
81
+ elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
82
+ return encoder_output.latent_dist.mode()
83
+ elif hasattr(encoder_output, "latents"):
84
+ return encoder_output.latents
85
+ else:
86
+ raise AttributeError("Could not access latents of provided encoder_output")
87
+
88
+
89
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
90
+ def retrieve_timesteps(
91
+ scheduler,
92
+ num_inference_steps: int | None = None,
93
+ device: str | torch.device | None = None,
94
+ timesteps: list[int] | None = None,
95
+ sigmas: list[float] | None = None,
96
+ **kwargs,
97
+ ):
98
+ r"""
99
+ Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
100
+ custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
101
+
102
+ Args:
103
+ scheduler (`SchedulerMixin`):
104
+ The scheduler to get timesteps from.
105
+ num_inference_steps (`int`):
106
+ The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
107
+ must be `None`.
108
+ device (`str` or `torch.device`, *optional*):
109
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
110
+ timesteps (`list[int]`, *optional*):
111
+ Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
112
+ `num_inference_steps` and `sigmas` must be `None`.
113
+ sigmas (`list[float]`, *optional*):
114
+ Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
115
+ `num_inference_steps` and `timesteps` must be `None`.
116
+
117
+ Returns:
118
+ `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
119
+ second element is the number of inference steps.
120
+ """
121
+ if timesteps is not None and sigmas is not None:
122
+ raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
123
+ if timesteps is not None:
124
+ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
125
+ if not accepts_timesteps:
126
+ raise ValueError(
127
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
128
+ f" timestep schedules. Please check whether you are using the correct scheduler."
129
+ )
130
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
131
+ timesteps = scheduler.timesteps
132
+ num_inference_steps = len(timesteps)
133
+ elif sigmas is not None:
134
+ accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
135
+ if not accept_sigmas:
136
+ raise ValueError(
137
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
138
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
139
+ )
140
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
141
+ timesteps = scheduler.timesteps
142
+ num_inference_steps = len(timesteps)
143
+ else:
144
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
145
+ timesteps = scheduler.timesteps
146
+ return timesteps, num_inference_steps
147
+
148
+
149
+ # Copied from diffusers.pipelines.z_image.pipeline_z_image.get_default_z_image_sigmas
150
+ def get_default_z_image_sigmas(num_inference_steps: int) -> list[float]:
151
+ return torch.linspace(1.0, 1 / num_inference_steps, num_inference_steps).tolist()
152
+
153
+
154
+ class ZImageImg2ImgPipeline(DiffusionPipeline, ZImageLoraLoaderMixin, FromSingleFileMixin):
155
+ r"""
156
+ The ZImage pipeline for image-to-image generation.
157
+
158
+ Args:
159
+ scheduler ([`FlowMatchEulerDiscreteScheduler`]):
160
+ A scheduler to be used in combination with `transformer` to denoise the encoded image latents.
161
+ vae ([`AutoencoderKL`]):
162
+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
163
+ text_encoder ([`PreTrainedModel`]):
164
+ A text encoder model to encode text prompts.
165
+ tokenizer ([`AutoTokenizer`]):
166
+ A tokenizer to tokenize text prompts.
167
+ transformer ([`ZImageTransformer2DModel`]):
168
+ A ZImage transformer model to denoise the encoded image latents.
169
+ """
170
+
171
+ model_cpu_offload_seq = "text_encoder->transformer->vae"
172
+ _optional_components = []
173
+ _callback_tensor_inputs = ["latents", "prompt_embeds"]
174
+
175
+ def __init__(
176
+ self,
177
+ scheduler: FlowMatchEulerDiscreteScheduler,
178
+ vae: AutoencoderKL,
179
+ text_encoder: PreTrainedModel,
180
+ tokenizer: AutoTokenizer,
181
+ transformer: ZImageTransformer2DModel,
182
+ ):
183
+ super().__init__()
184
+
185
+ self.register_modules(
186
+ vae=vae,
187
+ text_encoder=text_encoder,
188
+ tokenizer=tokenizer,
189
+ scheduler=scheduler,
190
+ transformer=transformer,
191
+ )
192
+ self.vae_scale_factor = (
193
+ 2 ** (len(self.vae.config.block_out_channels) - 1) if hasattr(self, "vae") and self.vae is not None else 8
194
+ )
195
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2)
196
+
197
+ # Copied from diffusers.pipelines.z_image.pipeline_z_image.ZImagePipeline.encode_prompt
198
+ def encode_prompt(
199
+ self,
200
+ prompt: str | list[str],
201
+ device: torch.device | None = None,
202
+ do_classifier_free_guidance: bool = True,
203
+ negative_prompt: str | list[str] | None = None,
204
+ prompt_embeds: list[torch.FloatTensor] | None = None,
205
+ negative_prompt_embeds: torch.FloatTensor | None = None,
206
+ max_sequence_length: int = 512,
207
+ ):
208
+ prompt = [prompt] if isinstance(prompt, str) else prompt
209
+ prompt_embeds = self._encode_prompt(
210
+ prompt=prompt,
211
+ device=device,
212
+ prompt_embeds=prompt_embeds,
213
+ max_sequence_length=max_sequence_length,
214
+ )
215
+
216
+ if do_classifier_free_guidance:
217
+ if negative_prompt is None:
218
+ negative_prompt = ["" for _ in prompt]
219
+ else:
220
+ negative_prompt = [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
221
+ assert len(prompt) == len(negative_prompt)
222
+ negative_prompt_embeds = self._encode_prompt(
223
+ prompt=negative_prompt,
224
+ device=device,
225
+ prompt_embeds=negative_prompt_embeds,
226
+ max_sequence_length=max_sequence_length,
227
+ )
228
+ else:
229
+ negative_prompt_embeds = []
230
+ return prompt_embeds, negative_prompt_embeds
231
+
232
+ # Copied from diffusers.pipelines.z_image.pipeline_z_image.ZImagePipeline._encode_prompt
233
+ def _encode_prompt(
234
+ self,
235
+ prompt: str | list[str],
236
+ device: torch.device | None = None,
237
+ prompt_embeds: list[torch.FloatTensor] | None = None,
238
+ max_sequence_length: int = 512,
239
+ ) -> list[torch.FloatTensor]:
240
+ device = device or self._execution_device
241
+
242
+ if prompt_embeds is not None:
243
+ return prompt_embeds
244
+
245
+ if isinstance(prompt, str):
246
+ prompt = [prompt]
247
+
248
+ for i, prompt_item in enumerate(prompt):
249
+ messages = [
250
+ {"role": "user", "content": prompt_item},
251
+ ]
252
+ prompt_item = self.tokenizer.apply_chat_template(
253
+ messages,
254
+ tokenize=False,
255
+ add_generation_prompt=True,
256
+ enable_thinking=True,
257
+ )
258
+ prompt[i] = prompt_item
259
+
260
+ text_inputs = self.tokenizer(
261
+ prompt,
262
+ padding="max_length",
263
+ max_length=max_sequence_length,
264
+ truncation=True,
265
+ return_tensors="pt",
266
+ )
267
+
268
+ text_input_ids = text_inputs.input_ids.to(device)
269
+ prompt_masks = text_inputs.attention_mask.to(device).bool()
270
+
271
+ prompt_embeds = self.text_encoder(
272
+ input_ids=text_input_ids,
273
+ attention_mask=prompt_masks,
274
+ output_hidden_states=True,
275
+ ).hidden_states[-2]
276
+
277
+ embeddings_list = []
278
+
279
+ for i in range(len(prompt_embeds)):
280
+ embeddings_list.append(prompt_embeds[i][prompt_masks[i]])
281
+
282
+ return embeddings_list
283
+
284
+ # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3_img2img.StableDiffusion3Img2ImgPipeline.get_timesteps
285
+ def get_timesteps(self, num_inference_steps, strength, device):
286
+ # get the original timestep using init_timestep
287
+ init_timestep = min(num_inference_steps * strength, num_inference_steps)
288
+
289
+ t_start = int(max(num_inference_steps - init_timestep, 0))
290
+ timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :]
291
+ if hasattr(self.scheduler, "set_begin_index"):
292
+ self.scheduler.set_begin_index(t_start * self.scheduler.order)
293
+
294
+ return timesteps, num_inference_steps - t_start
295
+
296
+ def prepare_latents(
297
+ self,
298
+ image,
299
+ timestep,
300
+ batch_size,
301
+ num_channels_latents,
302
+ height,
303
+ width,
304
+ dtype,
305
+ device,
306
+ generator,
307
+ latents=None,
308
+ ):
309
+ height = 2 * (int(height) // (self.vae_scale_factor * 2))
310
+ width = 2 * (int(width) // (self.vae_scale_factor * 2))
311
+
312
+ shape = (batch_size, num_channels_latents, height, width)
313
+
314
+ if latents is not None:
315
+ return latents.to(device=device, dtype=dtype)
316
+
317
+ # Encode the input image
318
+ image = image.to(device=device, dtype=dtype)
319
+ if image.shape[1] != num_channels_latents:
320
+ if isinstance(generator, list):
321
+ image_latents = [
322
+ retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i])
323
+ for i in range(image.shape[0])
324
+ ]
325
+ image_latents = torch.cat(image_latents, dim=0)
326
+ else:
327
+ image_latents = retrieve_latents(self.vae.encode(image), generator=generator)
328
+
329
+ # Apply scaling (inverse of decoding: decode does latents/scaling_factor + shift_factor)
330
+ image_latents = (image_latents - self.vae.config.shift_factor) * self.vae.config.scaling_factor
331
+ else:
332
+ image_latents = image
333
+
334
+ # Handle batch size expansion
335
+ if batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] == 0:
336
+ additional_image_per_prompt = batch_size // image_latents.shape[0]
337
+ image_latents = torch.cat([image_latents] * additional_image_per_prompt, dim=0)
338
+ elif batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] != 0:
339
+ raise ValueError(
340
+ f"Cannot duplicate `image` of batch size {image_latents.shape[0]} to {batch_size} text prompts."
341
+ )
342
+
343
+ # Add noise using flow matching scale_noise
344
+ noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
345
+ latents = self.scheduler.scale_noise(image_latents, timestep, noise)
346
+
347
+ return latents
348
+
349
+ @property
350
+ def guidance_scale(self):
351
+ return self._guidance_scale
352
+
353
+ @property
354
+ def do_classifier_free_guidance(self):
355
+ return self._guidance_scale > 0
356
+
357
+ @property
358
+ def joint_attention_kwargs(self):
359
+ return self._joint_attention_kwargs
360
+
361
+ @property
362
+ def num_timesteps(self):
363
+ return self._num_timesteps
364
+
365
+ @property
366
+ def interrupt(self):
367
+ return self._interrupt
368
+
369
+ @torch.no_grad()
370
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
371
+ def __call__(
372
+ self,
373
+ prompt: str | list[str] = None,
374
+ image: PipelineImageInput = None,
375
+ strength: float = 0.6,
376
+ height: int | None = None,
377
+ width: int | None = None,
378
+ num_inference_steps: int = 50,
379
+ sigmas: list[float] | None = None,
380
+ guidance_scale: float = 5.0,
381
+ cfg_normalization: bool = False,
382
+ cfg_truncation: float = 1.0,
383
+ negative_prompt: str | list[str] | None = None,
384
+ num_images_per_prompt: int | None = 1,
385
+ generator: torch.Generator | list[torch.Generator] | None = None,
386
+ latents: torch.FloatTensor | None = None,
387
+ prompt_embeds: list[torch.FloatTensor] | None = None,
388
+ negative_prompt_embeds: list[torch.FloatTensor] | None = None,
389
+ output_type: str | None = "pil",
390
+ return_dict: bool = True,
391
+ joint_attention_kwargs: dict[str, Any] | None = None,
392
+ callback_on_step_end: Callable[[int, int], None] | None = None,
393
+ callback_on_step_end_tensor_inputs: list[str] = ["latents"],
394
+ max_sequence_length: int = 512,
395
+ ):
396
+ r"""
397
+ Function invoked when calling the pipeline for image-to-image generation.
398
+
399
+ Args:
400
+ prompt (`str` or `list[str]`, *optional*):
401
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
402
+ instead.
403
+ image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`):
404
+ `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both
405
+ numpy array and pytorch tensor, the expected value range is between `[0, 1]`. If it's a tensor or a
406
+ list of tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or
407
+ a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)`.
408
+ strength (`float`, *optional*, defaults to 0.6):
409
+ Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a
410
+ starting point and more noise is added the higher the `strength`. The number of denoising steps depends
411
+ on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising
412
+ process runs for the full number of iterations specified in `num_inference_steps`. A value of 1
413
+ essentially ignores `image`.
414
+ height (`int`, *optional*, defaults to 1024):
415
+ The height in pixels of the generated image. If not provided, uses the input image height.
416
+ width (`int`, *optional*, defaults to 1024):
417
+ The width in pixels of the generated image. If not provided, uses the input image width.
418
+ num_inference_steps (`int`, *optional*, defaults to 50):
419
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
420
+ expense of slower inference.
421
+ sigmas (`list[float]`, *optional*):
422
+ Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
423
+ their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
424
+ will be used.
425
+ guidance_scale (`float`, *optional*, defaults to 5.0):
426
+ Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
427
+ `guidance_scale` is defined as `w` of equation 2. of [Imagen
428
+ Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
429
+ 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
430
+ usually at the expense of lower image quality.
431
+ cfg_normalization (`bool`, *optional*, defaults to False):
432
+ Whether to apply configuration normalization.
433
+ cfg_truncation (`float`, *optional*, defaults to 1.0):
434
+ The truncation value for configuration.
435
+ negative_prompt (`str` or `list[str]`, *optional*):
436
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
437
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
438
+ less than `1`).
439
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
440
+ The number of images to generate per prompt.
441
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
442
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
443
+ to make generation deterministic.
444
+ latents (`torch.FloatTensor`, *optional*):
445
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
446
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
447
+ tensor will be generated by sampling using the supplied random `generator`.
448
+ prompt_embeds (`list[torch.FloatTensor]`, *optional*):
449
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
450
+ provided, text embeddings will be generated from `prompt` input argument.
451
+ negative_prompt_embeds (`list[torch.FloatTensor]`, *optional*):
452
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
453
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
454
+ argument.
455
+ output_type (`str`, *optional*, defaults to `"pil"`):
456
+ The output format of the generate image. Choose between
457
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
458
+ return_dict (`bool`, *optional*, defaults to `True`):
459
+ Whether or not to return a [`~pipelines.stable_diffusion.ZImagePipelineOutput`] instead of a plain
460
+ tuple.
461
+ joint_attention_kwargs (`dict`, *optional*):
462
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
463
+ `self.processor` in
464
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
465
+ callback_on_step_end (`Callable`, *optional*):
466
+ A function that calls at the end of each denoising steps during the inference. The function is called
467
+ with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
468
+ callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
469
+ `callback_on_step_end_tensor_inputs`.
470
+ callback_on_step_end_tensor_inputs (`List`, *optional*):
471
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
472
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
473
+ `._callback_tensor_inputs` attribute of your pipeline class.
474
+ max_sequence_length (`int`, *optional*, defaults to 512):
475
+ Maximum sequence length to use with the `prompt`.
476
+
477
+ Examples:
478
+
479
+ Returns:
480
+ [`~pipelines.z_image.ZImagePipelineOutput`] or `tuple`: [`~pipelines.z_image.ZImagePipelineOutput`] if
481
+ `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the
482
+ generated images.
483
+ """
484
+ # 1. Check inputs and validate strength
485
+ if strength < 0 or strength > 1:
486
+ raise ValueError(f"The value of strength should be in [0.0, 1.0] but is {strength}")
487
+
488
+ # 2. Preprocess image
489
+ init_image = self.image_processor.preprocess(image)
490
+ init_image = init_image.to(dtype=torch.float32)
491
+
492
+ # Get dimensions from the preprocessed image if not specified
493
+ if height is None:
494
+ height = init_image.shape[-2]
495
+ if width is None:
496
+ width = init_image.shape[-1]
497
+
498
+ vae_scale = self.vae_scale_factor * 2
499
+ if height % vae_scale != 0:
500
+ raise ValueError(
501
+ f"Height must be divisible by {vae_scale} (got {height}). "
502
+ f"Please adjust the height to a multiple of {vae_scale}."
503
+ )
504
+ if width % vae_scale != 0:
505
+ raise ValueError(
506
+ f"Width must be divisible by {vae_scale} (got {width}). "
507
+ f"Please adjust the width to a multiple of {vae_scale}."
508
+ )
509
+
510
+ device = self._execution_device
511
+
512
+ self._guidance_scale = guidance_scale
513
+ self._joint_attention_kwargs = joint_attention_kwargs
514
+ self._interrupt = False
515
+ self._cfg_normalization = cfg_normalization
516
+ self._cfg_truncation = cfg_truncation
517
+
518
+ # 3. Define call parameters
519
+ if prompt is not None and isinstance(prompt, str):
520
+ batch_size = 1
521
+ elif prompt is not None and isinstance(prompt, list):
522
+ batch_size = len(prompt)
523
+ else:
524
+ batch_size = len(prompt_embeds)
525
+
526
+ # If prompt_embeds is provided and prompt is None, skip encoding
527
+ if prompt_embeds is not None and prompt is None:
528
+ if self.do_classifier_free_guidance and negative_prompt_embeds is None:
529
+ raise ValueError(
530
+ "When `prompt_embeds` is provided without `prompt`, "
531
+ "`negative_prompt_embeds` must also be provided for classifier-free guidance."
532
+ )
533
+ else:
534
+ (
535
+ prompt_embeds,
536
+ negative_prompt_embeds,
537
+ ) = self.encode_prompt(
538
+ prompt=prompt,
539
+ negative_prompt=negative_prompt,
540
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
541
+ prompt_embeds=prompt_embeds,
542
+ negative_prompt_embeds=negative_prompt_embeds,
543
+ device=device,
544
+ max_sequence_length=max_sequence_length,
545
+ )
546
+
547
+ # 4. Prepare latent variables
548
+ num_channels_latents = self.transformer.in_channels
549
+
550
+ # Repeat prompt_embeds for num_images_per_prompt
551
+ if num_images_per_prompt > 1:
552
+ prompt_embeds = [pe for pe in prompt_embeds for _ in range(num_images_per_prompt)]
553
+ if self.do_classifier_free_guidance and negative_prompt_embeds:
554
+ negative_prompt_embeds = [npe for npe in negative_prompt_embeds for _ in range(num_images_per_prompt)]
555
+
556
+ actual_batch_size = batch_size * num_images_per_prompt
557
+
558
+ # Calculate latent dimensions for image_seq_len
559
+ latent_height = 2 * (int(height) // (self.vae_scale_factor * 2))
560
+ latent_width = 2 * (int(width) // (self.vae_scale_factor * 2))
561
+ image_seq_len = (latent_height // 2) * (latent_width // 2)
562
+
563
+ # 5. Prepare timesteps
564
+ mu = calculate_shift(
565
+ image_seq_len,
566
+ self.scheduler.config.get("base_image_seq_len", 256),
567
+ self.scheduler.config.get("max_image_seq_len", 4096),
568
+ self.scheduler.config.get("base_shift", 0.5),
569
+ self.scheduler.config.get("max_shift", 1.15),
570
+ )
571
+ if sigmas is None:
572
+ sigmas = get_default_z_image_sigmas(num_inference_steps)
573
+ scheduler_kwargs = {"mu": mu}
574
+ timesteps, num_inference_steps = retrieve_timesteps(
575
+ self.scheduler,
576
+ num_inference_steps,
577
+ device,
578
+ sigmas=sigmas,
579
+ **scheduler_kwargs,
580
+ )
581
+
582
+ # 6. Adjust timesteps based on strength
583
+ timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
584
+ if num_inference_steps < 1:
585
+ raise ValueError(
586
+ f"After adjusting the num_inference_steps by strength parameter: {strength}, the number of pipeline "
587
+ f"steps is {num_inference_steps} which is < 1 and not appropriate for this pipeline."
588
+ )
589
+ latent_timestep = timesteps[:1].repeat(actual_batch_size)
590
+
591
+ # 7. Prepare latents from image
592
+ latents = self.prepare_latents(
593
+ init_image,
594
+ latent_timestep,
595
+ actual_batch_size,
596
+ num_channels_latents,
597
+ height,
598
+ width,
599
+ prompt_embeds[0].dtype,
600
+ device,
601
+ generator,
602
+ latents,
603
+ )
604
+
605
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
606
+ self._num_timesteps = len(timesteps)
607
+
608
+ # 8. Denoising loop
609
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
610
+ for i, t in enumerate(timesteps):
611
+ if self.interrupt:
612
+ continue
613
+
614
+ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
615
+ timestep = t.expand(latents.shape[0])
616
+ timestep = (1000 - timestep) / 1000
617
+ # Normalized time for time-aware config (0 at start, 1 at end)
618
+ t_norm = timestep[0].item()
619
+
620
+ # Handle cfg truncation
621
+ current_guidance_scale = self.guidance_scale
622
+ if (
623
+ self.do_classifier_free_guidance
624
+ and self._cfg_truncation is not None
625
+ and float(self._cfg_truncation) <= 1
626
+ ):
627
+ if t_norm > self._cfg_truncation:
628
+ current_guidance_scale = 0.0
629
+
630
+ # Run CFG only if configured AND scale is non-zero
631
+ apply_cfg = self.do_classifier_free_guidance and current_guidance_scale > 0
632
+
633
+ if apply_cfg:
634
+ latents_typed = latents.to(self.transformer.dtype)
635
+ latent_model_input = latents_typed.repeat(2, 1, 1, 1)
636
+ prompt_embeds_model_input = prompt_embeds + negative_prompt_embeds
637
+ timestep_model_input = timestep.repeat(2)
638
+ else:
639
+ latent_model_input = latents.to(self.transformer.dtype)
640
+ prompt_embeds_model_input = prompt_embeds
641
+ timestep_model_input = timestep
642
+
643
+ latent_model_input = latent_model_input.unsqueeze(2)
644
+ latent_model_input_list = list(latent_model_input.unbind(dim=0))
645
+
646
+ model_out_list = self.transformer(
647
+ latent_model_input_list,
648
+ timestep_model_input,
649
+ prompt_embeds_model_input,
650
+ )[0]
651
+
652
+ if apply_cfg:
653
+ # Perform CFG
654
+ pos_out = model_out_list[:actual_batch_size]
655
+ neg_out = model_out_list[actual_batch_size:]
656
+
657
+ noise_pred = []
658
+ for j in range(actual_batch_size):
659
+ pos = pos_out[j].float()
660
+ neg = neg_out[j].float()
661
+
662
+ pred = pos + current_guidance_scale * (pos - neg)
663
+
664
+ # Renormalization
665
+ if self._cfg_normalization and float(self._cfg_normalization) > 0.0:
666
+ ori_pos_norm = torch.linalg.vector_norm(pos)
667
+ new_pos_norm = torch.linalg.vector_norm(pred)
668
+ max_new_norm = ori_pos_norm * float(self._cfg_normalization)
669
+ if new_pos_norm > max_new_norm:
670
+ pred = pred * (max_new_norm / new_pos_norm)
671
+
672
+ noise_pred.append(pred)
673
+
674
+ noise_pred = torch.stack(noise_pred, dim=0)
675
+ else:
676
+ noise_pred = torch.stack([t.float() for t in model_out_list], dim=0)
677
+
678
+ noise_pred = noise_pred.squeeze(2)
679
+ noise_pred = -noise_pred
680
+
681
+ # compute the previous noisy sample x_t -> x_t-1
682
+ latents = self.scheduler.step(noise_pred.to(torch.float32), t, latents, return_dict=False)[0]
683
+ assert latents.dtype == torch.float32
684
+
685
+ if callback_on_step_end is not None:
686
+ callback_kwargs = {}
687
+ for k in callback_on_step_end_tensor_inputs:
688
+ callback_kwargs[k] = locals()[k]
689
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
690
+
691
+ latents = callback_outputs.pop("latents", latents)
692
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
693
+ negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
694
+
695
+ # call the callback, if provided
696
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
697
+ progress_bar.update()
698
+
699
+ if output_type == "latent":
700
+ image = latents
701
+
702
+ else:
703
+ latents = latents.to(self.vae.dtype)
704
+ latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor
705
+
706
+ image = self.vae.decode(latents, return_dict=False)[0]
707
+ image = self.image_processor.postprocess(image, output_type=output_type)
708
+
709
+ # Offload all models
710
+ self.maybe_free_model_hooks()
711
+
712
+ if not return_dict:
713
+ return (image,)
714
+
715
+ return ZImagePipelineOutput(images=image)
western_ai/europe/diffusers/src/diffusers/pipelines/z_image/pipeline_z_image_inpaint.py ADDED
@@ -0,0 +1,938 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 Alibaba Z-Image Team and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from typing import Any, Callable, Dict, List, Optional, Union
17
+
18
+ import torch
19
+ from transformers import AutoTokenizer, PreTrainedModel
20
+
21
+ from ...image_processor import PipelineImageInput, VaeImageProcessor
22
+ from ...loaders import FromSingleFileMixin, ZImageLoraLoaderMixin
23
+ from ...models.autoencoders import AutoencoderKL
24
+ from ...models.transformers import ZImageTransformer2DModel
25
+ from ...pipelines.pipeline_utils import DiffusionPipeline
26
+ from ...schedulers import FlowMatchEulerDiscreteScheduler
27
+ from ...utils import is_torch_xla_available, logging, replace_example_docstring
28
+ from ...utils.torch_utils import randn_tensor
29
+ from .pipeline_output import ZImagePipelineOutput
30
+
31
+
32
+ if is_torch_xla_available():
33
+ import torch_xla.core.xla_model as xm
34
+
35
+ XLA_AVAILABLE = True
36
+ else:
37
+ XLA_AVAILABLE = False
38
+
39
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
40
+
41
+ EXAMPLE_DOC_STRING = """
42
+ Examples:
43
+ ```py
44
+ >>> import torch
45
+ >>> from diffusers import ZImageInpaintPipeline
46
+ >>> from diffusers.utils import load_image
47
+
48
+ >>> pipe = ZImageInpaintPipeline.from_pretrained("Tongyi-MAI/Z-Image-Turbo", torch_dtype=torch.bfloat16)
49
+ >>> pipe.to("cuda")
50
+
51
+ >>> url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg"
52
+ >>> init_image = load_image(url).resize((1024, 1024))
53
+
54
+ >>> # Create a mask (white = inpaint, black = preserve)
55
+ >>> import numpy as np
56
+ >>> from PIL import Image
57
+
58
+ >>> mask = np.zeros((1024, 1024), dtype=np.uint8)
59
+ >>> mask[256:768, 256:768] = 255 # Inpaint center region
60
+ >>> mask_image = Image.fromarray(mask)
61
+
62
+ >>> prompt = "A beautiful lake with mountains in the background"
63
+ >>> image = pipe(
64
+ ... prompt,
65
+ ... image=init_image,
66
+ ... mask_image=mask_image,
67
+ ... strength=1.0,
68
+ ... num_inference_steps=8,
69
+ ... guidance_scale=0.0,
70
+ ... generator=torch.Generator("cuda").manual_seed(42),
71
+ ... ).images[0]
72
+ >>> image.save("zimage_inpaint.png")
73
+ ```
74
+ """
75
+
76
+
77
+ # Copied from diffusers.pipelines.flux.pipeline_flux.calculate_shift
78
+ def calculate_shift(
79
+ image_seq_len,
80
+ base_seq_len: int = 256,
81
+ max_seq_len: int = 4096,
82
+ base_shift: float = 0.5,
83
+ max_shift: float = 1.15,
84
+ ):
85
+ m = (max_shift - base_shift) / (max_seq_len - base_seq_len)
86
+ b = base_shift - m * base_seq_len
87
+ mu = image_seq_len * m + b
88
+ return mu
89
+
90
+
91
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
92
+ def retrieve_latents(
93
+ encoder_output: torch.Tensor, generator: torch.Generator | None = None, sample_mode: str = "sample"
94
+ ):
95
+ if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
96
+ return encoder_output.latent_dist.sample(generator)
97
+ elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
98
+ return encoder_output.latent_dist.mode()
99
+ elif hasattr(encoder_output, "latents"):
100
+ return encoder_output.latents
101
+ else:
102
+ raise AttributeError("Could not access latents of provided encoder_output")
103
+
104
+
105
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
106
+ def retrieve_timesteps(
107
+ scheduler,
108
+ num_inference_steps: int | None = None,
109
+ device: str | torch.device | None = None,
110
+ timesteps: list[int] | None = None,
111
+ sigmas: list[float] | None = None,
112
+ **kwargs,
113
+ ):
114
+ r"""
115
+ Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
116
+ custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
117
+
118
+ Args:
119
+ scheduler (`SchedulerMixin`):
120
+ The scheduler to get timesteps from.
121
+ num_inference_steps (`int`):
122
+ The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
123
+ must be `None`.
124
+ device (`str` or `torch.device`, *optional*):
125
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
126
+ timesteps (`list[int]`, *optional*):
127
+ Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
128
+ `num_inference_steps` and `sigmas` must be `None`.
129
+ sigmas (`list[float]`, *optional*):
130
+ Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
131
+ `num_inference_steps` and `timesteps` must be `None`.
132
+
133
+ Returns:
134
+ `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
135
+ second element is the number of inference steps.
136
+ """
137
+ if timesteps is not None and sigmas is not None:
138
+ raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
139
+ if timesteps is not None:
140
+ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
141
+ if not accepts_timesteps:
142
+ raise ValueError(
143
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
144
+ f" timestep schedules. Please check whether you are using the correct scheduler."
145
+ )
146
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
147
+ timesteps = scheduler.timesteps
148
+ num_inference_steps = len(timesteps)
149
+ elif sigmas is not None:
150
+ accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
151
+ if not accept_sigmas:
152
+ raise ValueError(
153
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
154
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
155
+ )
156
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
157
+ timesteps = scheduler.timesteps
158
+ num_inference_steps = len(timesteps)
159
+ else:
160
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
161
+ timesteps = scheduler.timesteps
162
+ return timesteps, num_inference_steps
163
+
164
+
165
+ # Copied from diffusers.pipelines.z_image.pipeline_z_image.get_default_z_image_sigmas
166
+ def get_default_z_image_sigmas(num_inference_steps: int) -> list[float]:
167
+ return torch.linspace(1.0, 1 / num_inference_steps, num_inference_steps).tolist()
168
+
169
+
170
+ class ZImageInpaintPipeline(DiffusionPipeline, ZImageLoraLoaderMixin, FromSingleFileMixin):
171
+ r"""
172
+ The ZImage pipeline for inpainting.
173
+
174
+ Args:
175
+ scheduler ([`FlowMatchEulerDiscreteScheduler`]):
176
+ A scheduler to be used in combination with `transformer` to denoise the encoded image latents.
177
+ vae ([`AutoencoderKL`]):
178
+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
179
+ text_encoder ([`PreTrainedModel`]):
180
+ A text encoder model to encode text prompts.
181
+ tokenizer ([`AutoTokenizer`]):
182
+ A tokenizer to tokenize text prompts.
183
+ transformer ([`ZImageTransformer2DModel`]):
184
+ A ZImage transformer model to denoise the encoded image latents.
185
+ """
186
+
187
+ model_cpu_offload_seq = "text_encoder->transformer->vae"
188
+ _optional_components = []
189
+ _callback_tensor_inputs = ["latents", "prompt_embeds", "mask", "masked_image_latents"]
190
+
191
+ def __init__(
192
+ self,
193
+ scheduler: FlowMatchEulerDiscreteScheduler,
194
+ vae: AutoencoderKL,
195
+ text_encoder: PreTrainedModel,
196
+ tokenizer: AutoTokenizer,
197
+ transformer: ZImageTransformer2DModel,
198
+ ):
199
+ super().__init__()
200
+
201
+ self.register_modules(
202
+ vae=vae,
203
+ text_encoder=text_encoder,
204
+ tokenizer=tokenizer,
205
+ scheduler=scheduler,
206
+ transformer=transformer,
207
+ )
208
+ self.vae_scale_factor = (
209
+ 2 ** (len(self.vae.config.block_out_channels) - 1) if hasattr(self, "vae") and self.vae is not None else 8
210
+ )
211
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2)
212
+ self.mask_processor = VaeImageProcessor(
213
+ vae_scale_factor=self.vae_scale_factor * 2,
214
+ do_normalize=False,
215
+ do_binarize=True,
216
+ do_convert_grayscale=True,
217
+ )
218
+
219
+ # Copied from diffusers.pipelines.z_image.pipeline_z_image.ZImagePipeline.encode_prompt
220
+ def encode_prompt(
221
+ self,
222
+ prompt: str | list[str],
223
+ device: torch.device | None = None,
224
+ do_classifier_free_guidance: bool = True,
225
+ negative_prompt: str | list[str] | None = None,
226
+ prompt_embeds: list[torch.FloatTensor] | None = None,
227
+ negative_prompt_embeds: torch.FloatTensor | None = None,
228
+ max_sequence_length: int = 512,
229
+ ):
230
+ prompt = [prompt] if isinstance(prompt, str) else prompt
231
+ prompt_embeds = self._encode_prompt(
232
+ prompt=prompt,
233
+ device=device,
234
+ prompt_embeds=prompt_embeds,
235
+ max_sequence_length=max_sequence_length,
236
+ )
237
+
238
+ if do_classifier_free_guidance:
239
+ if negative_prompt is None:
240
+ negative_prompt = ["" for _ in prompt]
241
+ else:
242
+ negative_prompt = [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
243
+ assert len(prompt) == len(negative_prompt)
244
+ negative_prompt_embeds = self._encode_prompt(
245
+ prompt=negative_prompt,
246
+ device=device,
247
+ prompt_embeds=negative_prompt_embeds,
248
+ max_sequence_length=max_sequence_length,
249
+ )
250
+ else:
251
+ negative_prompt_embeds = []
252
+ return prompt_embeds, negative_prompt_embeds
253
+
254
+ # Copied from diffusers.pipelines.z_image.pipeline_z_image.ZImagePipeline._encode_prompt
255
+ def _encode_prompt(
256
+ self,
257
+ prompt: str | list[str],
258
+ device: torch.device | None = None,
259
+ prompt_embeds: list[torch.FloatTensor] | None = None,
260
+ max_sequence_length: int = 512,
261
+ ) -> list[torch.FloatTensor]:
262
+ device = device or self._execution_device
263
+
264
+ if prompt_embeds is not None:
265
+ return prompt_embeds
266
+
267
+ if isinstance(prompt, str):
268
+ prompt = [prompt]
269
+
270
+ for i, prompt_item in enumerate(prompt):
271
+ messages = [
272
+ {"role": "user", "content": prompt_item},
273
+ ]
274
+ prompt_item = self.tokenizer.apply_chat_template(
275
+ messages,
276
+ tokenize=False,
277
+ add_generation_prompt=True,
278
+ enable_thinking=True,
279
+ )
280
+ prompt[i] = prompt_item
281
+
282
+ text_inputs = self.tokenizer(
283
+ prompt,
284
+ padding="max_length",
285
+ max_length=max_sequence_length,
286
+ truncation=True,
287
+ return_tensors="pt",
288
+ )
289
+
290
+ text_input_ids = text_inputs.input_ids.to(device)
291
+ prompt_masks = text_inputs.attention_mask.to(device).bool()
292
+
293
+ prompt_embeds = self.text_encoder(
294
+ input_ids=text_input_ids,
295
+ attention_mask=prompt_masks,
296
+ output_hidden_states=True,
297
+ ).hidden_states[-2]
298
+
299
+ embeddings_list = []
300
+
301
+ for i in range(len(prompt_embeds)):
302
+ embeddings_list.append(prompt_embeds[i][prompt_masks[i]])
303
+
304
+ return embeddings_list
305
+
306
+ # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3_img2img.StableDiffusion3Img2ImgPipeline.get_timesteps
307
+ def get_timesteps(self, num_inference_steps, strength, device):
308
+ # get the original timestep using init_timestep
309
+ init_timestep = min(num_inference_steps * strength, num_inference_steps)
310
+
311
+ t_start = int(max(num_inference_steps - init_timestep, 0))
312
+ timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :]
313
+ if hasattr(self.scheduler, "set_begin_index"):
314
+ self.scheduler.set_begin_index(t_start * self.scheduler.order)
315
+
316
+ return timesteps, num_inference_steps - t_start
317
+
318
+ def prepare_mask_latents(
319
+ self,
320
+ mask,
321
+ masked_image,
322
+ batch_size,
323
+ height,
324
+ width,
325
+ dtype,
326
+ device,
327
+ generator,
328
+ ):
329
+ """Prepare mask and masked image latents for inpainting.
330
+
331
+ Args:
332
+ mask: Binary mask tensor where 1 = inpaint region, 0 = preserve region.
333
+ masked_image: Original image with masked regions zeroed out.
334
+ batch_size: Number of images to generate.
335
+ height: Output image height.
336
+ width: Output image width.
337
+ dtype: Data type for the tensors.
338
+ device: Device to place tensors on.
339
+ generator: Random generator for reproducibility.
340
+
341
+ Returns:
342
+ Tuple of (mask, masked_image_latents) prepared for the denoising loop.
343
+ """
344
+ # Calculate latent dimensions
345
+ latent_height = 2 * (int(height) // (self.vae_scale_factor * 2))
346
+ latent_width = 2 * (int(width) // (self.vae_scale_factor * 2))
347
+
348
+ # Resize mask to latent dimensions
349
+ mask = torch.nn.functional.interpolate(mask, size=(latent_height, latent_width), mode="nearest")
350
+ mask = mask.to(device=device, dtype=dtype)
351
+
352
+ # Encode masked image to latents
353
+ masked_image = masked_image.to(device=device, dtype=dtype)
354
+ if isinstance(generator, list):
355
+ masked_image_latents = [
356
+ retrieve_latents(self.vae.encode(masked_image[i : i + 1]), generator=generator[i])
357
+ for i in range(masked_image.shape[0])
358
+ ]
359
+ masked_image_latents = torch.cat(masked_image_latents, dim=0)
360
+ else:
361
+ masked_image_latents = retrieve_latents(self.vae.encode(masked_image), generator=generator)
362
+
363
+ # Apply VAE scaling
364
+ masked_image_latents = (masked_image_latents - self.vae.config.shift_factor) * self.vae.config.scaling_factor
365
+
366
+ # Expand for batch size
367
+ if mask.shape[0] < batch_size:
368
+ if not batch_size % mask.shape[0] == 0:
369
+ raise ValueError(
370
+ "The passed mask and the required batch size don't match. Masks are supposed to be duplicated to"
371
+ f" a total batch size of {batch_size}, but {mask.shape[0]} masks were passed. Make sure the number"
372
+ " of masks that you pass is divisible by the total requested batch size."
373
+ )
374
+ mask = mask.repeat(batch_size // mask.shape[0], 1, 1, 1)
375
+ if masked_image_latents.shape[0] < batch_size:
376
+ if not batch_size % masked_image_latents.shape[0] == 0:
377
+ raise ValueError(
378
+ "The passed images and the required batch size don't match. Images are supposed to be duplicated"
379
+ f" to a total batch size of {batch_size}, but {masked_image_latents.shape[0]} images were passed."
380
+ " Make sure the number of images that you pass is divisible by the total requested batch size."
381
+ )
382
+ masked_image_latents = masked_image_latents.repeat(batch_size // masked_image_latents.shape[0], 1, 1, 1)
383
+
384
+ return mask, masked_image_latents
385
+
386
+ def prepare_latents(
387
+ self,
388
+ image,
389
+ timestep,
390
+ batch_size,
391
+ num_channels_latents,
392
+ height,
393
+ width,
394
+ dtype,
395
+ device,
396
+ generator,
397
+ latents=None,
398
+ ):
399
+ """Prepare latents for inpainting, returning noise and image_latents for blending.
400
+
401
+ Returns:
402
+ Tuple of (latents, noise, image_latents) where:
403
+ - latents: Noised image latents for denoising
404
+ - noise: The noise tensor used for blending
405
+ - image_latents: Clean image latents for blending
406
+ """
407
+ height = 2 * (int(height) // (self.vae_scale_factor * 2))
408
+ width = 2 * (int(width) // (self.vae_scale_factor * 2))
409
+
410
+ shape = (batch_size, num_channels_latents, height, width)
411
+
412
+ if latents is not None:
413
+ # Generate noise for blending even if latents are provided
414
+ noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
415
+ # Encode image for blending
416
+ image = image.to(device=device, dtype=dtype)
417
+ if isinstance(generator, list):
418
+ image_latents = [
419
+ retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i])
420
+ for i in range(image.shape[0])
421
+ ]
422
+ image_latents = torch.cat(image_latents, dim=0)
423
+ else:
424
+ image_latents = retrieve_latents(self.vae.encode(image), generator=generator)
425
+ image_latents = (image_latents - self.vae.config.shift_factor) * self.vae.config.scaling_factor
426
+ if batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] == 0:
427
+ image_latents = torch.cat([image_latents] * (batch_size // image_latents.shape[0]), dim=0)
428
+ return latents.to(device=device, dtype=dtype), noise, image_latents
429
+
430
+ # Encode the input image
431
+ image = image.to(device=device, dtype=dtype)
432
+ if image.shape[1] != num_channels_latents:
433
+ if isinstance(generator, list):
434
+ image_latents = [
435
+ retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i])
436
+ for i in range(image.shape[0])
437
+ ]
438
+ image_latents = torch.cat(image_latents, dim=0)
439
+ else:
440
+ image_latents = retrieve_latents(self.vae.encode(image), generator=generator)
441
+
442
+ # Apply scaling (inverse of decoding: decode does latents/scaling_factor + shift_factor)
443
+ image_latents = (image_latents - self.vae.config.shift_factor) * self.vae.config.scaling_factor
444
+ else:
445
+ image_latents = image
446
+
447
+ # Handle batch size expansion
448
+ if batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] == 0:
449
+ additional_image_per_prompt = batch_size // image_latents.shape[0]
450
+ image_latents = torch.cat([image_latents] * additional_image_per_prompt, dim=0)
451
+ elif batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] != 0:
452
+ raise ValueError(
453
+ f"Cannot duplicate `image` of batch size {image_latents.shape[0]} to {batch_size} text prompts."
454
+ )
455
+
456
+ # Generate noise for both initial noising and later blending
457
+ noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
458
+
459
+ # Add noise using flow matching scale_noise
460
+ latents = self.scheduler.scale_noise(image_latents, timestep, noise)
461
+
462
+ return latents, noise, image_latents
463
+
464
+ @property
465
+ def guidance_scale(self):
466
+ return self._guidance_scale
467
+
468
+ @property
469
+ def do_classifier_free_guidance(self):
470
+ return self._guidance_scale > 0
471
+
472
+ @property
473
+ def joint_attention_kwargs(self):
474
+ return self._joint_attention_kwargs
475
+
476
+ @property
477
+ def num_timesteps(self):
478
+ return self._num_timesteps
479
+
480
+ @property
481
+ def interrupt(self):
482
+ return self._interrupt
483
+
484
+ def check_inputs(
485
+ self,
486
+ prompt,
487
+ image,
488
+ mask_image,
489
+ strength,
490
+ height,
491
+ width,
492
+ output_type,
493
+ negative_prompt=None,
494
+ prompt_embeds=None,
495
+ negative_prompt_embeds=None,
496
+ callback_on_step_end_tensor_inputs=None,
497
+ ):
498
+ if strength < 0 or strength > 1:
499
+ raise ValueError(f"The value of strength should be in [0.0, 1.0] but is {strength}")
500
+
501
+ if callback_on_step_end_tensor_inputs is not None and not all(
502
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
503
+ ):
504
+ raise ValueError(
505
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
506
+ )
507
+
508
+ if prompt is not None and prompt_embeds is not None:
509
+ raise ValueError(
510
+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
511
+ " only forward one of the two."
512
+ )
513
+ elif prompt is None and prompt_embeds is None:
514
+ raise ValueError(
515
+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
516
+ )
517
+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
518
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
519
+
520
+ if negative_prompt is not None and negative_prompt_embeds is not None:
521
+ raise ValueError(
522
+ f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
523
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
524
+ )
525
+
526
+ if image is None:
527
+ raise ValueError("`image` input cannot be undefined for inpainting.")
528
+
529
+ if mask_image is None:
530
+ raise ValueError("`mask_image` input cannot be undefined for inpainting.")
531
+
532
+ if output_type not in ["latent", "pil", "np", "pt"]:
533
+ raise ValueError(f"`output_type` must be one of 'latent', 'pil', 'np', or 'pt', but got {output_type}")
534
+
535
+ @torch.no_grad()
536
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
537
+ def __call__(
538
+ self,
539
+ prompt: Union[str, List[str]] = None,
540
+ image: PipelineImageInput = None,
541
+ mask_image: PipelineImageInput = None,
542
+ masked_image_latents: Optional[torch.FloatTensor] = None,
543
+ strength: float = 1.0,
544
+ height: int | None = None,
545
+ width: int | None = None,
546
+ num_inference_steps: int = 50,
547
+ sigmas: list[float] | None = None,
548
+ guidance_scale: float = 5.0,
549
+ cfg_normalization: bool = False,
550
+ cfg_truncation: float = 1.0,
551
+ negative_prompt: Optional[Union[str, List[str]]] = None,
552
+ num_images_per_prompt: Optional[int] = 1,
553
+ generator: torch.Generator | list[torch.Generator] | None = None,
554
+ latents: Optional[torch.FloatTensor] = None,
555
+ prompt_embeds: Optional[List[torch.FloatTensor]] = None,
556
+ negative_prompt_embeds: Optional[List[torch.FloatTensor]] = None,
557
+ output_type: str = "pil",
558
+ return_dict: bool = True,
559
+ joint_attention_kwargs: Optional[Dict[str, Any]] = None,
560
+ callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
561
+ callback_on_step_end_tensor_inputs: List[str] = ["latents"],
562
+ max_sequence_length: int = 512,
563
+ ):
564
+ r"""
565
+ Function invoked when calling the pipeline for inpainting.
566
+
567
+ Args:
568
+ prompt (`str` or `List[str]`, *optional*):
569
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
570
+ instead.
571
+ image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`):
572
+ `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both
573
+ numpy array and pytorch tensor, the expected value range is between `[0, 1]`. If it's a tensor or a
574
+ list of tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or
575
+ a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)`.
576
+ mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`):
577
+ `Image`, numpy array or tensor representing a mask image for inpainting. White pixels (value 1) in the
578
+ mask will be inpainted, black pixels (value 0) will be preserved from the original image.
579
+ masked_image_latents (`torch.FloatTensor`, *optional*):
580
+ Pre-encoded masked image latents. If provided, the masked image encoding step will be skipped.
581
+ strength (`float`, *optional*, defaults to 1.0):
582
+ Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a
583
+ starting point and more noise is added the higher the `strength`. The number of denoising steps depends
584
+ on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising
585
+ process runs for the full number of iterations specified in `num_inference_steps`. A value of 1
586
+ essentially ignores `image` in the masked region.
587
+ height (`int`, *optional*, defaults to 1024):
588
+ The height in pixels of the generated image. If not provided, uses the input image height.
589
+ width (`int`, *optional*, defaults to 1024):
590
+ The width in pixels of the generated image. If not provided, uses the input image width.
591
+ num_inference_steps (`int`, *optional*, defaults to 50):
592
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
593
+ expense of slower inference.
594
+ sigmas (`List[float]`, *optional*):
595
+ Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
596
+ their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
597
+ will be used.
598
+ guidance_scale (`float`, *optional*, defaults to 5.0):
599
+ Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
600
+ `guidance_scale` is defined as `w` of equation 2. of [Imagen
601
+ Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
602
+ 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
603
+ usually at the expense of lower image quality.
604
+ cfg_normalization (`bool`, *optional*, defaults to False):
605
+ Whether to apply configuration normalization.
606
+ cfg_truncation (`float`, *optional*, defaults to 1.0):
607
+ The truncation value for configuration.
608
+ negative_prompt (`str` or `List[str]`, *optional*):
609
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
610
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
611
+ less than `1`).
612
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
613
+ The number of images to generate per prompt.
614
+ generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
615
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
616
+ to make generation deterministic.
617
+ latents (`torch.FloatTensor`, *optional*):
618
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
619
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
620
+ tensor will be generated by sampling using the supplied random `generator`.
621
+ prompt_embeds (`List[torch.FloatTensor]`, *optional*):
622
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
623
+ provided, text embeddings will be generated from `prompt` input argument.
624
+ negative_prompt_embeds (`List[torch.FloatTensor]`, *optional*):
625
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
626
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
627
+ argument.
628
+ output_type (`str`, *optional*, defaults to `"pil"`):
629
+ The output format of the generate image. Choose between
630
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
631
+ return_dict (`bool`, *optional*, defaults to `True`):
632
+ Whether or not to return a [`~pipelines.stable_diffusion.ZImagePipelineOutput`] instead of a plain
633
+ tuple.
634
+ joint_attention_kwargs (`dict`, *optional*):
635
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
636
+ `self.processor` in
637
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
638
+ callback_on_step_end (`Callable`, *optional*):
639
+ A function that calls at the end of each denoising steps during the inference. The function is called
640
+ with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
641
+ callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
642
+ `callback_on_step_end_tensor_inputs`.
643
+ callback_on_step_end_tensor_inputs (`List`, *optional*):
644
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
645
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
646
+ `._callback_tensor_inputs` attribute of your pipeline class.
647
+ max_sequence_length (`int`, *optional*, defaults to 512):
648
+ Maximum sequence length to use with the `prompt`.
649
+
650
+ Examples:
651
+
652
+ Returns:
653
+ [`~pipelines.z_image.ZImagePipelineOutput`] or `tuple`: [`~pipelines.z_image.ZImagePipelineOutput`] if
654
+ `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the
655
+ generated images.
656
+ """
657
+ # 1. Check inputs
658
+ self.check_inputs(
659
+ prompt=prompt,
660
+ image=image,
661
+ mask_image=mask_image,
662
+ strength=strength,
663
+ height=height,
664
+ width=width,
665
+ output_type=output_type,
666
+ negative_prompt=negative_prompt,
667
+ prompt_embeds=prompt_embeds,
668
+ negative_prompt_embeds=negative_prompt_embeds,
669
+ callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs,
670
+ )
671
+
672
+ # 2. Preprocess image and mask
673
+ init_image = self.image_processor.preprocess(image)
674
+ init_image = init_image.to(dtype=torch.float32)
675
+
676
+ # Get dimensions from the preprocessed image if not specified
677
+ if height is None:
678
+ height = init_image.shape[-2]
679
+ if width is None:
680
+ width = init_image.shape[-1]
681
+
682
+ vae_scale = self.vae_scale_factor * 2
683
+ if height % vae_scale != 0:
684
+ raise ValueError(
685
+ f"Height must be divisible by {vae_scale} (got {height}). "
686
+ f"Please adjust the height to a multiple of {vae_scale}."
687
+ )
688
+ if width % vae_scale != 0:
689
+ raise ValueError(
690
+ f"Width must be divisible by {vae_scale} (got {width}). "
691
+ f"Please adjust the width to a multiple of {vae_scale}."
692
+ )
693
+
694
+ # Preprocess mask
695
+ mask = self.mask_processor.preprocess(mask_image, height=height, width=width)
696
+
697
+ device = self._execution_device
698
+
699
+ self._guidance_scale = guidance_scale
700
+ self._joint_attention_kwargs = joint_attention_kwargs
701
+ self._interrupt = False
702
+ self._cfg_normalization = cfg_normalization
703
+ self._cfg_truncation = cfg_truncation
704
+
705
+ # 3. Define call parameters
706
+ if prompt is not None and isinstance(prompt, str):
707
+ batch_size = 1
708
+ elif prompt is not None and isinstance(prompt, list):
709
+ batch_size = len(prompt)
710
+ else:
711
+ batch_size = len(prompt_embeds)
712
+
713
+ # If prompt_embeds is provided and prompt is None, skip encoding
714
+ if prompt_embeds is not None and prompt is None:
715
+ if self.do_classifier_free_guidance and negative_prompt_embeds is None:
716
+ raise ValueError(
717
+ "When `prompt_embeds` is provided without `prompt`, "
718
+ "`negative_prompt_embeds` must also be provided for classifier-free guidance."
719
+ )
720
+ else:
721
+ (
722
+ prompt_embeds,
723
+ negative_prompt_embeds,
724
+ ) = self.encode_prompt(
725
+ prompt=prompt,
726
+ negative_prompt=negative_prompt,
727
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
728
+ prompt_embeds=prompt_embeds,
729
+ negative_prompt_embeds=negative_prompt_embeds,
730
+ device=device,
731
+ max_sequence_length=max_sequence_length,
732
+ )
733
+
734
+ # 4. Prepare latent variables
735
+ num_channels_latents = self.transformer.in_channels
736
+
737
+ # Repeat prompt_embeds for num_images_per_prompt
738
+ if num_images_per_prompt > 1:
739
+ prompt_embeds = [pe for pe in prompt_embeds for _ in range(num_images_per_prompt)]
740
+ if self.do_classifier_free_guidance and negative_prompt_embeds:
741
+ negative_prompt_embeds = [npe for npe in negative_prompt_embeds for _ in range(num_images_per_prompt)]
742
+
743
+ actual_batch_size = batch_size * num_images_per_prompt
744
+
745
+ # Calculate latent dimensions for image_seq_len
746
+ latent_height = 2 * (int(height) // (self.vae_scale_factor * 2))
747
+ latent_width = 2 * (int(width) // (self.vae_scale_factor * 2))
748
+ image_seq_len = (latent_height // 2) * (latent_width // 2)
749
+
750
+ # 5. Prepare timesteps
751
+ mu = calculate_shift(
752
+ image_seq_len,
753
+ self.scheduler.config.get("base_image_seq_len", 256),
754
+ self.scheduler.config.get("max_image_seq_len", 4096),
755
+ self.scheduler.config.get("base_shift", 0.5),
756
+ self.scheduler.config.get("max_shift", 1.15),
757
+ )
758
+ if sigmas is None:
759
+ sigmas = get_default_z_image_sigmas(num_inference_steps)
760
+ scheduler_kwargs = {"mu": mu}
761
+ timesteps, num_inference_steps = retrieve_timesteps(
762
+ self.scheduler,
763
+ num_inference_steps,
764
+ device,
765
+ sigmas=sigmas,
766
+ **scheduler_kwargs,
767
+ )
768
+
769
+ # 6. Adjust timesteps based on strength
770
+ timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
771
+ if num_inference_steps < 1:
772
+ raise ValueError(
773
+ f"After adjusting the num_inference_steps by strength parameter: {strength}, the number of pipeline "
774
+ f"steps is {num_inference_steps} which is < 1 and not appropriate for this pipeline."
775
+ )
776
+ latent_timestep = timesteps[:1].repeat(actual_batch_size)
777
+
778
+ # 7. Prepare latents from image (returns noise and image_latents for blending)
779
+ latents, noise, image_latents = self.prepare_latents(
780
+ init_image,
781
+ latent_timestep,
782
+ actual_batch_size,
783
+ num_channels_latents,
784
+ height,
785
+ width,
786
+ prompt_embeds[0].dtype,
787
+ device,
788
+ generator,
789
+ latents,
790
+ )
791
+
792
+ # 8. Prepare mask and masked image latents
793
+ # Create masked image: preserve only unmasked regions (mask=0)
794
+ if masked_image_latents is None:
795
+ masked_image = init_image * (mask < 0.5)
796
+ else:
797
+ masked_image = None # Will use provided masked_image_latents
798
+
799
+ mask, masked_image_latents = self.prepare_mask_latents(
800
+ mask,
801
+ masked_image if masked_image is not None else init_image,
802
+ actual_batch_size,
803
+ height,
804
+ width,
805
+ prompt_embeds[0].dtype,
806
+ device,
807
+ generator,
808
+ )
809
+
810
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
811
+ self._num_timesteps = len(timesteps)
812
+
813
+ # 9. Denoising loop
814
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
815
+ for i, t in enumerate(timesteps):
816
+ if self.interrupt:
817
+ continue
818
+
819
+ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
820
+ timestep = t.expand(latents.shape[0])
821
+ timestep = (1000 - timestep) / 1000
822
+ # Normalized time for time-aware config (0 at start, 1 at end)
823
+ t_norm = timestep[0].item()
824
+
825
+ # Handle cfg truncation
826
+ current_guidance_scale = self.guidance_scale
827
+ if (
828
+ self.do_classifier_free_guidance
829
+ and self._cfg_truncation is not None
830
+ and float(self._cfg_truncation) <= 1
831
+ ):
832
+ if t_norm > self._cfg_truncation:
833
+ current_guidance_scale = 0.0
834
+
835
+ # Run CFG only if configured AND scale is non-zero
836
+ apply_cfg = self.do_classifier_free_guidance and current_guidance_scale > 0
837
+
838
+ if apply_cfg:
839
+ latents_typed = latents.to(self.transformer.dtype)
840
+ latent_model_input = latents_typed.repeat(2, 1, 1, 1)
841
+ prompt_embeds_model_input = prompt_embeds + negative_prompt_embeds
842
+ timestep_model_input = timestep.repeat(2)
843
+ else:
844
+ latent_model_input = latents.to(self.transformer.dtype)
845
+ prompt_embeds_model_input = prompt_embeds
846
+ timestep_model_input = timestep
847
+
848
+ latent_model_input = latent_model_input.unsqueeze(2)
849
+ latent_model_input_list = list(latent_model_input.unbind(dim=0))
850
+
851
+ model_out_list = self.transformer(
852
+ latent_model_input_list,
853
+ timestep_model_input,
854
+ prompt_embeds_model_input,
855
+ )[0]
856
+
857
+ if apply_cfg:
858
+ # Perform CFG
859
+ pos_out = model_out_list[:actual_batch_size]
860
+ neg_out = model_out_list[actual_batch_size:]
861
+
862
+ noise_pred = []
863
+ for j in range(actual_batch_size):
864
+ pos = pos_out[j].float()
865
+ neg = neg_out[j].float()
866
+
867
+ pred = pos + current_guidance_scale * (pos - neg)
868
+
869
+ # Renormalization
870
+ if self._cfg_normalization and float(self._cfg_normalization) > 0.0:
871
+ ori_pos_norm = torch.linalg.vector_norm(pos)
872
+ new_pos_norm = torch.linalg.vector_norm(pred)
873
+ max_new_norm = ori_pos_norm * float(self._cfg_normalization)
874
+ if new_pos_norm > max_new_norm:
875
+ pred = pred * (max_new_norm / new_pos_norm)
876
+
877
+ noise_pred.append(pred)
878
+
879
+ noise_pred = torch.stack(noise_pred, dim=0)
880
+ else:
881
+ noise_pred = torch.stack([t.float() for t in model_out_list], dim=0)
882
+
883
+ noise_pred = noise_pred.squeeze(2)
884
+ noise_pred = -noise_pred
885
+
886
+ # compute the previous noisy sample x_t -> x_t-1
887
+ latents = self.scheduler.step(noise_pred.to(torch.float32), t, latents, return_dict=False)[0]
888
+ assert latents.dtype == torch.float32
889
+
890
+ # Inpainting blend: combine denoised latents with original image latents
891
+ init_latents_proper = image_latents
892
+
893
+ # Re-scale original latents to current noise level for proper blending
894
+ if i < len(timesteps) - 1:
895
+ noise_timestep = timesteps[i + 1]
896
+ init_latents_proper = self.scheduler.scale_noise(
897
+ init_latents_proper, torch.tensor([noise_timestep]), noise
898
+ )
899
+
900
+ # Blend: mask=1 for inpaint region (use denoised), mask=0 for preserve region (use original)
901
+ latents = (1 - mask) * init_latents_proper + mask * latents
902
+
903
+ if callback_on_step_end is not None:
904
+ callback_kwargs = {}
905
+ for k in callback_on_step_end_tensor_inputs:
906
+ callback_kwargs[k] = locals()[k]
907
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
908
+
909
+ latents = callback_outputs.pop("latents", latents)
910
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
911
+ negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
912
+ mask = callback_outputs.pop("mask", mask)
913
+ masked_image_latents = callback_outputs.pop("masked_image_latents", masked_image_latents)
914
+
915
+ # call the callback, if provided
916
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
917
+ progress_bar.update()
918
+
919
+ if XLA_AVAILABLE:
920
+ xm.mark_step()
921
+
922
+ if output_type == "latent":
923
+ image = latents
924
+
925
+ else:
926
+ latents = latents.to(self.vae.dtype)
927
+ latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor
928
+
929
+ image = self.vae.decode(latents, return_dict=False)[0]
930
+ image = self.image_processor.postprocess(image, output_type=output_type)
931
+
932
+ # Offload all models
933
+ self.maybe_free_model_hooks()
934
+
935
+ if not return_dict:
936
+ return (image,)
937
+
938
+ return ZImagePipelineOutput(images=image)
western_ai/europe/diffusers/src/diffusers/pipelines/z_image/pipeline_z_image_omni.py ADDED
@@ -0,0 +1,749 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 Alibaba Z-Image Team and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from typing import Any, Callable
17
+
18
+ import PIL
19
+ import torch
20
+ from transformers import AutoTokenizer, PreTrainedModel, Siglip2ImageProcessorFast, Siglip2VisionModel
21
+
22
+ from ...loaders import FromSingleFileMixin, ZImageLoraLoaderMixin
23
+ from ...models.autoencoders import AutoencoderKL
24
+ from ...models.transformers import ZImageTransformer2DModel
25
+ from ...pipelines.pipeline_utils import DiffusionPipeline
26
+ from ...schedulers import FlowMatchEulerDiscreteScheduler
27
+ from ...utils import logging, replace_example_docstring
28
+ from ...utils.torch_utils import randn_tensor
29
+ from ..flux2.image_processor import Flux2ImageProcessor
30
+ from .pipeline_output import ZImagePipelineOutput
31
+
32
+
33
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
34
+
35
+ EXAMPLE_DOC_STRING = """
36
+ Examples:
37
+ ```py
38
+ >>> import torch
39
+ >>> from diffusers import ZImageOmniPipeline
40
+
41
+ >>> pipe = ZImageOmniPipeline.from_pretrained("Z-a-o/Z-Image-Turbo", torch_dtype=torch.bfloat16)
42
+ >>> pipe.to("cuda")
43
+
44
+ >>> # Optionally, set the attention backend to flash-attn 2 or 3, default is SDPA in PyTorch.
45
+ >>> # (1) Use flash attention 2
46
+ >>> # pipe.transformer.set_attention_backend("flash")
47
+ >>> # (2) Use flash attention 3
48
+ >>> # pipe.transformer.set_attention_backend("_flash_3")
49
+
50
+ >>> prompt = "一幅为名为“造相「Z-IMAGE-TURBO」”的项目设计的创意海报。画面巧妙地将文字概念视觉化:一辆复古蒸汽小火车化身为巨大的拉链头,正拉开厚厚的冬日积雪,展露出一个生机盎然的春天。"
51
+ >>> image = pipe(
52
+ ... prompt,
53
+ ... height=1024,
54
+ ... width=1024,
55
+ ... num_inference_steps=8,
56
+ ... guidance_scale=0.0,
57
+ ... generator=torch.Generator("cuda").manual_seed(42),
58
+ ... ).images[0]
59
+ >>> image.save("zimage.png")
60
+ ```
61
+ """
62
+
63
+
64
+ # Copied from diffusers.pipelines.flux.pipeline_flux.calculate_shift
65
+ def calculate_shift(
66
+ image_seq_len,
67
+ base_seq_len: int = 256,
68
+ max_seq_len: int = 4096,
69
+ base_shift: float = 0.5,
70
+ max_shift: float = 1.15,
71
+ ):
72
+ m = (max_shift - base_shift) / (max_seq_len - base_seq_len)
73
+ b = base_shift - m * base_seq_len
74
+ mu = image_seq_len * m + b
75
+ return mu
76
+
77
+
78
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
79
+ def retrieve_timesteps(
80
+ scheduler,
81
+ num_inference_steps: int | None = None,
82
+ device: str | torch.device | None = None,
83
+ timesteps: list[int] | None = None,
84
+ sigmas: list[float] | None = None,
85
+ **kwargs,
86
+ ):
87
+ r"""
88
+ Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
89
+ custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
90
+
91
+ Args:
92
+ scheduler (`SchedulerMixin`):
93
+ The scheduler to get timesteps from.
94
+ num_inference_steps (`int`):
95
+ The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
96
+ must be `None`.
97
+ device (`str` or `torch.device`, *optional*):
98
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
99
+ timesteps (`list[int]`, *optional*):
100
+ Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
101
+ `num_inference_steps` and `sigmas` must be `None`.
102
+ sigmas (`list[float]`, *optional*):
103
+ Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
104
+ `num_inference_steps` and `timesteps` must be `None`.
105
+
106
+ Returns:
107
+ `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
108
+ second element is the number of inference steps.
109
+ """
110
+ if timesteps is not None and sigmas is not None:
111
+ raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
112
+ if timesteps is not None:
113
+ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
114
+ if not accepts_timesteps:
115
+ raise ValueError(
116
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
117
+ f" timestep schedules. Please check whether you are using the correct scheduler."
118
+ )
119
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
120
+ timesteps = scheduler.timesteps
121
+ num_inference_steps = len(timesteps)
122
+ elif sigmas is not None:
123
+ accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
124
+ if not accept_sigmas:
125
+ raise ValueError(
126
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
127
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
128
+ )
129
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
130
+ timesteps = scheduler.timesteps
131
+ num_inference_steps = len(timesteps)
132
+ else:
133
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
134
+ timesteps = scheduler.timesteps
135
+ return timesteps, num_inference_steps
136
+
137
+
138
+ # Copied from diffusers.pipelines.z_image.pipeline_z_image.get_default_z_image_sigmas
139
+ def get_default_z_image_sigmas(num_inference_steps: int) -> list[float]:
140
+ return torch.linspace(1.0, 1 / num_inference_steps, num_inference_steps).tolist()
141
+
142
+
143
+ class ZImageOmniPipeline(DiffusionPipeline, ZImageLoraLoaderMixin, FromSingleFileMixin):
144
+ model_cpu_offload_seq = "text_encoder->transformer->vae"
145
+ _optional_components = []
146
+ _callback_tensor_inputs = ["latents", "prompt_embeds"]
147
+
148
+ def __init__(
149
+ self,
150
+ scheduler: FlowMatchEulerDiscreteScheduler,
151
+ vae: AutoencoderKL,
152
+ text_encoder: PreTrainedModel,
153
+ tokenizer: AutoTokenizer,
154
+ transformer: ZImageTransformer2DModel,
155
+ siglip: Siglip2VisionModel,
156
+ siglip_processor: Siglip2ImageProcessorFast,
157
+ ):
158
+ super().__init__()
159
+
160
+ self.register_modules(
161
+ vae=vae,
162
+ text_encoder=text_encoder,
163
+ tokenizer=tokenizer,
164
+ scheduler=scheduler,
165
+ transformer=transformer,
166
+ siglip=siglip,
167
+ siglip_processor=siglip_processor,
168
+ )
169
+ self.vae_scale_factor = (
170
+ 2 ** (len(self.vae.config.block_out_channels) - 1) if hasattr(self, "vae") and self.vae is not None else 8
171
+ )
172
+ # self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2)
173
+ self.image_processor = Flux2ImageProcessor(vae_scale_factor=self.vae_scale_factor * 2)
174
+
175
+ def encode_prompt(
176
+ self,
177
+ prompt: str | list[str],
178
+ device: torch.device | None = None,
179
+ do_classifier_free_guidance: bool = True,
180
+ negative_prompt: str | list[str] | None = None,
181
+ prompt_embeds: list[torch.FloatTensor] | None = None,
182
+ negative_prompt_embeds: torch.FloatTensor | None = None,
183
+ max_sequence_length: int = 512,
184
+ num_condition_images: int = 0,
185
+ ):
186
+ prompt = [prompt] if isinstance(prompt, str) else prompt
187
+ prompt_embeds = self._encode_prompt(
188
+ prompt=prompt,
189
+ device=device,
190
+ prompt_embeds=prompt_embeds,
191
+ max_sequence_length=max_sequence_length,
192
+ num_condition_images=num_condition_images,
193
+ )
194
+
195
+ if do_classifier_free_guidance:
196
+ if negative_prompt is None:
197
+ negative_prompt = ["" for _ in prompt]
198
+ else:
199
+ negative_prompt = [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
200
+ assert len(prompt) == len(negative_prompt)
201
+ negative_prompt_embeds = self._encode_prompt(
202
+ prompt=negative_prompt,
203
+ device=device,
204
+ prompt_embeds=negative_prompt_embeds,
205
+ max_sequence_length=max_sequence_length,
206
+ num_condition_images=num_condition_images,
207
+ )
208
+ else:
209
+ negative_prompt_embeds = []
210
+ return prompt_embeds, negative_prompt_embeds
211
+
212
+ def _encode_prompt(
213
+ self,
214
+ prompt: str | list[str],
215
+ device: torch.device | None = None,
216
+ prompt_embeds: list[torch.FloatTensor] | None = None,
217
+ max_sequence_length: int = 512,
218
+ num_condition_images: int = 0,
219
+ ) -> list[torch.FloatTensor]:
220
+ device = device or self._execution_device
221
+
222
+ if prompt_embeds is not None:
223
+ return prompt_embeds
224
+
225
+ if isinstance(prompt, str):
226
+ prompt = [prompt]
227
+
228
+ for i, prompt_item in enumerate(prompt):
229
+ if num_condition_images == 0:
230
+ prompt[i] = ["<|im_start|>user\n" + prompt_item + "<|im_end|>\n<|im_start|>assistant\n"]
231
+ elif num_condition_images > 0:
232
+ prompt_list = ["<|im_start|>user\n<|vision_start|>"]
233
+ prompt_list += ["<|vision_end|><|vision_start|>"] * (num_condition_images - 1)
234
+ prompt_list += ["<|vision_end|>" + prompt_item + "<|im_end|>\n<|im_start|>assistant\n<|vision_start|>"]
235
+ prompt_list += ["<|vision_end|><|im_end|>"]
236
+ prompt[i] = prompt_list
237
+
238
+ flattened_prompt = []
239
+ prompt_list_lengths = []
240
+
241
+ for i in range(len(prompt)):
242
+ prompt_list_lengths.append(len(prompt[i]))
243
+ flattened_prompt.extend(prompt[i])
244
+
245
+ text_inputs = self.tokenizer(
246
+ flattened_prompt,
247
+ padding="max_length",
248
+ max_length=max_sequence_length,
249
+ truncation=True,
250
+ return_tensors="pt",
251
+ )
252
+
253
+ text_input_ids = text_inputs.input_ids.to(device)
254
+ prompt_masks = text_inputs.attention_mask.to(device).bool()
255
+
256
+ prompt_embeds = self.text_encoder(
257
+ input_ids=text_input_ids,
258
+ attention_mask=prompt_masks,
259
+ output_hidden_states=True,
260
+ ).hidden_states[-2]
261
+
262
+ embeddings_list = []
263
+ start_idx = 0
264
+ for i in range(len(prompt_list_lengths)):
265
+ batch_embeddings = []
266
+ end_idx = start_idx + prompt_list_lengths[i]
267
+ for j in range(start_idx, end_idx):
268
+ batch_embeddings.append(prompt_embeds[j][prompt_masks[j]])
269
+ embeddings_list.append(batch_embeddings)
270
+ start_idx = end_idx
271
+
272
+ return embeddings_list
273
+
274
+ def prepare_latents(
275
+ self,
276
+ batch_size,
277
+ num_channels_latents,
278
+ height,
279
+ width,
280
+ dtype,
281
+ device,
282
+ generator,
283
+ latents=None,
284
+ ):
285
+ height = 2 * (int(height) // (self.vae_scale_factor * 2))
286
+ width = 2 * (int(width) // (self.vae_scale_factor * 2))
287
+
288
+ shape = (batch_size, num_channels_latents, height, width)
289
+
290
+ if latents is None:
291
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
292
+ else:
293
+ if latents.shape != shape:
294
+ raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}")
295
+ latents = latents.to(device)
296
+ return latents
297
+
298
+ def prepare_image_latents(
299
+ self,
300
+ images: list[torch.Tensor],
301
+ batch_size,
302
+ device,
303
+ dtype,
304
+ ):
305
+ image_latents = []
306
+ for image in images:
307
+ image = image.to(device=device, dtype=dtype)
308
+ image_latent = (
309
+ self.vae.encode(image.bfloat16()).latent_dist.mode()[0] - self.vae.config.shift_factor
310
+ ) * self.vae.config.scaling_factor
311
+ image_latent = image_latent.unsqueeze(1).to(dtype)
312
+ image_latents.append(image_latent) # (16, 128, 128)
313
+
314
+ # image_latents = [image_latents] * batch_size
315
+ image_latents = [image_latents.copy() for _ in range(batch_size)]
316
+
317
+ return image_latents
318
+
319
+ def prepare_siglip_embeds(
320
+ self,
321
+ images: list[torch.Tensor],
322
+ batch_size,
323
+ device,
324
+ dtype,
325
+ ):
326
+ siglip_embeds = []
327
+ for image in images:
328
+ siglip_inputs = self.siglip_processor(images=[image], return_tensors="pt").to(device)
329
+ shape = siglip_inputs.spatial_shapes[0]
330
+ hidden_state = self.siglip(**siglip_inputs).last_hidden_state
331
+ B, N, C = hidden_state.shape
332
+ hidden_state = hidden_state[:, : shape[0] * shape[1]]
333
+ hidden_state = hidden_state.view(shape[0], shape[1], C)
334
+ siglip_embeds.append(hidden_state.to(dtype))
335
+
336
+ # siglip_embeds = [siglip_embeds] * batch_size
337
+ siglip_embeds = [siglip_embeds.copy() for _ in range(batch_size)]
338
+
339
+ return siglip_embeds
340
+
341
+ @property
342
+ def guidance_scale(self):
343
+ return self._guidance_scale
344
+
345
+ @property
346
+ def do_classifier_free_guidance(self):
347
+ return self._guidance_scale > 0
348
+
349
+ @property
350
+ def joint_attention_kwargs(self):
351
+ return self._joint_attention_kwargs
352
+
353
+ @property
354
+ def num_timesteps(self):
355
+ return self._num_timesteps
356
+
357
+ @property
358
+ def interrupt(self):
359
+ return self._interrupt
360
+
361
+ @torch.no_grad()
362
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
363
+ def __call__(
364
+ self,
365
+ image: list[PIL.Image.Image, PIL.Image.Image] | None = None,
366
+ prompt: str | list[str] = None,
367
+ height: int | None = None,
368
+ width: int | None = None,
369
+ num_inference_steps: int = 50,
370
+ sigmas: list[float] | None = None,
371
+ guidance_scale: float = 5.0,
372
+ cfg_normalization: bool = False,
373
+ cfg_truncation: float = 1.0,
374
+ negative_prompt: str | list[str] | None = None,
375
+ num_images_per_prompt: int | None = 1,
376
+ generator: torch.Generator | list[torch.Generator] | None = None,
377
+ latents: torch.FloatTensor | None = None,
378
+ prompt_embeds: list[torch.FloatTensor] | None = None,
379
+ negative_prompt_embeds: list[torch.FloatTensor] | None = None,
380
+ output_type: str | None = "pil",
381
+ return_dict: bool = True,
382
+ joint_attention_kwargs: dict[str, Any] | None = None,
383
+ callback_on_step_end: Callable[[int, int], None] | None = None,
384
+ callback_on_step_end_tensor_inputs: list[str] = ["latents"],
385
+ max_sequence_length: int = 512,
386
+ ):
387
+ r"""
388
+ Function invoked when calling the pipeline for generation.
389
+
390
+ Args:
391
+ image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `list[torch.Tensor]`, `list[PIL.Image.Image]`, or `list[np.ndarray]`):
392
+ `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both
393
+ numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list
394
+ or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a
395
+ list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image
396
+ latents as `image`, but if passing latents directly it is not encoded again.
397
+ prompt (`str` or `list[str]`, *optional*):
398
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
399
+ instead.
400
+ height (`int`, *optional*, defaults to 1024):
401
+ The height in pixels of the generated image.
402
+ width (`int`, *optional*, defaults to 1024):
403
+ The width in pixels of the generated image.
404
+ num_inference_steps (`int`, *optional*, defaults to 50):
405
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
406
+ expense of slower inference.
407
+ sigmas (`list[float]`, *optional*):
408
+ Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
409
+ their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
410
+ will be used.
411
+ guidance_scale (`float`, *optional*, defaults to 5.0):
412
+ Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
413
+ `guidance_scale` is defined as `w` of equation 2. of [Imagen
414
+ Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
415
+ 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
416
+ usually at the expense of lower image quality.
417
+ cfg_normalization (`bool`, *optional*, defaults to False):
418
+ Whether to apply configuration normalization.
419
+ cfg_truncation (`float`, *optional*, defaults to 1.0):
420
+ The truncation value for configuration.
421
+ negative_prompt (`str` or `list[str]`, *optional*):
422
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
423
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
424
+ less than `1`).
425
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
426
+ The number of images to generate per prompt.
427
+ generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
428
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
429
+ to make generation deterministic.
430
+ latents (`torch.FloatTensor`, *optional*):
431
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
432
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
433
+ tensor will be generated by sampling using the supplied random `generator`.
434
+ prompt_embeds (`list[torch.FloatTensor]`, *optional*):
435
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
436
+ provided, text embeddings will be generated from `prompt` input argument.
437
+ negative_prompt_embeds (`list[torch.FloatTensor]`, *optional*):
438
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
439
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
440
+ argument.
441
+ output_type (`str`, *optional*, defaults to `"pil"`):
442
+ The output format of the generate image. Choose between
443
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
444
+ return_dict (`bool`, *optional*, defaults to `True`):
445
+ Whether or not to return a [`~pipelines.stable_diffusion.ZImagePipelineOutput`] instead of a plain
446
+ tuple.
447
+ joint_attention_kwargs (`dict`, *optional*):
448
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
449
+ `self.processor` in
450
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
451
+ callback_on_step_end (`Callable`, *optional*):
452
+ A function that calls at the end of each denoising steps during the inference. The function is called
453
+ with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
454
+ callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
455
+ `callback_on_step_end_tensor_inputs`.
456
+ callback_on_step_end_tensor_inputs (`List`, *optional*):
457
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
458
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
459
+ `._callback_tensor_inputs` attribute of your pipeline class.
460
+ max_sequence_length (`int`, *optional*, defaults to 512):
461
+ Maximum sequence length to use with the `prompt`.
462
+
463
+ Examples:
464
+
465
+ Returns:
466
+ [`~pipelines.z_image.ZImagePipelineOutput`] or `tuple`: [`~pipelines.z_image.ZImagePipelineOutput`] if
467
+ `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the
468
+ generated images.
469
+ """
470
+
471
+ if image is not None and not isinstance(image, list):
472
+ image = [image]
473
+ num_condition_images = len(image) if image is not None else 0
474
+
475
+ device = self._execution_device
476
+
477
+ self._guidance_scale = guidance_scale
478
+ self._joint_attention_kwargs = joint_attention_kwargs
479
+ self._interrupt = False
480
+ self._cfg_normalization = cfg_normalization
481
+ self._cfg_truncation = cfg_truncation
482
+
483
+ # 2. Define call parameters
484
+ if prompt is not None and isinstance(prompt, str):
485
+ batch_size = 1
486
+ elif prompt is not None and isinstance(prompt, list):
487
+ batch_size = len(prompt)
488
+ else:
489
+ batch_size = len(prompt_embeds)
490
+
491
+ # If prompt_embeds is provided and prompt is None, skip encoding
492
+ if prompt_embeds is not None and prompt is None:
493
+ if self.do_classifier_free_guidance and negative_prompt_embeds is None:
494
+ raise ValueError(
495
+ "When `prompt_embeds` is provided without `prompt`, "
496
+ "`negative_prompt_embeds` must also be provided for classifier-free guidance."
497
+ )
498
+ else:
499
+ (
500
+ prompt_embeds,
501
+ negative_prompt_embeds,
502
+ ) = self.encode_prompt(
503
+ prompt=prompt,
504
+ negative_prompt=negative_prompt,
505
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
506
+ prompt_embeds=prompt_embeds,
507
+ negative_prompt_embeds=negative_prompt_embeds,
508
+ device=device,
509
+ max_sequence_length=max_sequence_length,
510
+ num_condition_images=num_condition_images,
511
+ )
512
+
513
+ # 3. Process condition images. Copied from diffusers.pipelines.flux2.pipeline_flux2
514
+ condition_images = []
515
+ resized_images = []
516
+ if image is not None:
517
+ for img in image:
518
+ self.image_processor.check_image_input(img)
519
+ for img in image:
520
+ image_width, image_height = img.size
521
+ if image_width * image_height > 1024 * 1024:
522
+ if height is not None and width is not None:
523
+ img = self.image_processor._resize_to_target_area(img, height * width)
524
+ else:
525
+ img = self.image_processor._resize_to_target_area(img, 1024 * 1024)
526
+ image_width, image_height = img.size
527
+ resized_images.append(img)
528
+
529
+ multiple_of = self.vae_scale_factor * 2
530
+ image_width = (image_width // multiple_of) * multiple_of
531
+ image_height = (image_height // multiple_of) * multiple_of
532
+ img = self.image_processor.preprocess(img, height=image_height, width=image_width, resize_mode="crop")
533
+ condition_images.append(img)
534
+
535
+ if len(condition_images) > 0:
536
+ height = height or image_height
537
+ width = width or image_width
538
+
539
+ else:
540
+ height = height or 1024
541
+ width = width or 1024
542
+
543
+ vae_scale = self.vae_scale_factor * 2
544
+ if height % vae_scale != 0:
545
+ raise ValueError(
546
+ f"Height must be divisible by {vae_scale} (got {height}). "
547
+ f"Please adjust the height to a multiple of {vae_scale}."
548
+ )
549
+ if width % vae_scale != 0:
550
+ raise ValueError(
551
+ f"Width must be divisible by {vae_scale} (got {width}). "
552
+ f"Please adjust the width to a multiple of {vae_scale}."
553
+ )
554
+
555
+ # 4. Prepare latent variables
556
+ num_channels_latents = self.transformer.in_channels
557
+
558
+ latents = self.prepare_latents(
559
+ batch_size * num_images_per_prompt,
560
+ num_channels_latents,
561
+ height,
562
+ width,
563
+ torch.float32,
564
+ device,
565
+ generator,
566
+ latents,
567
+ )
568
+
569
+ condition_latents = self.prepare_image_latents(
570
+ images=condition_images,
571
+ batch_size=batch_size * num_images_per_prompt,
572
+ device=device,
573
+ dtype=torch.float32,
574
+ )
575
+ condition_latents = [[lat.to(self.transformer.dtype) for lat in lats] for lats in condition_latents]
576
+ if self.do_classifier_free_guidance:
577
+ negative_condition_latents = [[lat.clone() for lat in batch] for batch in condition_latents]
578
+
579
+ condition_siglip_embeds = self.prepare_siglip_embeds(
580
+ images=resized_images,
581
+ batch_size=batch_size * num_images_per_prompt,
582
+ device=device,
583
+ dtype=torch.float32,
584
+ )
585
+ condition_siglip_embeds = [[se.to(self.transformer.dtype) for se in sels] for sels in condition_siglip_embeds]
586
+ if self.do_classifier_free_guidance:
587
+ negative_condition_siglip_embeds = [[se.clone() for se in batch] for batch in condition_siglip_embeds]
588
+
589
+ # Repeat prompt_embeds for num_images_per_prompt
590
+ if num_images_per_prompt > 1:
591
+ prompt_embeds = [pe for pe in prompt_embeds for _ in range(num_images_per_prompt)]
592
+ if self.do_classifier_free_guidance and negative_prompt_embeds:
593
+ negative_prompt_embeds = [npe for npe in negative_prompt_embeds for _ in range(num_images_per_prompt)]
594
+
595
+ condition_siglip_embeds = [None if sels == [] else sels + [None] for sels in condition_siglip_embeds]
596
+ if self.do_classifier_free_guidance:
597
+ negative_condition_siglip_embeds = [
598
+ None if sels == [] else sels + [None] for sels in negative_condition_siglip_embeds
599
+ ]
600
+
601
+ actual_batch_size = batch_size * num_images_per_prompt
602
+ image_seq_len = (latents.shape[2] // 2) * (latents.shape[3] // 2)
603
+
604
+ # 5. Prepare timesteps
605
+ mu = calculate_shift(
606
+ image_seq_len,
607
+ self.scheduler.config.get("base_image_seq_len", 256),
608
+ self.scheduler.config.get("max_image_seq_len", 4096),
609
+ self.scheduler.config.get("base_shift", 0.5),
610
+ self.scheduler.config.get("max_shift", 1.15),
611
+ )
612
+ if sigmas is None:
613
+ sigmas = get_default_z_image_sigmas(num_inference_steps)
614
+ scheduler_kwargs = {"mu": mu}
615
+ timesteps, num_inference_steps = retrieve_timesteps(
616
+ self.scheduler,
617
+ num_inference_steps,
618
+ device,
619
+ sigmas=sigmas,
620
+ **scheduler_kwargs,
621
+ )
622
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
623
+ self._num_timesteps = len(timesteps)
624
+
625
+ # 6. Denoising loop
626
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
627
+ for i, t in enumerate(timesteps):
628
+ if self.interrupt:
629
+ continue
630
+
631
+ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
632
+ timestep = t.expand(latents.shape[0])
633
+ timestep = (1000 - timestep) / 1000
634
+ # Normalized time for time-aware config (0 at start, 1 at end)
635
+ t_norm = timestep[0].item()
636
+
637
+ # Handle cfg truncation
638
+ current_guidance_scale = self.guidance_scale
639
+ if (
640
+ self.do_classifier_free_guidance
641
+ and self._cfg_truncation is not None
642
+ and float(self._cfg_truncation) <= 1
643
+ ):
644
+ if t_norm > self._cfg_truncation:
645
+ current_guidance_scale = 0.0
646
+
647
+ # Run CFG only if configured AND scale is non-zero
648
+ apply_cfg = self.do_classifier_free_guidance and current_guidance_scale > 0
649
+
650
+ if apply_cfg:
651
+ latents_typed = latents.to(self.transformer.dtype)
652
+ latent_model_input = latents_typed.repeat(2, 1, 1, 1)
653
+ prompt_embeds_model_input = prompt_embeds + negative_prompt_embeds
654
+ condition_latents_model_input = condition_latents + negative_condition_latents
655
+ condition_siglip_embeds_model_input = condition_siglip_embeds + negative_condition_siglip_embeds
656
+ timestep_model_input = timestep.repeat(2)
657
+ else:
658
+ latent_model_input = latents.to(self.transformer.dtype)
659
+ prompt_embeds_model_input = prompt_embeds
660
+ condition_latents_model_input = condition_latents
661
+ condition_siglip_embeds_model_input = condition_siglip_embeds
662
+ timestep_model_input = timestep
663
+
664
+ latent_model_input = latent_model_input.unsqueeze(2)
665
+ latent_model_input_list = list(latent_model_input.unbind(dim=0))
666
+
667
+ # Combine condition latents with target latent
668
+ current_batch_size = len(latent_model_input_list)
669
+ x_combined = [
670
+ condition_latents_model_input[i] + [latent_model_input_list[i]] for i in range(current_batch_size)
671
+ ]
672
+ # Create noise mask: 0 for condition images (clean), 1 for target image (noisy)
673
+ image_noise_mask = [
674
+ [0] * len(condition_latents_model_input[i]) + [1] for i in range(current_batch_size)
675
+ ]
676
+
677
+ model_out_list = self.transformer(
678
+ x=x_combined,
679
+ t=timestep_model_input,
680
+ cap_feats=prompt_embeds_model_input,
681
+ siglip_feats=condition_siglip_embeds_model_input,
682
+ image_noise_mask=image_noise_mask,
683
+ return_dict=False,
684
+ )[0]
685
+
686
+ if apply_cfg:
687
+ # Perform CFG
688
+ pos_out = model_out_list[:actual_batch_size]
689
+ neg_out = model_out_list[actual_batch_size:]
690
+
691
+ noise_pred = []
692
+ for j in range(actual_batch_size):
693
+ pos = pos_out[j].float()
694
+ neg = neg_out[j].float()
695
+
696
+ pred = pos + current_guidance_scale * (pos - neg)
697
+
698
+ # Renormalization
699
+ if self._cfg_normalization and float(self._cfg_normalization) > 0.0:
700
+ ori_pos_norm = torch.linalg.vector_norm(pos)
701
+ new_pos_norm = torch.linalg.vector_norm(pred)
702
+ max_new_norm = ori_pos_norm * float(self._cfg_normalization)
703
+ if new_pos_norm > max_new_norm:
704
+ pred = pred * (max_new_norm / new_pos_norm)
705
+
706
+ noise_pred.append(pred)
707
+
708
+ noise_pred = torch.stack(noise_pred, dim=0)
709
+ else:
710
+ noise_pred = torch.stack([t.float() for t in model_out_list], dim=0)
711
+
712
+ noise_pred = noise_pred.squeeze(2)
713
+ noise_pred = -noise_pred
714
+
715
+ # compute the previous noisy sample x_t -> x_t-1
716
+ latents = self.scheduler.step(noise_pred.to(torch.float32), t, latents, return_dict=False)[0]
717
+ assert latents.dtype == torch.float32
718
+
719
+ if callback_on_step_end is not None:
720
+ callback_kwargs = {}
721
+ for k in callback_on_step_end_tensor_inputs:
722
+ callback_kwargs[k] = locals()[k]
723
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
724
+
725
+ latents = callback_outputs.pop("latents", latents)
726
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
727
+ negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
728
+
729
+ # call the callback, if provided
730
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
731
+ progress_bar.update()
732
+
733
+ if output_type == "latent":
734
+ image = latents
735
+
736
+ else:
737
+ latents = latents.to(self.vae.dtype)
738
+ latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor
739
+
740
+ image = self.vae.decode(latents, return_dict=False)[0]
741
+ image = self.image_processor.postprocess(image, output_type=output_type)
742
+
743
+ # Offload all models
744
+ self.maybe_free_model_hooks()
745
+
746
+ if not return_dict:
747
+ return (image,)
748
+
749
+ return ZImagePipelineOutput(images=image)
western_ai/europe/diffusers/src/diffusers/py.typed ADDED
File without changes
western_ai/europe/diffusers/src/diffusers/quantizers/__init__.py ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2026 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ from .auto import DiffusersAutoQuantizer
17
+ from .base import DiffusersQuantizer
18
+ from .pipe_quant_config import PipelineQuantizationConfig
western_ai/europe/diffusers/src/diffusers/quantizers/auto.py ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 The HuggingFace Inc. team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """
15
+ Adapted from
16
+ https://github.com/huggingface/transformers/blob/c409cd81777fb27aadc043ed3d8339dbc020fb3b/src/transformers/quantizers/auto.py
17
+ """
18
+
19
+ import warnings
20
+
21
+ from .autoround import AutoRoundQuantizer
22
+ from .bitsandbytes import BnB4BitDiffusersQuantizer, BnB8BitDiffusersQuantizer
23
+ from .gguf import GGUFQuantizer
24
+ from .modelopt import NVIDIAModelOptQuantizer
25
+ from .quantization_config import (
26
+ AutoRoundConfig,
27
+ BitsAndBytesConfig,
28
+ GGUFQuantizationConfig,
29
+ NVIDIAModelOptConfig,
30
+ QuantizationConfigMixin,
31
+ QuantizationMethod,
32
+ QuantoConfig,
33
+ TorchAoConfig,
34
+ )
35
+ from .quanto import QuantoQuantizer
36
+ from .torchao import TorchAoHfQuantizer
37
+
38
+
39
+ AUTO_QUANTIZER_MAPPING = {
40
+ "bitsandbytes_4bit": BnB4BitDiffusersQuantizer,
41
+ "bitsandbytes_8bit": BnB8BitDiffusersQuantizer,
42
+ "gguf": GGUFQuantizer,
43
+ "quanto": QuantoQuantizer,
44
+ "torchao": TorchAoHfQuantizer,
45
+ "modelopt": NVIDIAModelOptQuantizer,
46
+ "auto-round": AutoRoundQuantizer,
47
+ }
48
+
49
+ AUTO_QUANTIZATION_CONFIG_MAPPING = {
50
+ "bitsandbytes_4bit": BitsAndBytesConfig,
51
+ "bitsandbytes_8bit": BitsAndBytesConfig,
52
+ "gguf": GGUFQuantizationConfig,
53
+ "quanto": QuantoConfig,
54
+ "torchao": TorchAoConfig,
55
+ "modelopt": NVIDIAModelOptConfig,
56
+ "auto-round": AutoRoundConfig,
57
+ }
58
+
59
+
60
+ class DiffusersAutoQuantizer:
61
+ """
62
+ The auto diffusers quantizer class that takes care of automatically instantiating to the correct
63
+ `DiffusersQuantizer` given the `QuantizationConfig`.
64
+ """
65
+
66
+ @classmethod
67
+ def from_dict(cls, quantization_config_dict: dict):
68
+ quant_method = quantization_config_dict.get("quant_method", None)
69
+ # We need a special care for bnb models to make sure everything is BC ..
70
+ if quantization_config_dict.get("load_in_8bit", False) or quantization_config_dict.get("load_in_4bit", False):
71
+ suffix = "_4bit" if quantization_config_dict.get("load_in_4bit", False) else "_8bit"
72
+ quant_method = QuantizationMethod.BITS_AND_BYTES + suffix
73
+ elif quant_method is None:
74
+ raise ValueError(
75
+ "The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the model has been correctly quantized"
76
+ )
77
+
78
+ if quant_method not in AUTO_QUANTIZATION_CONFIG_MAPPING.keys():
79
+ raise ValueError(
80
+ f"Unknown quantization type, got {quant_method} - supported types are:"
81
+ f" {list(AUTO_QUANTIZER_MAPPING.keys())}"
82
+ )
83
+
84
+ target_cls = AUTO_QUANTIZATION_CONFIG_MAPPING[quant_method]
85
+ return target_cls.from_dict(quantization_config_dict)
86
+
87
+ @classmethod
88
+ def from_config(cls, quantization_config: QuantizationConfigMixin | dict, **kwargs):
89
+ # Convert it to a QuantizationConfig if the q_config is a dict
90
+ if isinstance(quantization_config, dict):
91
+ quantization_config = cls.from_dict(quantization_config)
92
+
93
+ quant_method = quantization_config.quant_method
94
+
95
+ # Again, we need a special care for bnb as we have a single quantization config
96
+ # class for both 4-bit and 8-bit quantization
97
+ if quant_method == QuantizationMethod.BITS_AND_BYTES:
98
+ if quantization_config.load_in_8bit:
99
+ quant_method += "_8bit"
100
+ else:
101
+ quant_method += "_4bit"
102
+
103
+ if quant_method not in AUTO_QUANTIZER_MAPPING.keys():
104
+ raise ValueError(
105
+ f"Unknown quantization type, got {quant_method} - supported types are:"
106
+ f" {list(AUTO_QUANTIZER_MAPPING.keys())}"
107
+ )
108
+
109
+ target_cls = AUTO_QUANTIZER_MAPPING[quant_method]
110
+ return target_cls(quantization_config, **kwargs)
111
+
112
+ @classmethod
113
+ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
114
+ model_config = cls.load_config(pretrained_model_name_or_path, **kwargs)
115
+ if getattr(model_config, "quantization_config", None) is None:
116
+ raise ValueError(
117
+ f"Did not found a `quantization_config` in {pretrained_model_name_or_path}. Make sure that the model is correctly quantized."
118
+ )
119
+ quantization_config_dict = model_config.quantization_config
120
+ quantization_config = cls.from_dict(quantization_config_dict)
121
+ # Update with potential kwargs that are passed through from_pretrained.
122
+ quantization_config.update(kwargs)
123
+
124
+ return cls.from_config(quantization_config)
125
+
126
+ @classmethod
127
+ def merge_quantization_configs(
128
+ cls,
129
+ quantization_config: dict | QuantizationConfigMixin,
130
+ quantization_config_from_args: QuantizationConfigMixin | None,
131
+ ):
132
+ """
133
+ handles situations where both quantization_config from args and quantization_config from model config are
134
+ present.
135
+ """
136
+ if quantization_config_from_args is not None:
137
+ warning_msg = (
138
+ "You passed `quantization_config` or equivalent parameters to `from_pretrained` but the model you're loading"
139
+ " already has a `quantization_config` attribute. The `quantization_config` from the model will be used."
140
+ )
141
+ else:
142
+ warning_msg = ""
143
+
144
+ if isinstance(quantization_config, dict):
145
+ quantization_config = cls.from_dict(quantization_config)
146
+
147
+ if isinstance(quantization_config, NVIDIAModelOptConfig):
148
+ quantization_config.check_model_patching()
149
+
150
+ if quantization_config_from_args is not None and isinstance(quantization_config, AutoRoundConfig):
151
+ # For AutoRound, allow overriding fields like `backend` from user args,
152
+ # since the model config may store a default value (e.g. backend="auto").
153
+ for key, value in quantization_config_from_args.__dict__.items():
154
+ if key in ("quant_method",):
155
+ continue
156
+ if hasattr(quantization_config, key) and getattr(quantization_config, key) != value:
157
+ warnings.warn(
158
+ f"Overriding `{key}` in the model's quantization_config with value {value!r} "
159
+ f"from the user-provided `quantization_config`."
160
+ )
161
+ setattr(quantization_config, key, value)
162
+
163
+ if warning_msg != "":
164
+ warnings.warn(warning_msg)
165
+
166
+ return quantization_config
western_ai/europe/diffusers/src/diffusers/quantizers/autoround/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ from .autoround_quantizer import AutoRoundQuantizer
western_ai/europe/diffusers/src/diffusers/quantizers/autoround/autoround_quantizer.py ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 The Intel and The HuggingFace Inc. teams. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from typing import TYPE_CHECKING
16
+
17
+ from ...utils import is_auto_round_available, logging
18
+ from ..base import DiffusersQuantizer
19
+
20
+
21
+ if TYPE_CHECKING:
22
+ from ...models.modeling_utils import ModelMixin
23
+
24
+
25
+ logger = logging.get_logger(__name__)
26
+
27
+
28
+ class AutoRoundQuantizer(DiffusersQuantizer):
29
+ r"""
30
+ Diffusers Quantizer for AutoRound (https://github.com/intel/auto-round).
31
+
32
+ AutoRound is a weight-only quantization method that uses sign gradient descent to jointly optimize rounding values
33
+ and min-max ranges for weights. It supports W4A16 (4-bit weight, 16-bit activation) quantization for efficient
34
+ inference.
35
+
36
+ This quantizer only supports loading pre-quantized AutoRound models. On-the-fly quantization (calibration) is not
37
+ supported through this interface.
38
+ """
39
+
40
+ # AutoRound requires data calibration — we only support loading pre-quantized checkpoints.
41
+ requires_calibration = True
42
+ required_packages = ["auto_round"]
43
+
44
+ def __init__(self, quantization_config, **kwargs):
45
+ super().__init__(quantization_config, **kwargs)
46
+
47
+ def validate_environment(self, *args, **kwargs):
48
+ """
49
+ Validates that the auto-round library (>= 0.5) is installed and captures the device_map for later use during
50
+ model conversion.
51
+ """
52
+ self.device_map = kwargs.get("device_map", None)
53
+ if not is_auto_round_available():
54
+ raise ImportError(
55
+ "Loading an AutoRound quantized model requires the auto-round library "
56
+ "(`pip install 'auto-round>=0.13.0'`)"
57
+ )
58
+ if not self.pre_quantized:
59
+ raise ValueError(
60
+ "AutoRound quantizer in diffusers only supports loading pre-quantized models. "
61
+ "To quantize a model from scratch, use the AutoRound CLI or Python API "
62
+ "(https://github.com/intel/auto-round) directly, then load the result with Diffusers."
63
+ )
64
+
65
+ def _process_model_before_weight_loading(
66
+ self,
67
+ model: "ModelMixin",
68
+ device_map,
69
+ keep_in_fp32_modules: list[str] = [],
70
+ **kwargs,
71
+ ):
72
+ """
73
+ Replaces target nn.Linear layers with AutoRound's quantized QuantLinear layers before weights are loaded from
74
+ the checkpoint.
75
+
76
+ Uses `auto_round.inference.convert_model.convert_hf_model` which:
77
+ - Inspects the model architecture and the quantization config (bits, group_size, sym, backend).
78
+ - Replaces eligible nn.Linear modules with the appropriate QuantLinear variant (the packed-weight layer that
79
+ stores qweight, scales, qzeros).
80
+ - Returns the converted model and a set of used backend names.
81
+
82
+ `infer_target_device` resolves the device_map into a single target device string that AutoRound uses to select
83
+ the correct kernel backend (e.g. "cuda", "cpu").
84
+ """
85
+ from auto_round.inference.convert_model import convert_hf_model, infer_target_device
86
+
87
+ target_device = infer_target_device(self.device_map)
88
+ model, used_backends = convert_hf_model(model, target_device)
89
+ self.used_backends = used_backends
90
+
91
+ def _process_model_after_weight_loading(self, model, **kwargs):
92
+ """
93
+ Finalizes the model after all quantized weights (qweight, scales, qzeros, etc.) have been loaded into the
94
+ QuantLinear layers.
95
+
96
+ Uses `auto_round.inference.convert_model.post_init` which:
97
+ - Performs backend-specific finalization (e.g. repacking weights into the kernel's expected memory layout,
98
+ moving buffers to the correct device).
99
+ - Freezes quantized parameters (requires_grad=False).
100
+ - Prepares the model for inference.
101
+
102
+ """
103
+ from auto_round.inference.convert_model import post_init
104
+
105
+ post_init(model, self.used_backends)
106
+
107
+ return model
108
+
109
+ @property
110
+ def is_trainable(self) -> bool:
111
+ """AutoRound W4A16 pre-quantized models do not support training."""
112
+ return False
113
+
114
+ @property
115
+ def is_serializable(self):
116
+ """AutoRound quantized models can be serialized (the quantization config may be
117
+ updated by the backend, e.g. for GPTQ/AWQ-compatible formats)."""
118
+ return True
119
+
120
+ @property
121
+ def is_compileable(self) -> bool:
122
+ return True
western_ai/europe/diffusers/src/diffusers/quantizers/base.py ADDED
@@ -0,0 +1,265 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 The HuggingFace Inc. team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """
16
+ Adapted from
17
+ https://github.com/huggingface/transformers/blob/52cb4034ada381fe1ffe8d428a1076e5411a8026/src/transformers/quantizers/base.py
18
+ """
19
+
20
+ from abc import ABC, abstractmethod
21
+ from typing import TYPE_CHECKING, Any
22
+
23
+ from ..utils import is_torch_available
24
+ from .quantization_config import QuantizationConfigMixin
25
+
26
+
27
+ if TYPE_CHECKING:
28
+ from ..models.modeling_utils import ModelMixin
29
+
30
+ if is_torch_available():
31
+ import torch
32
+
33
+
34
+ class DiffusersQuantizer(ABC):
35
+ """
36
+ Abstract class of the HuggingFace quantizer. Supports for now quantizing HF diffusers models for inference and/or
37
+ quantization. This class is used only for diffusers.models.modeling_utils.ModelMixin.from_pretrained and cannot be
38
+ easily used outside the scope of that method yet.
39
+
40
+ Attributes
41
+ quantization_config (`diffusers.quantizers.quantization_config.QuantizationConfigMixin`):
42
+ The quantization config that defines the quantization parameters of your model that you want to quantize.
43
+ modules_to_not_convert (`list[str]`, *optional*):
44
+ The list of module names to not convert when quantizing the model.
45
+ required_packages (`list[str]`, *optional*):
46
+ The list of required pip packages to install prior to using the quantizer
47
+ requires_calibration (`bool`):
48
+ Whether the quantization method requires to calibrate the model before using it.
49
+ """
50
+
51
+ requires_calibration = False
52
+ required_packages = None
53
+
54
+ def __init__(self, quantization_config: QuantizationConfigMixin, **kwargs):
55
+ self.quantization_config = quantization_config
56
+
57
+ # -- Handle extra kwargs below --
58
+ self.modules_to_not_convert = kwargs.pop("modules_to_not_convert", [])
59
+ self.pre_quantized = kwargs.pop("pre_quantized", True)
60
+
61
+ if not self.pre_quantized and self.requires_calibration:
62
+ raise ValueError(
63
+ f"The quantization method {quantization_config.quant_method} does require the model to be pre-quantized."
64
+ f" You explicitly passed `pre_quantized=False` meaning your model weights are not quantized. Make sure to "
65
+ f"pass `pre_quantized=True` while knowing what you are doing."
66
+ )
67
+
68
+ def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":
69
+ """
70
+ Some quantization methods require to explicitly set the dtype of the model to a target dtype. You need to
71
+ override this method in case you want to make sure that behavior is preserved
72
+
73
+ Args:
74
+ torch_dtype (`torch.dtype`):
75
+ The input dtype that is passed in `from_pretrained`
76
+ """
77
+ return torch_dtype
78
+
79
+ def update_device_map(self, device_map: dict[str, Any] | None) -> dict[str, Any] | None:
80
+ """
81
+ Override this method if you want to pass a override the existing device map with a new one. E.g. for
82
+ bitsandbytes, since `accelerate` is a hard requirement, if no device_map is passed, the device_map is set to
83
+ `"auto"``
84
+
85
+ Args:
86
+ device_map (`dict | str`, *optional*):
87
+ The device_map that is passed through the `from_pretrained` method.
88
+ """
89
+ return device_map
90
+
91
+ def adjust_target_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":
92
+ """
93
+ Override this method if you want to adjust the `target_dtype` variable used in `from_pretrained` to compute the
94
+ device_map in case the device_map is a `str`. E.g. for bitsandbytes we force-set `target_dtype` to `torch.int8`
95
+ and for 4-bit we pass a custom enum `accelerate.CustomDtype.int4`.
96
+
97
+ Args:
98
+ torch_dtype (`torch.dtype`, *optional*):
99
+ The torch_dtype that is used to compute the device_map.
100
+ """
101
+ return torch_dtype
102
+
103
+ def update_missing_keys(self, model, missing_keys: list[str], prefix: str) -> list[str]:
104
+ """
105
+ Override this method if you want to adjust the `missing_keys`.
106
+
107
+ Args:
108
+ missing_keys (`list[str]`, *optional*):
109
+ The list of missing keys in the checkpoint compared to the state dict of the model
110
+ """
111
+ return missing_keys
112
+
113
+ def get_special_dtypes_update(self, model, torch_dtype: "torch.dtype") -> dict[str, "torch.dtype"]:
114
+ """
115
+ returns dtypes for modules that are not quantized - used for the computation of the device_map in case one
116
+ passes a str as a device_map. The method will use the `modules_to_not_convert` that is modified in
117
+ `_process_model_before_weight_loading`. `diffusers` models don't have any `modules_to_not_convert` attributes
118
+ yet but this can change soon in the future.
119
+
120
+ Args:
121
+ model (`~diffusers.models.modeling_utils.ModelMixin`):
122
+ The model to quantize
123
+ torch_dtype (`torch.dtype`):
124
+ The dtype passed in `from_pretrained` method.
125
+ """
126
+
127
+ return {
128
+ name: torch_dtype
129
+ for name, _ in model.named_parameters()
130
+ if any(m in name for m in self.modules_to_not_convert)
131
+ }
132
+
133
+ def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | str]:
134
+ """adjust max_memory argument for infer_auto_device_map() if extra memory is needed for quantization"""
135
+ return max_memory
136
+
137
+ def check_if_quantized_param(
138
+ self,
139
+ model: "ModelMixin",
140
+ param_value: "torch.Tensor",
141
+ param_name: str,
142
+ state_dict: dict[str, Any],
143
+ **kwargs,
144
+ ) -> bool:
145
+ """
146
+ checks if a loaded state_dict component is part of quantized param + some validation; only defined for
147
+ quantization methods that require to create a new parameters for quantization.
148
+ """
149
+ return False
150
+
151
+ def create_quantized_param(self, *args, **kwargs) -> "torch.nn.Parameter":
152
+ """
153
+ takes needed components from state_dict and creates quantized param.
154
+ """
155
+ return
156
+
157
+ def check_quantized_param_shape(self, *args, **kwargs):
158
+ """
159
+ checks if the quantized param has expected shape.
160
+ """
161
+ return True
162
+
163
+ def validate_environment(self, *args, **kwargs):
164
+ """
165
+ This method is used to potentially check for potential conflicts with arguments that are passed in
166
+ `from_pretrained`. You need to define it for all future quantizers that are integrated with diffusers. If no
167
+ explicit check are needed, simply return nothing.
168
+ """
169
+ return
170
+
171
+ def maybe_update_loaded_keys(self, loaded_keys: list[str], checkpoint_files: list[str]) -> list[str]:
172
+ return loaded_keys
173
+
174
+ def maybe_update_state_dict(self, state_dict: dict[str, Any]) -> dict[str, Any]:
175
+ return state_dict
176
+
177
+ @property
178
+ def supports_parallel_loading(self) -> bool:
179
+ return True
180
+
181
+ def get_state_dict_and_metadata(
182
+ self, state_dict: dict[str, Any], safe_serialization: bool = False
183
+ ) -> tuple[dict[str, Any], dict[str, Any]]:
184
+ return state_dict, {}
185
+
186
+ @property
187
+ def supports_safetensors_serialization(self) -> bool:
188
+ return True
189
+
190
+ def preprocess_model(self, model: "ModelMixin", **kwargs):
191
+ """
192
+ Setting model attributes and/or converting model before weights loading. At this point the model should be
193
+ initialized on the meta device so you can freely manipulate the skeleton of the model in order to replace
194
+ modules in-place. Make sure to override the abstract method `_process_model_before_weight_loading`.
195
+
196
+ Args:
197
+ model (`~diffusers.models.modeling_utils.ModelMixin`):
198
+ The model to quantize
199
+ kwargs (`dict`, *optional*):
200
+ The keyword arguments that are passed along `_process_model_before_weight_loading`.
201
+ """
202
+ model.is_quantized = True
203
+ model.quantization_method = self.quantization_config.quant_method
204
+ return self._process_model_before_weight_loading(model, **kwargs)
205
+
206
+ def postprocess_model(self, model: "ModelMixin", **kwargs):
207
+ """
208
+ Post-process the model post weights loading. Make sure to override the abstract method
209
+ `_process_model_after_weight_loading`.
210
+
211
+ Args:
212
+ model (`~diffusers.models.modeling_utils.ModelMixin`):
213
+ The model to quantize
214
+ kwargs (`dict`, *optional*):
215
+ The keyword arguments that are passed along `_process_model_after_weight_loading`.
216
+ """
217
+ return self._process_model_after_weight_loading(model, **kwargs)
218
+
219
+ def dequantize(self, model):
220
+ """
221
+ Potentially dequantize the model to retrieve the original model, with some loss in accuracy / performance. Note
222
+ not all quantization schemes support this.
223
+ """
224
+ model = self._dequantize(model)
225
+
226
+ # Delete quantizer and quantization config
227
+ del model.hf_quantizer
228
+ model.is_quantized = False
229
+
230
+ return model
231
+
232
+ def get_cuda_warm_up_factor(self):
233
+ """
234
+ The factor to be used in `caching_allocator_warmup` to get the number of bytes to pre-allocate to warm up cuda.
235
+ A factor of 2 means we allocate all bytes in the empty model (since we allocate in fp16), a factor of 4 means
236
+ we allocate half the memory of the weights residing in the empty model, etc...
237
+ """
238
+ # By default we return 4, i.e. half the model size (this corresponds to the case where the model is not
239
+ # really pre-processed, i.e. we do not have the info that weights are going to be 8 bits before actual
240
+ # weight loading)
241
+ return 4
242
+
243
+ def _dequantize(self, model):
244
+ raise NotImplementedError(
245
+ f"{self.quantization_config.quant_method} has no implementation of `dequantize`, please raise an issue on GitHub."
246
+ )
247
+
248
+ @abstractmethod
249
+ def _process_model_before_weight_loading(self, model, **kwargs): ...
250
+
251
+ @abstractmethod
252
+ def _process_model_after_weight_loading(self, model, **kwargs): ...
253
+
254
+ @property
255
+ @abstractmethod
256
+ def is_serializable(self): ...
257
+
258
+ @property
259
+ @abstractmethod
260
+ def is_trainable(self): ...
261
+
262
+ @property
263
+ def is_compileable(self) -> bool:
264
+ """Flag indicating whether the quantized model can be compiled"""
265
+ return False
western_ai/europe/diffusers/src/diffusers/quantizers/bitsandbytes/__init__.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ from .bnb_quantizer import BnB4BitDiffusersQuantizer, BnB8BitDiffusersQuantizer
2
+ from .utils import dequantize_and_replace, dequantize_bnb_weight, replace_with_bnb_linear
western_ai/europe/diffusers/src/diffusers/quantizers/bitsandbytes/bnb_quantizer.py ADDED
@@ -0,0 +1,581 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 The HuggingFace Inc. team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """
15
+ Adapted from
16
+ https://github.com/huggingface/transformers/blob/c409cd81777fb27aadc043ed3d8339dbc020fb3b/src/transformers/quantizers/quantizer_bnb_4bit.py
17
+ """
18
+
19
+ from typing import TYPE_CHECKING, Any
20
+
21
+ from ...utils import get_module_from_name
22
+ from ..base import DiffusersQuantizer
23
+
24
+
25
+ if TYPE_CHECKING:
26
+ from ...models.modeling_utils import ModelMixin
27
+
28
+ from ...utils import (
29
+ is_accelerate_available,
30
+ is_accelerate_version,
31
+ is_bitsandbytes_available,
32
+ is_bitsandbytes_version,
33
+ is_torch_available,
34
+ logging,
35
+ )
36
+
37
+
38
+ if is_torch_available():
39
+ import torch
40
+
41
+ logger = logging.get_logger(__name__)
42
+
43
+
44
+ class BnB4BitDiffusersQuantizer(DiffusersQuantizer):
45
+ """
46
+ 4-bit quantization from bitsandbytes.py quantization method:
47
+ before loading: converts transformer layers into Linear4bit during loading: load 16bit weight and pass to the
48
+ layer object after: quantizes individual weights in Linear4bit into 4bit at the first .cuda() call saving:
49
+ from state dict, as usual; saves weights and `quant_state` components
50
+ loading:
51
+ need to locate `quant_state` components and pass to Param4bit constructor
52
+ """
53
+
54
+ use_keep_in_fp32_modules = True
55
+ requires_calibration = False
56
+
57
+ def __init__(self, quantization_config, **kwargs):
58
+ super().__init__(quantization_config, **kwargs)
59
+
60
+ if self.quantization_config.llm_int8_skip_modules is not None:
61
+ self.modules_to_not_convert = self.quantization_config.llm_int8_skip_modules
62
+
63
+ def validate_environment(self, *args, **kwargs):
64
+ if not (torch.cuda.is_available() or torch.xpu.is_available() or torch.mps.is_available()):
65
+ raise RuntimeError("No GPU found. A GPU is needed for quantization.")
66
+ if not is_accelerate_available() or is_accelerate_version("<", "0.26.0"):
67
+ raise ImportError(
68
+ "Using `bitsandbytes` 4-bit quantization requires Accelerate: `pip install 'accelerate>=0.26.0'`"
69
+ )
70
+ if not is_bitsandbytes_available() or is_bitsandbytes_version("<", "0.43.3"):
71
+ raise ImportError(
72
+ "Using `bitsandbytes` 4-bit quantization requires the latest version of bitsandbytes: `pip install -U bitsandbytes`"
73
+ )
74
+
75
+ if kwargs.get("from_flax", False):
76
+ raise ValueError(
77
+ "Converting into 4-bit weights from flax weights is currently not supported, please make"
78
+ " sure the weights are in PyTorch format."
79
+ )
80
+
81
+ device_map = kwargs.get("device_map", None)
82
+ if (
83
+ device_map is not None
84
+ and isinstance(device_map, dict)
85
+ and not self.quantization_config.llm_int8_enable_fp32_cpu_offload
86
+ ):
87
+ device_map_without_no_convert = {
88
+ key: device_map[key] for key in device_map.keys() if key not in self.modules_to_not_convert
89
+ }
90
+ if "cpu" in device_map_without_no_convert.values() or "disk" in device_map_without_no_convert.values():
91
+ raise ValueError(
92
+ "Some modules are dispatched on the CPU or the disk. Make sure you have enough GPU RAM to fit the "
93
+ "quantized model. If you want to dispatch the model on the CPU or the disk while keeping these modules "
94
+ "in 32-bit, you need to set `load_in_8bit_fp32_cpu_offload=True` and pass a custom `device_map` to "
95
+ "`from_pretrained`. Check "
96
+ "https://huggingface.co/docs/transformers/main/en/main_classes/quantization#offload-between-cpu-and-gpu "
97
+ "for more details. "
98
+ )
99
+
100
+ def adjust_target_dtype(self, target_dtype: "torch.dtype") -> "torch.dtype":
101
+ if target_dtype != torch.int8:
102
+ from accelerate.utils import CustomDtype
103
+
104
+ logger.info("target_dtype {target_dtype} is replaced by `CustomDtype.INT4` for 4-bit BnB quantization")
105
+ return CustomDtype.INT4
106
+ else:
107
+ raise ValueError(f"Wrong `target_dtype` ({target_dtype}) provided.")
108
+
109
+ def check_if_quantized_param(
110
+ self,
111
+ model: "ModelMixin",
112
+ param_value: "torch.Tensor",
113
+ param_name: str,
114
+ state_dict: dict[str, Any],
115
+ **kwargs,
116
+ ) -> bool:
117
+ import bitsandbytes as bnb
118
+
119
+ module, tensor_name = get_module_from_name(model, param_name)
120
+ if isinstance(module._parameters.get(tensor_name, None), bnb.nn.Params4bit):
121
+ # Add here check for loaded components' dtypes once serialization is implemented
122
+ return True
123
+ elif isinstance(module, bnb.nn.Linear4bit) and tensor_name == "bias":
124
+ # bias could be loaded by regular set_module_tensor_to_device() from accelerate,
125
+ # but it would wrongly use uninitialized weight there.
126
+ return True
127
+ else:
128
+ return False
129
+
130
+ def create_quantized_param(
131
+ self,
132
+ model: "ModelMixin",
133
+ param_value: "torch.Tensor",
134
+ param_name: str,
135
+ target_device: "torch.device",
136
+ state_dict: dict[str, Any],
137
+ unexpected_keys: list[str] | None = None,
138
+ **kwargs,
139
+ ):
140
+ import bitsandbytes as bnb
141
+
142
+ module, tensor_name = get_module_from_name(model, param_name)
143
+
144
+ if tensor_name not in module._parameters:
145
+ raise ValueError(f"{module} does not have a parameter or a buffer named {tensor_name}.")
146
+
147
+ old_value = getattr(module, tensor_name)
148
+
149
+ if tensor_name == "bias":
150
+ if param_value is None:
151
+ new_value = old_value.to(target_device)
152
+ else:
153
+ new_value = param_value.to(target_device)
154
+
155
+ new_value = torch.nn.Parameter(new_value, requires_grad=old_value.requires_grad)
156
+ module._parameters[tensor_name] = new_value
157
+ return
158
+
159
+ if not isinstance(module._parameters[tensor_name], bnb.nn.Params4bit):
160
+ raise ValueError("this function only loads `Linear4bit components`")
161
+ if (
162
+ old_value.device == torch.device("meta")
163
+ and target_device not in ["meta", torch.device("meta")]
164
+ and param_value is None
165
+ ):
166
+ raise ValueError(f"{tensor_name} is on the meta device, we need a `value` to put in on {target_device}.")
167
+
168
+ # construct `new_value` for the module._parameters[tensor_name]:
169
+ if self.pre_quantized:
170
+ # 4bit loading. Collecting components for restoring quantized weight
171
+ # This can be expanded to make a universal call for any quantized weight loading
172
+
173
+ if not self.is_serializable:
174
+ raise ValueError(
175
+ "Detected int4 weights but the version of bitsandbytes is not compatible with int4 serialization. "
176
+ "Make sure to download the latest `bitsandbytes` version. `pip install --upgrade bitsandbytes`."
177
+ )
178
+
179
+ if (param_name + ".quant_state.bitsandbytes__fp4" not in state_dict) and (
180
+ param_name + ".quant_state.bitsandbytes__nf4" not in state_dict
181
+ ):
182
+ raise ValueError(
183
+ f"Supplied state dict for {param_name} does not contain `bitsandbytes__*` and possibly other `quantized_stats` components."
184
+ )
185
+
186
+ quantized_stats = {}
187
+ for k, v in state_dict.items():
188
+ # `startswith` to counter for edge cases where `param_name`
189
+ # substring can be present in multiple places in the `state_dict`
190
+ if param_name + "." in k and k.startswith(param_name):
191
+ quantized_stats[k] = v
192
+ if unexpected_keys is not None and k in unexpected_keys:
193
+ unexpected_keys.remove(k)
194
+
195
+ new_value = bnb.nn.Params4bit.from_prequantized(
196
+ data=param_value,
197
+ quantized_stats=quantized_stats,
198
+ requires_grad=False,
199
+ device=target_device,
200
+ )
201
+ else:
202
+ new_value = param_value.to("cpu")
203
+ kwargs = old_value.__dict__
204
+ new_value = bnb.nn.Params4bit(new_value, requires_grad=False, **kwargs).to(target_device)
205
+
206
+ module._parameters[tensor_name] = new_value
207
+
208
+ def check_quantized_param_shape(self, param_name, current_param, loaded_param):
209
+ current_param_shape = current_param.shape
210
+ loaded_param_shape = loaded_param.shape
211
+
212
+ n = current_param_shape.numel()
213
+ inferred_shape = (n,) if "bias" in param_name else ((n + 1) // 2, 1)
214
+ if loaded_param_shape != inferred_shape:
215
+ raise ValueError(
216
+ f"Expected the flattened shape of the current param ({param_name}) to be {loaded_param_shape} but is {inferred_shape}."
217
+ )
218
+ else:
219
+ return True
220
+
221
+ def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | str]:
222
+ # need more space for buffers that are created during quantization
223
+ max_memory = {key: val * 0.90 for key, val in max_memory.items()}
224
+ return max_memory
225
+
226
+ def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":
227
+ if torch_dtype is None:
228
+ # We force the `dtype` to be float16, this is a requirement from `bitsandbytes`
229
+ logger.info(
230
+ "Overriding torch_dtype=%s with `torch_dtype=torch.float16` due to "
231
+ "requirements of `bitsandbytes` to enable model loading in 8-bit or 4-bit. "
232
+ "Pass your own torch_dtype to specify the dtype of the remaining non-linear layers or pass"
233
+ " torch_dtype=torch.float16 to remove this warning.",
234
+ torch_dtype,
235
+ )
236
+ torch_dtype = torch.float16
237
+ return torch_dtype
238
+
239
+ def update_device_map(self, device_map):
240
+ if device_map is None:
241
+ if torch.xpu.is_available():
242
+ current_device = f"xpu:{torch.xpu.current_device()}"
243
+ elif torch.mps.is_available():
244
+ current_device = "mps"
245
+ else:
246
+ current_device = f"cuda:{torch.cuda.current_device()}"
247
+ device_map = {"": current_device}
248
+ logger.info(
249
+ "The device_map was not initialized. "
250
+ "Setting device_map to {"
251
+ ": {current_device}}. "
252
+ "If you want to use the model for inference, please set device_map ='auto' "
253
+ )
254
+ return device_map
255
+
256
+ def _process_model_before_weight_loading(
257
+ self,
258
+ model: "ModelMixin",
259
+ device_map,
260
+ keep_in_fp32_modules: list[str] = [],
261
+ **kwargs,
262
+ ):
263
+ from .utils import replace_with_bnb_linear
264
+
265
+ load_in_8bit_fp32_cpu_offload = self.quantization_config.llm_int8_enable_fp32_cpu_offload
266
+
267
+ # We may keep some modules such as the `proj_out` in their original dtype for numerical stability reasons
268
+ self.modules_to_not_convert = self.quantization_config.llm_int8_skip_modules
269
+
270
+ if not isinstance(self.modules_to_not_convert, list):
271
+ self.modules_to_not_convert = [self.modules_to_not_convert]
272
+
273
+ self.modules_to_not_convert.extend(keep_in_fp32_modules)
274
+
275
+ # Extend `self.modules_to_not_convert` to keys that are supposed to be offloaded to `cpu` or `disk`
276
+ if isinstance(device_map, dict) and len(device_map.keys()) > 1:
277
+ keys_on_cpu = [key for key, value in device_map.items() if value in ["disk", "cpu"]]
278
+
279
+ if len(keys_on_cpu) > 0 and not load_in_8bit_fp32_cpu_offload:
280
+ raise ValueError(
281
+ "If you want to offload some keys to `cpu` or `disk`, you need to set "
282
+ "`llm_int8_enable_fp32_cpu_offload=True`. Note that these modules will not be "
283
+ " converted to 8-bit but kept in 32-bit."
284
+ )
285
+ self.modules_to_not_convert.extend(keys_on_cpu)
286
+
287
+ # Purge `None`.
288
+ # Unlike `transformers`, we don't know if we should always keep certain modules in FP32
289
+ # in case of diffusion transformer models. For language models and others alike, `lm_head`
290
+ # and tied modules are usually kept in FP32.
291
+ self.modules_to_not_convert = [module for module in self.modules_to_not_convert if module is not None]
292
+
293
+ model = replace_with_bnb_linear(
294
+ model, modules_to_not_convert=self.modules_to_not_convert, quantization_config=self.quantization_config
295
+ )
296
+ model.config.quantization_config = self.quantization_config
297
+ model.is_loaded_in_4bit = True
298
+
299
+ def _process_model_after_weight_loading(self, model: "ModelMixin", **kwargs):
300
+ model.is_4bit_serializable = self.is_serializable
301
+ return model
302
+
303
+ @property
304
+ def is_serializable(self):
305
+ # Because we're mandating `bitsandbytes` 0.43.3.
306
+ return True
307
+
308
+ @property
309
+ def is_trainable(self) -> bool:
310
+ # Because we're mandating `bitsandbytes` 0.43.3.
311
+ return True
312
+
313
+ def _dequantize(self, model):
314
+ from .utils import dequantize_and_replace
315
+
316
+ is_model_on_cpu = model.device.type == "cpu"
317
+ if is_model_on_cpu:
318
+ logger.info(
319
+ "Model was found to be on CPU (could happen as a result of `enable_model_cpu_offload()`). So, moving it to GPU. After dequantization, will move the model back to CPU again to preserve the previous device."
320
+ )
321
+ if torch.xpu.is_available():
322
+ model.to(torch.xpu.current_device())
323
+ else:
324
+ model.to(torch.cuda.current_device())
325
+
326
+ model = dequantize_and_replace(
327
+ model, self.modules_to_not_convert, quantization_config=self.quantization_config
328
+ )
329
+ if is_model_on_cpu:
330
+ model.to("cpu")
331
+ return model
332
+
333
+
334
+ class BnB8BitDiffusersQuantizer(DiffusersQuantizer):
335
+ """
336
+ 8-bit quantization from bitsandbytes quantization method:
337
+ before loading: converts transformer layers into Linear8bitLt during loading: load 16bit weight and pass to the
338
+ layer object after: quantizes individual weights in Linear8bitLt into 8bit at fitst .cuda() call
339
+ saving:
340
+ from state dict, as usual; saves weights and 'SCB' component
341
+ loading:
342
+ need to locate SCB component and pass to the Linear8bitLt object
343
+ """
344
+
345
+ use_keep_in_fp32_modules = True
346
+ requires_calibration = False
347
+
348
+ def __init__(self, quantization_config, **kwargs):
349
+ super().__init__(quantization_config, **kwargs)
350
+
351
+ if self.quantization_config.llm_int8_skip_modules is not None:
352
+ self.modules_to_not_convert = self.quantization_config.llm_int8_skip_modules
353
+
354
+ def validate_environment(self, *args, **kwargs):
355
+ if not (torch.cuda.is_available() or torch.xpu.is_available()):
356
+ raise RuntimeError("No GPU found. A GPU is needed for quantization.")
357
+ if not is_accelerate_available() or is_accelerate_version("<", "0.26.0"):
358
+ raise ImportError(
359
+ "Using `bitsandbytes` 8-bit quantization requires Accelerate: `pip install 'accelerate>=0.26.0'`"
360
+ )
361
+ if not is_bitsandbytes_available() or is_bitsandbytes_version("<", "0.43.3"):
362
+ raise ImportError(
363
+ "Using `bitsandbytes` 8-bit quantization requires the latest version of bitsandbytes: `pip install -U bitsandbytes`"
364
+ )
365
+
366
+ if kwargs.get("from_flax", False):
367
+ raise ValueError(
368
+ "Converting into 8-bit weights from flax weights is currently not supported, please make"
369
+ " sure the weights are in PyTorch format."
370
+ )
371
+
372
+ device_map = kwargs.get("device_map", None)
373
+ if (
374
+ device_map is not None
375
+ and isinstance(device_map, dict)
376
+ and not self.quantization_config.llm_int8_enable_fp32_cpu_offload
377
+ ):
378
+ device_map_without_no_convert = {
379
+ key: device_map[key] for key in device_map.keys() if key not in self.modules_to_not_convert
380
+ }
381
+ if "cpu" in device_map_without_no_convert.values() or "disk" in device_map_without_no_convert.values():
382
+ raise ValueError(
383
+ "Some modules are dispatched on the CPU or the disk. Make sure you have enough GPU RAM to fit the "
384
+ "quantized model. If you want to dispatch the model on the CPU or the disk while keeping these modules "
385
+ "in 32-bit, you need to set `load_in_8bit_fp32_cpu_offload=True` and pass a custom `device_map` to "
386
+ "`from_pretrained`. Check "
387
+ "https://huggingface.co/docs/transformers/main/en/main_classes/quantization#offload-between-cpu-and-gpu "
388
+ "for more details. "
389
+ )
390
+
391
+ # Copied from diffusers.quantizers.bitsandbytes.bnb_quantizer.BnB4BitDiffusersQuantizer.adjust_max_memory
392
+ def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | str]:
393
+ # need more space for buffers that are created during quantization
394
+ max_memory = {key: val * 0.90 for key, val in max_memory.items()}
395
+ return max_memory
396
+
397
+ # Copied from diffusers.quantizers.bitsandbytes.bnb_quantizer.BnB4BitDiffusersQuantizer.update_torch_dtype
398
+ def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":
399
+ if torch_dtype is None:
400
+ # We force the `dtype` to be float16, this is a requirement from `bitsandbytes`
401
+ logger.info(
402
+ "Overriding torch_dtype=%s with `torch_dtype=torch.float16` due to "
403
+ "requirements of `bitsandbytes` to enable model loading in 8-bit or 4-bit. "
404
+ "Pass your own torch_dtype to specify the dtype of the remaining non-linear layers or pass"
405
+ " torch_dtype=torch.float16 to remove this warning.",
406
+ torch_dtype,
407
+ )
408
+ torch_dtype = torch.float16
409
+ return torch_dtype
410
+
411
+ # Copied from diffusers.quantizers.bitsandbytes.bnb_quantizer.BnB4BitDiffusersQuantizer.update_device_map
412
+ def update_device_map(self, device_map):
413
+ if device_map is None:
414
+ if torch.xpu.is_available():
415
+ current_device = f"xpu:{torch.xpu.current_device()}"
416
+ elif torch.mps.is_available():
417
+ current_device = "mps"
418
+ else:
419
+ current_device = f"cuda:{torch.cuda.current_device()}"
420
+ device_map = {"": current_device}
421
+ logger.info(
422
+ "The device_map was not initialized. "
423
+ "Setting device_map to {"
424
+ ": {current_device}}. "
425
+ "If you want to use the model for inference, please set device_map ='auto' "
426
+ )
427
+ return device_map
428
+
429
+ def adjust_target_dtype(self, target_dtype: "torch.dtype") -> "torch.dtype":
430
+ if target_dtype != torch.int8:
431
+ logger.info("target_dtype {target_dtype} is replaced by `torch.int8` for 8-bit BnB quantization")
432
+ return torch.int8
433
+
434
+ def check_if_quantized_param(
435
+ self,
436
+ model: "ModelMixin",
437
+ param_value: "torch.Tensor",
438
+ param_name: str,
439
+ state_dict: dict[str, Any],
440
+ **kwargs,
441
+ ):
442
+ import bitsandbytes as bnb
443
+
444
+ module, tensor_name = get_module_from_name(model, param_name)
445
+ if isinstance(module._parameters.get(tensor_name, None), bnb.nn.Int8Params):
446
+ if self.pre_quantized:
447
+ if param_name.replace("weight", "SCB") not in state_dict.keys():
448
+ raise ValueError("Missing quantization component `SCB`")
449
+ if param_value.dtype != torch.int8:
450
+ raise ValueError(
451
+ f"Incompatible dtype `{param_value.dtype}` when loading 8-bit prequantized weight. Expected `torch.int8`."
452
+ )
453
+ return True
454
+ return False
455
+
456
+ def create_quantized_param(
457
+ self,
458
+ model: "ModelMixin",
459
+ param_value: "torch.Tensor",
460
+ param_name: str,
461
+ target_device: "torch.device",
462
+ state_dict: dict[str, Any],
463
+ unexpected_keys: list[str] | None = None,
464
+ **kwargs,
465
+ ):
466
+ import bitsandbytes as bnb
467
+
468
+ fp16_statistics_key = param_name.replace("weight", "SCB")
469
+ fp16_weights_format_key = param_name.replace("weight", "weight_format")
470
+
471
+ fp16_statistics = state_dict.get(fp16_statistics_key, None)
472
+ fp16_weights_format = state_dict.get(fp16_weights_format_key, None)
473
+
474
+ module, tensor_name = get_module_from_name(model, param_name)
475
+ if tensor_name not in module._parameters:
476
+ raise ValueError(f"{module} does not have a parameter or a buffer named {tensor_name}.")
477
+
478
+ old_value = getattr(module, tensor_name)
479
+
480
+ if not isinstance(module._parameters[tensor_name], bnb.nn.Int8Params):
481
+ raise ValueError(f"Parameter `{tensor_name}` should only be a `bnb.nn.Int8Params` instance.")
482
+ if (
483
+ old_value.device == torch.device("meta")
484
+ and target_device not in ["meta", torch.device("meta")]
485
+ and param_value is None
486
+ ):
487
+ raise ValueError(f"{tensor_name} is on the meta device, we need a `value` to put in on {target_device}.")
488
+
489
+ new_value = param_value.to("cpu")
490
+ if self.pre_quantized and not self.is_serializable:
491
+ raise ValueError(
492
+ "Detected int8 weights but the version of bitsandbytes is not compatible with int8 serialization. "
493
+ "Make sure to download the latest `bitsandbytes` version. `pip install --upgrade bitsandbytes`."
494
+ )
495
+
496
+ kwargs = old_value.__dict__
497
+ new_value = bnb.nn.Int8Params(new_value, requires_grad=False, **kwargs).to(target_device)
498
+
499
+ module._parameters[tensor_name] = new_value
500
+ if fp16_statistics is not None:
501
+ setattr(module.weight, "SCB", fp16_statistics.to(target_device))
502
+ if unexpected_keys is not None:
503
+ unexpected_keys.remove(fp16_statistics_key)
504
+
505
+ # We just need to pop the `weight_format` keys from the state dict to remove unneeded
506
+ # messages. The correct format is correctly retrieved during the first forward pass.
507
+ if fp16_weights_format is not None and unexpected_keys is not None:
508
+ unexpected_keys.remove(fp16_weights_format_key)
509
+
510
+ # Copied from diffusers.quantizers.bitsandbytes.bnb_quantizer.BnB4BitDiffusersQuantizer._process_model_after_weight_loading with 4bit->8bit
511
+ def _process_model_after_weight_loading(self, model: "ModelMixin", **kwargs):
512
+ model.is_8bit_serializable = self.is_serializable
513
+ return model
514
+
515
+ # Copied from diffusers.quantizers.bitsandbytes.bnb_quantizer.BnB4BitDiffusersQuantizer._process_model_before_weight_loading with 4bit->8bit
516
+ def _process_model_before_weight_loading(
517
+ self,
518
+ model: "ModelMixin",
519
+ device_map,
520
+ keep_in_fp32_modules: list[str] = [],
521
+ **kwargs,
522
+ ):
523
+ from .utils import replace_with_bnb_linear
524
+
525
+ load_in_8bit_fp32_cpu_offload = self.quantization_config.llm_int8_enable_fp32_cpu_offload
526
+
527
+ # We may keep some modules such as the `proj_out` in their original dtype for numerical stability reasons
528
+ self.modules_to_not_convert = self.quantization_config.llm_int8_skip_modules
529
+
530
+ if not isinstance(self.modules_to_not_convert, list):
531
+ self.modules_to_not_convert = [self.modules_to_not_convert]
532
+
533
+ self.modules_to_not_convert.extend(keep_in_fp32_modules)
534
+
535
+ # Extend `self.modules_to_not_convert` to keys that are supposed to be offloaded to `cpu` or `disk`
536
+ if isinstance(device_map, dict) and len(device_map.keys()) > 1:
537
+ keys_on_cpu = [key for key, value in device_map.items() if value in ["disk", "cpu"]]
538
+
539
+ if len(keys_on_cpu) > 0 and not load_in_8bit_fp32_cpu_offload:
540
+ raise ValueError(
541
+ "If you want to offload some keys to `cpu` or `disk`, you need to set "
542
+ "`llm_int8_enable_fp32_cpu_offload=True`. Note that these modules will not be "
543
+ " converted to 8-bit but kept in 32-bit."
544
+ )
545
+ self.modules_to_not_convert.extend(keys_on_cpu)
546
+
547
+ # Purge `None`.
548
+ # Unlike `transformers`, we don't know if we should always keep certain modules in FP32
549
+ # in case of diffusion transformer models. For language models and others alike, `lm_head`
550
+ # and tied modules are usually kept in FP32.
551
+ self.modules_to_not_convert = [module for module in self.modules_to_not_convert if module is not None]
552
+
553
+ model = replace_with_bnb_linear(
554
+ model, modules_to_not_convert=self.modules_to_not_convert, quantization_config=self.quantization_config
555
+ )
556
+ model.config.quantization_config = self.quantization_config
557
+ model.is_loaded_in_8bit = True
558
+
559
+ @property
560
+ # Copied from diffusers.quantizers.bitsandbytes.bnb_quantizer.BnB4BitDiffusersQuantizer.is_serializable
561
+ def is_serializable(self):
562
+ # Because we're mandating `bitsandbytes` 0.43.3.
563
+ return True
564
+
565
+ @property
566
+ # Copied from diffusers.quantizers.bitsandbytes.bnb_quantizer.BnB4BitDiffusersQuantizer.is_serializable
567
+ def is_trainable(self) -> bool:
568
+ # Because we're mandating `bitsandbytes` 0.43.3.
569
+ return True
570
+
571
+ @property
572
+ def is_compileable(self) -> bool:
573
+ return True
574
+
575
+ def _dequantize(self, model):
576
+ from .utils import dequantize_and_replace
577
+
578
+ model = dequantize_and_replace(
579
+ model, self.modules_to_not_convert, quantization_config=self.quantization_config
580
+ )
581
+ return model
western_ai/europe/diffusers/src/diffusers/quantizers/bitsandbytes/utils.py ADDED
@@ -0,0 +1,318 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 The HuggingFace Inc. team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """
15
+ Adapted from
16
+ https://github.com/huggingface/transformers/blob/c409cd81777fb27aadc043ed3d8339dbc020fb3b/src/transformers/integrations/bitsandbytes.py
17
+ """
18
+
19
+ import inspect
20
+ from inspect import signature
21
+
22
+ from ...utils import is_accelerate_available, is_bitsandbytes_available, is_torch_available, logging
23
+ from ..quantization_config import QuantizationMethod
24
+
25
+
26
+ if is_torch_available():
27
+ import torch
28
+ import torch.nn as nn
29
+
30
+ if is_bitsandbytes_available():
31
+ import bitsandbytes as bnb
32
+
33
+ if is_accelerate_available():
34
+ import accelerate
35
+ from accelerate import init_empty_weights
36
+ from accelerate.hooks import add_hook_to_module, remove_hook_from_module
37
+
38
+ logger = logging.get_logger(__name__)
39
+
40
+
41
+ def _replace_with_bnb_linear(
42
+ model,
43
+ modules_to_not_convert=None,
44
+ current_key_name=None,
45
+ quantization_config=None,
46
+ has_been_replaced=False,
47
+ ):
48
+ """
49
+ Private method that wraps the recursion for module replacement.
50
+
51
+ Returns the converted model and a boolean that indicates if the conversion has been successful or not.
52
+ """
53
+ for name, module in model.named_children():
54
+ if current_key_name is None:
55
+ current_key_name = []
56
+ current_key_name.append(name)
57
+
58
+ if isinstance(module, nn.Linear) and name not in modules_to_not_convert:
59
+ # Check if the current key is not in the `modules_to_not_convert`
60
+ current_key_name_str = ".".join(current_key_name)
61
+ if not any(
62
+ (key + "." in current_key_name_str) or (key == current_key_name_str) for key in modules_to_not_convert
63
+ ):
64
+ with init_empty_weights():
65
+ in_features = module.in_features
66
+ out_features = module.out_features
67
+
68
+ if quantization_config.quantization_method() == "llm_int8":
69
+ model._modules[name] = bnb.nn.Linear8bitLt(
70
+ in_features,
71
+ out_features,
72
+ module.bias is not None,
73
+ has_fp16_weights=quantization_config.llm_int8_has_fp16_weight,
74
+ threshold=quantization_config.llm_int8_threshold,
75
+ )
76
+ has_been_replaced = True
77
+ else:
78
+ if (
79
+ quantization_config.llm_int8_skip_modules is not None
80
+ and name in quantization_config.llm_int8_skip_modules
81
+ ):
82
+ pass
83
+ else:
84
+ extra_kwargs = (
85
+ {"quant_storage": quantization_config.bnb_4bit_quant_storage}
86
+ if "quant_storage" in list(signature(bnb.nn.Linear4bit).parameters)
87
+ else {}
88
+ )
89
+ model._modules[name] = bnb.nn.Linear4bit(
90
+ in_features,
91
+ out_features,
92
+ module.bias is not None,
93
+ quantization_config.bnb_4bit_compute_dtype,
94
+ compress_statistics=quantization_config.bnb_4bit_use_double_quant,
95
+ quant_type=quantization_config.bnb_4bit_quant_type,
96
+ **extra_kwargs,
97
+ )
98
+ has_been_replaced = True
99
+ # Store the module class in case we need to transpose the weight later
100
+ model._modules[name].source_cls = type(module)
101
+ # Force requires grad to False to avoid unexpected errors
102
+ model._modules[name].requires_grad_(False)
103
+ if len(list(module.children())) > 0:
104
+ _, has_been_replaced = _replace_with_bnb_linear(
105
+ module,
106
+ modules_to_not_convert,
107
+ current_key_name,
108
+ quantization_config,
109
+ has_been_replaced=has_been_replaced,
110
+ )
111
+ # Remove the last key for recursion
112
+ current_key_name.pop(-1)
113
+ return model, has_been_replaced
114
+
115
+
116
+ def replace_with_bnb_linear(model, modules_to_not_convert=None, current_key_name=None, quantization_config=None):
117
+ """
118
+ Helper function to replace the `nn.Linear` layers within `model` with either `bnb.nn.Linear8bit` or
119
+ `bnb.nn.Linear4bit` using the `bitsandbytes` library.
120
+
121
+ References:
122
+ * `bnb.nn.Linear8bit`: [LLM.int8(): 8-bit Matrix Multiplication for Transformers at
123
+ Scale](https://huggingface.co/papers/2208.07339)
124
+ * `bnb.nn.Linear4bit`: [QLoRA: Efficient Finetuning of Quantized
125
+ LLMs](https://huggingface.co/papers/2305.14314)
126
+
127
+ Parameters:
128
+ model (`torch.nn.Module`):
129
+ Input model or `torch.nn.Module` as the function is run recursively.
130
+ modules_to_not_convert (`list[`str`]`, *optional*, defaults to `[]`):
131
+ Names of the modules to not convert in `Linear8bitLt`. In practice we keep the `modules_to_not_convert` in
132
+ full precision for numerical stability reasons.
133
+ current_key_name (`list[`str`]`, *optional*):
134
+ An array to track the current key of the recursion. This is used to check whether the current key (part of
135
+ it) is not in the list of modules to not convert (for instances modules that are offloaded to `cpu` or
136
+ `disk`).
137
+ quantization_config ('transformers.utils.quantization_config.BitsAndBytesConfig'):
138
+ To configure and manage settings related to quantization, a technique used to compress neural network
139
+ models by reducing the precision of the weights and activations, thus making models more efficient in terms
140
+ of both storage and computation.
141
+ """
142
+ model, _ = _replace_with_bnb_linear(model, modules_to_not_convert, current_key_name, quantization_config)
143
+
144
+ has_been_replaced = any(
145
+ isinstance(replaced_module, (bnb.nn.Linear4bit, bnb.nn.Linear8bitLt))
146
+ for _, replaced_module in model.named_modules()
147
+ )
148
+ if not has_been_replaced:
149
+ logger.warning(
150
+ "You are loading your model in 8bit or 4bit but no linear modules were found in your model."
151
+ " Please double check your model architecture, or submit an issue on github if you think this is"
152
+ " a bug."
153
+ )
154
+
155
+ return model
156
+
157
+
158
+ # Adapted from PEFT: https://github.com/huggingface/peft/blob/6d458b300fc2ed82e19f796b53af4c97d03ea604/src/peft/utils/integrations.py#L81
159
+ def dequantize_bnb_weight(weight: "torch.nn.Parameter", state=None, dtype: "torch.dtype" = None):
160
+ """
161
+ Helper function to dequantize 4bit or 8bit bnb weights.
162
+
163
+ If the weight is not a bnb quantized weight, it will be returned as is.
164
+ """
165
+ if not isinstance(weight, torch.nn.Parameter):
166
+ raise TypeError(f"Input weight should be of type nn.Parameter, got {type(weight)} instead")
167
+
168
+ cls_name = weight.__class__.__name__
169
+ if cls_name not in ("Params4bit", "Int8Params"):
170
+ return weight
171
+
172
+ if cls_name == "Params4bit":
173
+ output_tensor = bnb.functional.dequantize_4bit(weight.data, weight.quant_state)
174
+ msg = f"The model is going to be dequantized in {output_tensor.dtype} - if you want to upcast it to another dtype, make sure to pass the desired dtype when quantizing the model through `bnb_4bit_quant_type` argument of `BitsAndBytesConfig`"
175
+ if dtype:
176
+ msg = f"The model is going to be first dequantized in {output_tensor.dtype} and type-casted to {dtype}"
177
+ output_tensor = output_tensor.to(dtype)
178
+ logger.warning_once(msg)
179
+ return output_tensor
180
+
181
+ if state.SCB is None:
182
+ state.SCB = weight.SCB
183
+
184
+ if hasattr(bnb.functional, "int8_vectorwise_dequant"):
185
+ # Use bitsandbytes API if available (requires v0.45.0+)
186
+ dequantized = bnb.functional.int8_vectorwise_dequant(weight.data, state.SCB)
187
+ else:
188
+ # Multiply by (scale/127) to dequantize.
189
+ dequantized = weight.data * state.SCB.view(-1, 1) * 7.874015718698502e-3
190
+
191
+ if dtype:
192
+ dequantized = dequantized.to(dtype)
193
+ return dequantized
194
+
195
+
196
+ def _create_accelerate_new_hook(old_hook):
197
+ r"""
198
+ Creates a new hook based on the old hook. Use it only if you know what you are doing ! This method is a copy of:
199
+ https://github.com/huggingface/peft/blob/748f7968f3a31ec06a1c2b0328993319ad9a150a/src/peft/utils/other.py#L245 with
200
+ some changes
201
+ """
202
+ old_hook_cls = getattr(accelerate.hooks, old_hook.__class__.__name__)
203
+ old_hook_attr = old_hook.__dict__
204
+ filtered_old_hook_attr = {}
205
+ old_hook_init_signature = inspect.signature(old_hook_cls.__init__)
206
+ for k in old_hook_attr.keys():
207
+ if k in old_hook_init_signature.parameters:
208
+ filtered_old_hook_attr[k] = old_hook_attr[k]
209
+ new_hook = old_hook_cls(**filtered_old_hook_attr)
210
+ return new_hook
211
+
212
+
213
+ def _dequantize_and_replace(
214
+ model,
215
+ dtype,
216
+ modules_to_not_convert=None,
217
+ current_key_name=None,
218
+ quantization_config=None,
219
+ has_been_replaced=False,
220
+ ):
221
+ """
222
+ Converts a quantized model into its dequantized original version. The newly converted model will have some
223
+ performance drop compared to the original model before quantization - use it only for specific usecases such as
224
+ QLoRA adapters merging.
225
+
226
+ Returns the converted model and a boolean that indicates if the conversion has been successful or not.
227
+ """
228
+ quant_method = quantization_config.quantization_method()
229
+
230
+ target_cls = bnb.nn.Linear8bitLt if quant_method == "llm_int8" else bnb.nn.Linear4bit
231
+
232
+ for name, module in model.named_children():
233
+ if current_key_name is None:
234
+ current_key_name = []
235
+ current_key_name.append(name)
236
+
237
+ if isinstance(module, target_cls) and name not in modules_to_not_convert:
238
+ # Check if the current key is not in the `modules_to_not_convert`
239
+ current_key_name_str = ".".join(current_key_name)
240
+
241
+ if not any(
242
+ (key + "." in current_key_name_str) or (key == current_key_name_str) for key in modules_to_not_convert
243
+ ):
244
+ bias = getattr(module, "bias", None)
245
+
246
+ device = module.weight.device
247
+ with init_empty_weights():
248
+ new_module = torch.nn.Linear(module.in_features, module.out_features, bias=bias is not None)
249
+
250
+ if quant_method == "llm_int8":
251
+ state = module.state
252
+ else:
253
+ state = None
254
+
255
+ new_module.weight = torch.nn.Parameter(dequantize_bnb_weight(module.weight, state, dtype))
256
+
257
+ if bias is not None:
258
+ new_module.bias = bias
259
+
260
+ # Create a new hook and attach it in case we use accelerate
261
+ if hasattr(module, "_hf_hook"):
262
+ old_hook = module._hf_hook
263
+ new_hook = _create_accelerate_new_hook(old_hook)
264
+
265
+ remove_hook_from_module(module)
266
+ add_hook_to_module(new_module, new_hook)
267
+
268
+ new_module.to(device)
269
+ model._modules[name] = new_module
270
+ has_been_replaced = True
271
+ if len(list(module.children())) > 0:
272
+ _, has_been_replaced = _dequantize_and_replace(
273
+ module,
274
+ dtype=dtype,
275
+ modules_to_not_convert=modules_to_not_convert,
276
+ current_key_name=current_key_name,
277
+ quantization_config=quantization_config,
278
+ has_been_replaced=has_been_replaced,
279
+ )
280
+ # Remove the last key for recursion
281
+ current_key_name.pop(-1)
282
+ return model, has_been_replaced
283
+
284
+
285
+ def dequantize_and_replace(
286
+ model,
287
+ modules_to_not_convert=None,
288
+ quantization_config=None,
289
+ ):
290
+ model, _ = _dequantize_and_replace(
291
+ model,
292
+ dtype=model.dtype,
293
+ modules_to_not_convert=modules_to_not_convert,
294
+ quantization_config=quantization_config,
295
+ )
296
+ has_been_replaced = any(
297
+ isinstance(replaced_module, torch.nn.Linear) for _, replaced_module in model.named_modules()
298
+ )
299
+ if not has_been_replaced:
300
+ logger.warning(
301
+ "Some linear modules were not dequantized. This could lead to unexpected behaviour. Please check your model."
302
+ )
303
+
304
+ return model
305
+
306
+
307
+ def _check_bnb_status(module) -> bool | bool:
308
+ is_loaded_in_4bit_bnb = (
309
+ hasattr(module, "is_loaded_in_4bit")
310
+ and module.is_loaded_in_4bit
311
+ and getattr(module, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES
312
+ )
313
+ is_loaded_in_8bit_bnb = (
314
+ hasattr(module, "is_loaded_in_8bit")
315
+ and module.is_loaded_in_8bit
316
+ and getattr(module, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES
317
+ )
318
+ return is_loaded_in_4bit_bnb or is_loaded_in_8bit_bnb, is_loaded_in_4bit_bnb, is_loaded_in_8bit_bnb
western_ai/europe/diffusers/src/diffusers/quantizers/gguf/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ from .gguf_quantizer import GGUFQuantizer
western_ai/europe/diffusers/src/diffusers/quantizers/gguf/gguf_quantizer.py ADDED
@@ -0,0 +1,190 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from typing import TYPE_CHECKING, Any
4
+
5
+ from ..base import DiffusersQuantizer
6
+
7
+
8
+ if TYPE_CHECKING:
9
+ from ...models.modeling_utils import ModelMixin
10
+
11
+
12
+ from ...utils import (
13
+ get_module_from_name,
14
+ is_accelerate_available,
15
+ is_accelerate_version,
16
+ is_gguf_available,
17
+ is_gguf_version,
18
+ is_torch_available,
19
+ logging,
20
+ )
21
+
22
+
23
+ if is_torch_available() and is_gguf_available():
24
+ import torch
25
+
26
+ from .utils import (
27
+ GGML_QUANT_SIZES,
28
+ GGUFParameter,
29
+ _dequantize_gguf_and_restore_linear,
30
+ _quant_shape_from_byte_shape,
31
+ _replace_with_gguf_linear,
32
+ dequantize_gguf_tensor,
33
+ )
34
+
35
+
36
+ logger = logging.get_logger(__name__)
37
+
38
+
39
+ class GGUFQuantizer(DiffusersQuantizer):
40
+ use_keep_in_fp32_modules = True
41
+
42
+ def __init__(self, quantization_config, **kwargs):
43
+ super().__init__(quantization_config, **kwargs)
44
+
45
+ self.compute_dtype = quantization_config.compute_dtype
46
+ self.pre_quantized = quantization_config.pre_quantized
47
+ self.modules_to_not_convert = quantization_config.modules_to_not_convert or []
48
+
49
+ if not isinstance(self.modules_to_not_convert, list):
50
+ self.modules_to_not_convert = [self.modules_to_not_convert]
51
+
52
+ def validate_environment(self, *args, **kwargs):
53
+ if not is_accelerate_available() or is_accelerate_version("<", "0.26.0"):
54
+ raise ImportError(
55
+ "Loading GGUF Parameters requires `accelerate` installed in your environment: `pip install 'accelerate>=0.26.0'`"
56
+ )
57
+ if not is_gguf_available() or is_gguf_version("<", "0.10.0"):
58
+ raise ImportError(
59
+ "To load GGUF format files you must have `gguf` installed in your environment: `pip install gguf>=0.10.0`"
60
+ )
61
+
62
+ # Copied from diffusers.quantizers.bitsandbytes.bnb_quantizer.BnB4BitDiffusersQuantizer.adjust_max_memory
63
+ def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | str]:
64
+ # need more space for buffers that are created during quantization
65
+ max_memory = {key: val * 0.90 for key, val in max_memory.items()}
66
+ return max_memory
67
+
68
+ def adjust_target_dtype(self, target_dtype: "torch.dtype") -> "torch.dtype":
69
+ if target_dtype != torch.uint8:
70
+ logger.info(f"target_dtype {target_dtype} is replaced by `torch.uint8` for GGUF quantization")
71
+ return torch.uint8
72
+
73
+ def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":
74
+ if torch_dtype is None:
75
+ torch_dtype = self.compute_dtype
76
+ return torch_dtype
77
+
78
+ def check_quantized_param_shape(self, param_name, current_param, loaded_param):
79
+ loaded_param_shape = loaded_param.shape
80
+ current_param_shape = current_param.shape
81
+ quant_type = loaded_param.quant_type
82
+
83
+ block_size, type_size = GGML_QUANT_SIZES[quant_type]
84
+
85
+ inferred_shape = _quant_shape_from_byte_shape(loaded_param_shape, type_size, block_size)
86
+ if inferred_shape != current_param_shape:
87
+ raise ValueError(
88
+ f"{param_name} has an expected shape of: {current_param_shape}, but the loaded GGUF weight decodes "
89
+ f"to shape: {inferred_shape}"
90
+ )
91
+
92
+ return True
93
+
94
+ def check_if_quantized_param(
95
+ self,
96
+ model: "ModelMixin",
97
+ param_value: "GGUFParameter" | "torch.Tensor",
98
+ param_name: str,
99
+ state_dict: dict[str, Any],
100
+ **kwargs,
101
+ ) -> bool:
102
+ if isinstance(param_value, GGUFParameter):
103
+ return True
104
+
105
+ return False
106
+
107
+ def create_quantized_param(
108
+ self,
109
+ model: "ModelMixin",
110
+ param_value: "GGUFParameter" | "torch.Tensor",
111
+ param_name: str,
112
+ target_device: "torch.device",
113
+ state_dict: dict[str, Any] | None = None,
114
+ unexpected_keys: list[str] | None = None,
115
+ **kwargs,
116
+ ):
117
+ module, tensor_name = get_module_from_name(model, param_name)
118
+ if tensor_name not in module._parameters and tensor_name not in module._buffers:
119
+ raise ValueError(f"{module} does not have a parameter or a buffer named {tensor_name}.")
120
+
121
+ # If the GGUFParameter should not be quantized (for example, it is a submodule of any excluded module),
122
+ # dequantize it and set the (dequantized) parameter to the proper dtype.
123
+ if isinstance(param_value, GGUFParameter) and any(
124
+ m in param_name.split(".") for m in self.modules_to_not_convert
125
+ ):
126
+ keep_in_fp32 = getattr(self, "keep_in_fp32_modules", [])
127
+ param_should_be_fp32 = any(m in param_name.split(".") for m in keep_in_fp32)
128
+ target_dtype = torch.float32 if param_should_be_fp32 else self.compute_dtype
129
+ if param_should_be_fp32:
130
+ logger.warning(f"Quantized parameter {param_name} is required to remain in FP32, dequantizing now.")
131
+ else:
132
+ logger.warning(
133
+ f"Quantized parameter {param_name} is excluded by `modules_to_not_convert`, dequantizing now."
134
+ )
135
+ param_value = dequantize_gguf_tensor(param_value).to(target_dtype)
136
+
137
+ if tensor_name in module._parameters:
138
+ module._parameters[tensor_name] = param_value.to(target_device)
139
+ if tensor_name in module._buffers:
140
+ module._buffers[tensor_name] = param_value.to(target_device)
141
+
142
+ def _process_model_before_weight_loading(
143
+ self,
144
+ model: "ModelMixin",
145
+ device_map,
146
+ keep_in_fp32_modules: list[str] = [],
147
+ **kwargs,
148
+ ):
149
+ state_dict = kwargs.get("state_dict", None)
150
+
151
+ self.keep_in_fp32_modules = [module for module in keep_in_fp32_modules if module is not None]
152
+ self.modules_to_not_convert.extend(self.keep_in_fp32_modules)
153
+ self.modules_to_not_convert = [module for module in self.modules_to_not_convert if module is not None]
154
+
155
+ _replace_with_gguf_linear(
156
+ model, self.compute_dtype, state_dict, modules_to_not_convert=self.modules_to_not_convert
157
+ )
158
+
159
+ def _process_model_after_weight_loading(self, model: "ModelMixin", **kwargs):
160
+ return model
161
+
162
+ @property
163
+ def is_serializable(self):
164
+ return False
165
+
166
+ @property
167
+ def is_trainable(self) -> bool:
168
+ return False
169
+
170
+ @property
171
+ def is_compileable(self) -> bool:
172
+ return True
173
+
174
+ def _dequantize(self, model):
175
+ is_model_on_cpu = model.device.type == "cpu"
176
+ if is_model_on_cpu:
177
+ logger.info(
178
+ "Model was found to be on CPU (could happen as a result of `enable_model_cpu_offload()`). So, moving it to accelerator. After dequantization, will move the model back to CPU again to preserve the previous device."
179
+ )
180
+ device = (
181
+ torch.accelerator.current_accelerator()
182
+ if hasattr(torch, "accelerator")
183
+ else torch.cuda.current_device()
184
+ )
185
+ model.to(device)
186
+
187
+ model = _dequantize_gguf_and_restore_linear(model, self.modules_to_not_convert)
188
+ if is_model_on_cpu:
189
+ model.to("cpu")
190
+ return model
western_ai/europe/diffusers/src/diffusers/quantizers/gguf/utils.py ADDED
@@ -0,0 +1,612 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 The HuggingFace Team and City96. All rights reserved.
2
+ # #
3
+ # # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # # you may not use this file except in compliance with the License.
5
+ # # You may obtain a copy of the License at
6
+ # #
7
+ # # http://www.apache.org/licenses/LICENSE-2.0
8
+ # #
9
+ # # Unless required by applicable law or agreed to in writing, software
10
+ # # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # # See the License for the specific language governing permissions and
13
+ # # limitations under the License.
14
+
15
+ import inspect
16
+ import os
17
+ from contextlib import nullcontext
18
+
19
+ import gguf
20
+ import torch
21
+ import torch.nn as nn
22
+
23
+ from ...utils import is_accelerate_available, is_kernels_available
24
+
25
+
26
+ if is_accelerate_available():
27
+ import accelerate
28
+ from accelerate import init_empty_weights
29
+ from accelerate.hooks import add_hook_to_module, remove_hook_from_module
30
+
31
+
32
+ can_use_cuda_kernels = (
33
+ os.getenv("DIFFUSERS_GGUF_CUDA_KERNELS", "false").lower() in ["1", "true", "yes"]
34
+ and torch.cuda.is_available()
35
+ and torch.cuda.get_device_capability()[0] >= 7
36
+ )
37
+ if can_use_cuda_kernels and is_kernels_available():
38
+ from kernels import get_kernel
39
+
40
+ ops = get_kernel("Isotr0py/ggml")
41
+ else:
42
+ ops = None
43
+
44
+ UNQUANTIZED_TYPES = {gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16, gguf.GGMLQuantizationType.BF16}
45
+ STANDARD_QUANT_TYPES = {
46
+ gguf.GGMLQuantizationType.Q4_0,
47
+ gguf.GGMLQuantizationType.Q4_1,
48
+ gguf.GGMLQuantizationType.Q5_0,
49
+ gguf.GGMLQuantizationType.Q5_1,
50
+ gguf.GGMLQuantizationType.Q8_0,
51
+ gguf.GGMLQuantizationType.Q8_1,
52
+ }
53
+ KQUANT_TYPES = {
54
+ gguf.GGMLQuantizationType.Q2_K,
55
+ gguf.GGMLQuantizationType.Q3_K,
56
+ gguf.GGMLQuantizationType.Q4_K,
57
+ gguf.GGMLQuantizationType.Q5_K,
58
+ gguf.GGMLQuantizationType.Q6_K,
59
+ }
60
+ IMATRIX_QUANT_TYPES = {
61
+ gguf.GGMLQuantizationType.IQ1_M,
62
+ gguf.GGMLQuantizationType.IQ1_S,
63
+ gguf.GGMLQuantizationType.IQ2_XXS,
64
+ gguf.GGMLQuantizationType.IQ2_XS,
65
+ gguf.GGMLQuantizationType.IQ2_S,
66
+ gguf.GGMLQuantizationType.IQ3_XXS,
67
+ gguf.GGMLQuantizationType.IQ3_S,
68
+ gguf.GGMLQuantizationType.IQ4_XS,
69
+ gguf.GGMLQuantizationType.IQ4_NL,
70
+ }
71
+ # TODO(Isotr0py): Currently, we don't have MMQ kernel for I-Matrix quantization.
72
+ # Consolidate DEQUANT_TYPES, MMVQ_QUANT_TYPES and MMQ_QUANT_TYPES after we add
73
+ # MMQ kernel for I-Matrix quantization.
74
+ DEQUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES | IMATRIX_QUANT_TYPES
75
+ MMVQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES | IMATRIX_QUANT_TYPES
76
+ MMQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES
77
+
78
+
79
+ def _fused_mul_mat_gguf(x: torch.Tensor, qweight: torch.Tensor, qweight_type: int) -> torch.Tensor:
80
+ # there is no need to call any kernel for fp16/bf16
81
+ if qweight_type in UNQUANTIZED_TYPES:
82
+ weight = dequantize_gguf_tensor(qweight)
83
+ return x @ weight.to(x.dtype).T
84
+
85
+ # TODO(Isotr0py): GGUF's MMQ and MMVQ implementation are designed for
86
+ # contiguous batching and inefficient with diffusers' batching,
87
+ # so we disabled it now.
88
+
89
+ # elif qweight_type in MMVQ_QUANT_TYPES:
90
+ # y = ops.ggml_mul_mat_vec_a8(qweight, x, qweight_type, qweight.shape[0])
91
+ # elif qweight_type in MMQ_QUANT_TYPES:
92
+ # y = ops.ggml_mul_mat_a8(qweight, x, qweight_type, qweight.shape[0])
93
+
94
+ # If there is no available MMQ kernel, fallback to dequantize
95
+ if qweight_type in DEQUANT_TYPES:
96
+ block_size, type_size = gguf.GGML_QUANT_SIZES[qweight_type]
97
+ shape = (qweight.shape[0], qweight.shape[1] // type_size * block_size)
98
+ weight = ops.ggml_dequantize(qweight, qweight_type, *shape)
99
+ y = x @ weight.to(x.dtype).T
100
+ else:
101
+ # Raise an error if the quantization type is not supported.
102
+ # Might be useful if llama.cpp adds a new quantization type.
103
+ # Wrap to GGMLQuantizationType IntEnum to make sure it's a valid type.
104
+ qweight_type = gguf.GGMLQuantizationType(qweight_type)
105
+ raise NotImplementedError(f"Unsupported GGUF quantization type: {qweight_type}")
106
+ return y.as_tensor()
107
+
108
+
109
+ # Copied from diffusers.quantizers.bitsandbytes.utils._create_accelerate_new_hook
110
+ def _create_accelerate_new_hook(old_hook):
111
+ r"""
112
+ Creates a new hook based on the old hook. Use it only if you know what you are doing ! This method is a copy of:
113
+ https://github.com/huggingface/peft/blob/748f7968f3a31ec06a1c2b0328993319ad9a150a/src/peft/utils/other.py#L245 with
114
+ some changes
115
+ """
116
+ old_hook_cls = getattr(accelerate.hooks, old_hook.__class__.__name__)
117
+ old_hook_attr = old_hook.__dict__
118
+ filtered_old_hook_attr = {}
119
+ old_hook_init_signature = inspect.signature(old_hook_cls.__init__)
120
+ for k in old_hook_attr.keys():
121
+ if k in old_hook_init_signature.parameters:
122
+ filtered_old_hook_attr[k] = old_hook_attr[k]
123
+ new_hook = old_hook_cls(**filtered_old_hook_attr)
124
+ return new_hook
125
+
126
+
127
+ def _replace_with_gguf_linear(model, compute_dtype, state_dict, prefix="", modules_to_not_convert=[]):
128
+ def _should_convert_to_gguf(state_dict, prefix):
129
+ weight_key = prefix + "weight"
130
+ return weight_key in state_dict and isinstance(state_dict[weight_key], GGUFParameter)
131
+
132
+ has_children = list(model.children())
133
+ if not has_children:
134
+ return
135
+
136
+ for name, module in model.named_children():
137
+ if name in modules_to_not_convert:
138
+ continue
139
+ module_prefix = prefix + name + "."
140
+ _replace_with_gguf_linear(module, compute_dtype, state_dict, module_prefix, modules_to_not_convert)
141
+
142
+ if (
143
+ isinstance(module, nn.Linear)
144
+ and _should_convert_to_gguf(state_dict, module_prefix)
145
+ and name not in modules_to_not_convert
146
+ ):
147
+ ctx = init_empty_weights if is_accelerate_available() else nullcontext
148
+ with ctx():
149
+ model._modules[name] = GGUFLinear(
150
+ module.in_features,
151
+ module.out_features,
152
+ module.bias is not None,
153
+ compute_dtype=compute_dtype,
154
+ )
155
+ model._modules[name].source_cls = type(module)
156
+ # Force requires_grad to False to avoid unexpected errors
157
+ model._modules[name].requires_grad_(False)
158
+
159
+ return model
160
+
161
+
162
+ def _dequantize_gguf_and_restore_linear(model, modules_to_not_convert=[]):
163
+ for name, module in model.named_children():
164
+ if isinstance(module, GGUFLinear) and name not in modules_to_not_convert:
165
+ device = module.weight.device
166
+ bias = getattr(module, "bias", None)
167
+
168
+ ctx = init_empty_weights if is_accelerate_available() else nullcontext
169
+ with ctx():
170
+ new_module = nn.Linear(
171
+ module.in_features,
172
+ module.out_features,
173
+ module.bias is not None,
174
+ device=device,
175
+ )
176
+ new_module.weight = nn.Parameter(dequantize_gguf_tensor(module.weight))
177
+ if bias is not None:
178
+ new_module.bias = bias
179
+
180
+ # Create a new hook and attach it in case we use accelerate
181
+ if hasattr(module, "_hf_hook"):
182
+ old_hook = module._hf_hook
183
+ new_hook = _create_accelerate_new_hook(old_hook)
184
+
185
+ remove_hook_from_module(module)
186
+ add_hook_to_module(new_module, new_hook)
187
+
188
+ new_module.to(device)
189
+ model._modules[name] = new_module
190
+
191
+ has_children = list(module.children())
192
+ if has_children:
193
+ _dequantize_gguf_and_restore_linear(module, modules_to_not_convert)
194
+
195
+ return model
196
+
197
+
198
+ # dequantize operations based on torch ports of GGUF dequantize_functions
199
+ # from City96
200
+ # more info: https://github.com/city96/ComfyUI-GGUF/blob/main/dequant.py
201
+
202
+
203
+ QK_K = 256
204
+ K_SCALE_SIZE = 12
205
+
206
+
207
+ def to_uint32(x):
208
+ x = x.view(torch.uint8).to(torch.int32)
209
+ return (x[:, 0] | x[:, 1] << 8 | x[:, 2] << 16 | x[:, 3] << 24).unsqueeze(1)
210
+
211
+
212
+ def split_block_dims(blocks, *args):
213
+ n_max = blocks.shape[1]
214
+ dims = list(args) + [n_max - sum(args)]
215
+ return torch.split(blocks, dims, dim=1)
216
+
217
+
218
+ def get_scale_min(scales):
219
+ n_blocks = scales.shape[0]
220
+ scales = scales.view(torch.uint8)
221
+ scales = scales.reshape((n_blocks, 3, 4))
222
+
223
+ d, m, m_d = torch.split(scales, scales.shape[-2] // 3, dim=-2)
224
+
225
+ sc = torch.cat([d & 0x3F, (m_d & 0x0F) | ((d >> 2) & 0x30)], dim=-1)
226
+ min = torch.cat([m & 0x3F, (m_d >> 4) | ((m >> 2) & 0x30)], dim=-1)
227
+
228
+ return (sc.reshape((n_blocks, 8)), min.reshape((n_blocks, 8)))
229
+
230
+
231
+ def dequantize_blocks_Q8_0(blocks, block_size, type_size, dtype=None):
232
+ d, x = split_block_dims(blocks, 2)
233
+ d = d.view(torch.float16).to(dtype)
234
+ x = x.view(torch.int8)
235
+ return d * x
236
+
237
+
238
+ def dequantize_blocks_Q5_1(blocks, block_size, type_size, dtype=None):
239
+ n_blocks = blocks.shape[0]
240
+
241
+ d, m, qh, qs = split_block_dims(blocks, 2, 2, 4)
242
+ d = d.view(torch.float16).to(dtype)
243
+ m = m.view(torch.float16).to(dtype)
244
+ qh = to_uint32(qh)
245
+
246
+ qh = qh.reshape((n_blocks, 1)) >> torch.arange(32, device=d.device, dtype=torch.int32).reshape(1, 32)
247
+ ql = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor(
248
+ [0, 4], device=d.device, dtype=torch.uint8
249
+ ).reshape(1, 1, 2, 1)
250
+ qh = (qh & 1).to(torch.uint8)
251
+ ql = (ql & 0x0F).reshape((n_blocks, -1))
252
+
253
+ qs = ql | (qh << 4)
254
+ return (d * qs) + m
255
+
256
+
257
+ def dequantize_blocks_Q5_0(blocks, block_size, type_size, dtype=None):
258
+ n_blocks = blocks.shape[0]
259
+
260
+ d, qh, qs = split_block_dims(blocks, 2, 4)
261
+ d = d.view(torch.float16).to(dtype)
262
+ qh = to_uint32(qh)
263
+
264
+ qh = qh.reshape(n_blocks, 1) >> torch.arange(32, device=d.device, dtype=torch.int32).reshape(1, 32)
265
+ ql = qs.reshape(n_blocks, -1, 1, block_size // 2) >> torch.tensor(
266
+ [0, 4], device=d.device, dtype=torch.uint8
267
+ ).reshape(1, 1, 2, 1)
268
+
269
+ qh = (qh & 1).to(torch.uint8)
270
+ ql = (ql & 0x0F).reshape(n_blocks, -1)
271
+
272
+ qs = (ql | (qh << 4)).to(torch.int8) - 16
273
+ return d * qs
274
+
275
+
276
+ def dequantize_blocks_Q4_1(blocks, block_size, type_size, dtype=None):
277
+ n_blocks = blocks.shape[0]
278
+
279
+ d, m, qs = split_block_dims(blocks, 2, 2)
280
+ d = d.view(torch.float16).to(dtype)
281
+ m = m.view(torch.float16).to(dtype)
282
+
283
+ qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor(
284
+ [0, 4], device=d.device, dtype=torch.uint8
285
+ ).reshape(1, 1, 2, 1)
286
+ qs = (qs & 0x0F).reshape(n_blocks, -1)
287
+
288
+ return (d * qs) + m
289
+
290
+
291
+ def dequantize_blocks_Q4_0(blocks, block_size, type_size, dtype=None):
292
+ n_blocks = blocks.shape[0]
293
+
294
+ d, qs = split_block_dims(blocks, 2)
295
+ d = d.view(torch.float16).to(dtype)
296
+
297
+ qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor(
298
+ [0, 4], device=d.device, dtype=torch.uint8
299
+ ).reshape((1, 1, 2, 1))
300
+ qs = (qs & 0x0F).reshape((n_blocks, -1)).to(torch.int8) - 8
301
+ return d * qs
302
+
303
+
304
+ def dequantize_blocks_Q6_K(blocks, block_size, type_size, dtype=None):
305
+ n_blocks = blocks.shape[0]
306
+
307
+ (
308
+ ql,
309
+ qh,
310
+ scales,
311
+ d,
312
+ ) = split_block_dims(blocks, QK_K // 2, QK_K // 4, QK_K // 16)
313
+
314
+ scales = scales.view(torch.int8).to(dtype)
315
+ d = d.view(torch.float16).to(dtype)
316
+ d = (d * scales).reshape((n_blocks, QK_K // 16, 1))
317
+
318
+ ql = ql.reshape((n_blocks, -1, 1, 64)) >> torch.tensor([0, 4], device=d.device, dtype=torch.uint8).reshape(
319
+ (1, 1, 2, 1)
320
+ )
321
+ ql = (ql & 0x0F).reshape((n_blocks, -1, 32))
322
+ qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6], device=d.device, dtype=torch.uint8).reshape(
323
+ (1, 1, 4, 1)
324
+ )
325
+ qh = (qh & 0x03).reshape((n_blocks, -1, 32))
326
+ q = (ql | (qh << 4)).to(torch.int8) - 32
327
+ q = q.reshape((n_blocks, QK_K // 16, -1))
328
+
329
+ return (d * q).reshape((n_blocks, QK_K))
330
+
331
+
332
+ def dequantize_blocks_Q5_K(blocks, block_size, type_size, dtype=None):
333
+ n_blocks = blocks.shape[0]
334
+
335
+ d, dmin, scales, qh, qs = split_block_dims(blocks, 2, 2, K_SCALE_SIZE, QK_K // 8)
336
+
337
+ d = d.view(torch.float16).to(dtype)
338
+ dmin = dmin.view(torch.float16).to(dtype)
339
+
340
+ sc, m = get_scale_min(scales)
341
+
342
+ d = (d * sc).reshape((n_blocks, -1, 1))
343
+ dm = (dmin * m).reshape((n_blocks, -1, 1))
344
+
345
+ ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d.device, dtype=torch.uint8).reshape(
346
+ (1, 1, 2, 1)
347
+ )
348
+ qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.arange(0, 8, device=d.device, dtype=torch.uint8).reshape(
349
+ (1, 1, 8, 1)
350
+ )
351
+ ql = (ql & 0x0F).reshape((n_blocks, -1, 32))
352
+ qh = (qh & 0x01).reshape((n_blocks, -1, 32))
353
+ q = ql | (qh << 4)
354
+
355
+ return (d * q - dm).reshape((n_blocks, QK_K))
356
+
357
+
358
+ def dequantize_blocks_Q4_K(blocks, block_size, type_size, dtype=None):
359
+ n_blocks = blocks.shape[0]
360
+
361
+ d, dmin, scales, qs = split_block_dims(blocks, 2, 2, K_SCALE_SIZE)
362
+ d = d.view(torch.float16).to(dtype)
363
+ dmin = dmin.view(torch.float16).to(dtype)
364
+
365
+ sc, m = get_scale_min(scales)
366
+
367
+ d = (d * sc).reshape((n_blocks, -1, 1))
368
+ dm = (dmin * m).reshape((n_blocks, -1, 1))
369
+
370
+ qs = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d.device, dtype=torch.uint8).reshape(
371
+ (1, 1, 2, 1)
372
+ )
373
+ qs = (qs & 0x0F).reshape((n_blocks, -1, 32))
374
+
375
+ return (d * qs - dm).reshape((n_blocks, QK_K))
376
+
377
+
378
+ def dequantize_blocks_Q3_K(blocks, block_size, type_size, dtype=None):
379
+ n_blocks = blocks.shape[0]
380
+
381
+ hmask, qs, scales, d = split_block_dims(blocks, QK_K // 8, QK_K // 4, 12)
382
+ d = d.view(torch.float16).to(dtype)
383
+
384
+ lscales, hscales = scales[:, :8], scales[:, 8:]
385
+ lscales = lscales.reshape((n_blocks, 1, 8)) >> torch.tensor([0, 4], device=d.device, dtype=torch.uint8).reshape(
386
+ (1, 2, 1)
387
+ )
388
+ lscales = lscales.reshape((n_blocks, 16))
389
+ hscales = hscales.reshape((n_blocks, 1, 4)) >> torch.tensor(
390
+ [0, 2, 4, 6], device=d.device, dtype=torch.uint8
391
+ ).reshape((1, 4, 1))
392
+ hscales = hscales.reshape((n_blocks, 16))
393
+ scales = (lscales & 0x0F) | ((hscales & 0x03) << 4)
394
+ scales = scales.to(torch.int8) - 32
395
+
396
+ dl = (d * scales).reshape((n_blocks, 16, 1))
397
+
398
+ ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6], device=d.device, dtype=torch.uint8).reshape(
399
+ (1, 1, 4, 1)
400
+ )
401
+ qh = hmask.reshape(n_blocks, -1, 1, 32) >> torch.arange(0, 8, device=d.device, dtype=torch.uint8).reshape(
402
+ (1, 1, 8, 1)
403
+ )
404
+ ql = ql.reshape((n_blocks, 16, QK_K // 16)) & 3
405
+ qh = (qh.reshape((n_blocks, 16, QK_K // 16)) & 1) ^ 1
406
+ q = ql.to(torch.int8) - (qh << 2).to(torch.int8)
407
+
408
+ return (dl * q).reshape((n_blocks, QK_K))
409
+
410
+
411
+ def dequantize_blocks_Q2_K(blocks, block_size, type_size, dtype=None):
412
+ n_blocks = blocks.shape[0]
413
+
414
+ scales, qs, d, dmin = split_block_dims(blocks, QK_K // 16, QK_K // 4, 2)
415
+ d = d.view(torch.float16).to(dtype)
416
+ dmin = dmin.view(torch.float16).to(dtype)
417
+
418
+ # (n_blocks, 16, 1)
419
+ dl = (d * (scales & 0xF)).reshape((n_blocks, QK_K // 16, 1))
420
+ ml = (dmin * (scales >> 4)).reshape((n_blocks, QK_K // 16, 1))
421
+
422
+ shift = torch.tensor([0, 2, 4, 6], device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1))
423
+
424
+ qs = (qs.reshape((n_blocks, -1, 1, 32)) >> shift) & 3
425
+ qs = qs.reshape((n_blocks, QK_K // 16, 16))
426
+ qs = dl * qs - ml
427
+
428
+ return qs.reshape((n_blocks, -1))
429
+
430
+
431
+ def dequantize_blocks_BF16(blocks, block_size, type_size, dtype=None):
432
+ return (blocks.view(torch.int16).to(torch.int32) << 16).view(torch.float32)
433
+
434
+
435
+ # this part from calcuis (gguf.org)
436
+ # more info: https://github.com/calcuis/gguf-connector/blob/main/src/gguf_connector/quant2c.py
437
+
438
+
439
+ def dequantize_blocks_IQ4_NL(blocks, block_size, type_size, dtype=None):
440
+ kvalues = torch.tensor(
441
+ [-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113],
442
+ dtype=torch.float32,
443
+ device=blocks.device,
444
+ )
445
+ n_blocks = blocks.shape[0]
446
+ d, qs = split_block_dims(blocks, 2)
447
+ d = d.view(torch.float16).to(dtype)
448
+ qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor(
449
+ [0, 4], device=blocks.device, dtype=torch.uint8
450
+ ).reshape((1, 1, 2, 1))
451
+ qs = (qs & 15).reshape((n_blocks, -1)).to(torch.int64)
452
+ kvalues = kvalues.view(1, 1, 16)
453
+ qs = qs.unsqueeze(-1)
454
+ qs = torch.gather(kvalues.expand(qs.shape[0], qs.shape[1], 16), 2, qs)
455
+ qs = qs.squeeze(-1).to(dtype)
456
+ return d * qs
457
+
458
+
459
+ def dequantize_blocks_IQ4_XS(blocks, block_size, type_size, dtype=None):
460
+ kvalues = torch.tensor(
461
+ [-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113],
462
+ dtype=torch.float32,
463
+ device=blocks.device,
464
+ )
465
+ n_blocks = blocks.shape[0]
466
+ d, scales_h, scales_l, qs = split_block_dims(blocks, 2, 2, QK_K // 64)
467
+ d = d.view(torch.float16).to(dtype)
468
+ scales_h = scales_h.view(torch.int16)
469
+ scales_l = scales_l.reshape((n_blocks, -1, 1)) >> torch.tensor(
470
+ [0, 4], device=blocks.device, dtype=torch.uint8
471
+ ).reshape((1, 1, 2))
472
+ scales_h = scales_h.reshape((n_blocks, 1, -1)) >> torch.tensor(
473
+ [2 * i for i in range(QK_K // 32)], device=blocks.device, dtype=torch.uint8
474
+ ).reshape((1, -1, 1))
475
+ scales_l = scales_l.reshape((n_blocks, -1)) & 0x0F
476
+ scales_h = scales_h.reshape((n_blocks, -1)) & 0x03
477
+ scales = (scales_l | (scales_h << 4)) - 32
478
+ dl = (d * scales.to(dtype)).reshape((n_blocks, -1, 1))
479
+ shifts_q = torch.tensor([0, 4], device=blocks.device, dtype=torch.uint8).reshape(1, 1, 2, 1)
480
+ qs = qs.reshape((n_blocks, -1, 1, 16)) >> shifts_q
481
+ qs = (qs & 15).reshape((n_blocks, -1, 32)).to(torch.int64)
482
+ kvalues = kvalues.view(1, 1, 1, 16)
483
+ qs = qs.unsqueeze(-1)
484
+ qs = torch.gather(kvalues.expand(qs.shape[0], qs.shape[1], qs.shape[2], 16), 3, qs)
485
+ qs = qs.squeeze(-1).to(dtype)
486
+ return (dl * qs).reshape(n_blocks, -1)
487
+
488
+
489
+ GGML_QUANT_SIZES = gguf.GGML_QUANT_SIZES
490
+ dequantize_functions = {
491
+ gguf.GGMLQuantizationType.IQ4_NL: dequantize_blocks_IQ4_NL,
492
+ gguf.GGMLQuantizationType.IQ4_XS: dequantize_blocks_IQ4_XS,
493
+ gguf.GGMLQuantizationType.BF16: dequantize_blocks_BF16,
494
+ gguf.GGMLQuantizationType.Q8_0: dequantize_blocks_Q8_0,
495
+ gguf.GGMLQuantizationType.Q5_1: dequantize_blocks_Q5_1,
496
+ gguf.GGMLQuantizationType.Q5_0: dequantize_blocks_Q5_0,
497
+ gguf.GGMLQuantizationType.Q4_1: dequantize_blocks_Q4_1,
498
+ gguf.GGMLQuantizationType.Q4_0: dequantize_blocks_Q4_0,
499
+ gguf.GGMLQuantizationType.Q6_K: dequantize_blocks_Q6_K,
500
+ gguf.GGMLQuantizationType.Q5_K: dequantize_blocks_Q5_K,
501
+ gguf.GGMLQuantizationType.Q4_K: dequantize_blocks_Q4_K,
502
+ gguf.GGMLQuantizationType.Q3_K: dequantize_blocks_Q3_K,
503
+ gguf.GGMLQuantizationType.Q2_K: dequantize_blocks_Q2_K,
504
+ }
505
+ SUPPORTED_GGUF_QUANT_TYPES = list(dequantize_functions.keys())
506
+
507
+
508
+ def _quant_shape_from_byte_shape(shape, type_size, block_size):
509
+ return (*shape[:-1], shape[-1] // type_size * block_size)
510
+
511
+
512
+ def dequantize_gguf_tensor(tensor):
513
+ if not hasattr(tensor, "quant_type"):
514
+ return tensor
515
+
516
+ quant_type = tensor.quant_type
517
+ dequant_fn = dequantize_functions[quant_type]
518
+
519
+ block_size, type_size = GGML_QUANT_SIZES[quant_type]
520
+
521
+ # Conver to plain tensor to avoid unnecessary __torch_function__ overhead.
522
+ tensor = tensor.as_tensor()
523
+
524
+ tensor = tensor.view(torch.uint8)
525
+ shape = _quant_shape_from_byte_shape(tensor.shape, type_size, block_size)
526
+
527
+ n_blocks = tensor.numel() // type_size
528
+ blocks = tensor.reshape((n_blocks, type_size))
529
+
530
+ dequant = dequant_fn(blocks, block_size, type_size)
531
+ dequant = dequant.reshape(shape)
532
+
533
+ return dequant
534
+
535
+
536
+ class GGUFParameter(torch.nn.Parameter):
537
+ def __new__(cls, data, requires_grad=False, quant_type=None):
538
+ data = data if data is not None else torch.empty(0)
539
+ self = torch.Tensor._make_subclass(cls, data, requires_grad)
540
+ self.quant_type = quant_type
541
+ block_size, type_size = GGML_QUANT_SIZES[quant_type]
542
+ self.quant_shape = _quant_shape_from_byte_shape(self.shape, type_size, block_size)
543
+
544
+ return self
545
+
546
+ def as_tensor(self):
547
+ return torch.Tensor._make_subclass(torch.Tensor, self, self.requires_grad)
548
+
549
+ @staticmethod
550
+ def _extract_quant_type(args):
551
+ # When converting from original format checkpoints we often use splits, cats etc on tensors
552
+ # this method ensures that the returned tensor type from those operations remains GGUFParameter
553
+ # so that we preserve quant_type information
554
+ for arg in args:
555
+ if isinstance(arg, list) and isinstance(arg[0], GGUFParameter):
556
+ return arg[0].quant_type
557
+ if isinstance(arg, GGUFParameter):
558
+ return arg.quant_type
559
+ return None
560
+
561
+ @classmethod
562
+ def __torch_function__(cls, func, types, args=(), kwargs=None):
563
+ if kwargs is None:
564
+ kwargs = {}
565
+
566
+ result = super().__torch_function__(func, types, args, kwargs)
567
+
568
+ if isinstance(result, torch.Tensor):
569
+ quant_type = cls._extract_quant_type(args)
570
+ return cls(result, quant_type=quant_type)
571
+ # Handle tuples and lists
572
+ elif type(result) in (list, tuple):
573
+ # Preserve the original type (tuple or list)
574
+ quant_type = cls._extract_quant_type(args)
575
+ wrapped = [cls(x, quant_type=quant_type) if isinstance(x, torch.Tensor) else x for x in result]
576
+ return type(result)(wrapped)
577
+ else:
578
+ return result
579
+
580
+
581
+ class GGUFLinear(nn.Linear):
582
+ def __init__(
583
+ self,
584
+ in_features,
585
+ out_features,
586
+ bias=False,
587
+ compute_dtype=None,
588
+ device=None,
589
+ ) -> None:
590
+ super().__init__(in_features, out_features, bias, device)
591
+ self.compute_dtype = compute_dtype
592
+ self.device = device
593
+
594
+ def forward(self, inputs: torch.Tensor):
595
+ if ops is not None and self.weight.is_cuda and inputs.is_cuda:
596
+ return self.forward_cuda(inputs)
597
+ return self.forward_native(inputs)
598
+
599
+ def forward_native(self, inputs: torch.Tensor):
600
+ weight = dequantize_gguf_tensor(self.weight)
601
+ weight = weight.to(self.compute_dtype)
602
+ bias = self.bias.to(self.compute_dtype) if self.bias is not None else None
603
+
604
+ output = torch.nn.functional.linear(inputs, weight, bias)
605
+ return output
606
+
607
+ def forward_cuda(self, inputs: torch.Tensor):
608
+ quant_type = self.weight.quant_type
609
+ output = _fused_mul_mat_gguf(inputs.to(self.compute_dtype), self.weight, quant_type)
610
+ if self.bias is not None:
611
+ output += self.bias.to(self.compute_dtype)
612
+ return output
western_ai/europe/diffusers/src/diffusers/quantizers/modelopt/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ from .modelopt_quantizer import NVIDIAModelOptQuantizer
western_ai/europe/diffusers/src/diffusers/quantizers/modelopt/modelopt_quantizer.py ADDED
@@ -0,0 +1,190 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import TYPE_CHECKING, Any
2
+
3
+ from ...utils import (
4
+ get_module_from_name,
5
+ is_accelerate_available,
6
+ is_nvidia_modelopt_available,
7
+ is_torch_available,
8
+ logging,
9
+ )
10
+ from ..base import DiffusersQuantizer
11
+
12
+
13
+ if TYPE_CHECKING:
14
+ from ...models.modeling_utils import ModelMixin
15
+
16
+
17
+ if is_torch_available():
18
+ import torch
19
+ import torch.nn as nn
20
+
21
+ if is_accelerate_available():
22
+ from accelerate.utils import set_module_tensor_to_device
23
+
24
+
25
+ logger = logging.get_logger(__name__)
26
+
27
+
28
+ class NVIDIAModelOptQuantizer(DiffusersQuantizer):
29
+ r"""
30
+ Diffusers Quantizer for Nvidia-Model Optimizer
31
+ """
32
+
33
+ use_keep_in_fp32_modules = True
34
+ requires_calibration = False
35
+ required_packages = ["nvidia_modelopt"]
36
+
37
+ def __init__(self, quantization_config, **kwargs):
38
+ super().__init__(quantization_config, **kwargs)
39
+
40
+ def validate_environment(self, *args, **kwargs):
41
+ if not is_nvidia_modelopt_available():
42
+ raise ImportError(
43
+ "Loading an nvidia-modelopt quantized model requires nvidia-modelopt library (`pip install nvidia-modelopt`)"
44
+ )
45
+
46
+ self.offload = False
47
+
48
+ device_map = kwargs.get("device_map", None)
49
+ if isinstance(device_map, dict):
50
+ if "cpu" in device_map.values() or "disk" in device_map.values():
51
+ if self.pre_quantized:
52
+ raise ValueError(
53
+ "You are attempting to perform cpu/disk offload with a pre-quantized modelopt model "
54
+ "This is not supported yet. Please remove the CPU or disk device from the `device_map` argument."
55
+ )
56
+ else:
57
+ self.offload = True
58
+
59
+ def check_if_quantized_param(
60
+ self,
61
+ model: "ModelMixin",
62
+ param_value: "torch.Tensor",
63
+ param_name: str,
64
+ state_dict: dict[str, Any],
65
+ **kwargs,
66
+ ):
67
+ # ModelOpt imports diffusers internally. This is here to prevent circular imports
68
+ from modelopt.torch.quantization.utils import is_quantized
69
+
70
+ module, tensor_name = get_module_from_name(model, param_name)
71
+ if self.pre_quantized:
72
+ return True
73
+ elif is_quantized(module) and "weight" in tensor_name:
74
+ return True
75
+ return False
76
+
77
+ def create_quantized_param(
78
+ self,
79
+ model: "ModelMixin",
80
+ param_value: "torch.Tensor",
81
+ param_name: str,
82
+ target_device: "torch.device",
83
+ *args,
84
+ **kwargs,
85
+ ):
86
+ """
87
+ Create the quantized parameter by calling .calibrate() after setting it to the module.
88
+ """
89
+ # ModelOpt imports diffusers internally. This is here to prevent circular imports
90
+ import modelopt.torch.quantization as mtq
91
+
92
+ dtype = kwargs.get("dtype", torch.float32)
93
+ module, tensor_name = get_module_from_name(model, param_name)
94
+ if self.pre_quantized:
95
+ module._parameters[tensor_name] = torch.nn.Parameter(param_value.to(device=target_device))
96
+ else:
97
+ set_module_tensor_to_device(model, param_name, target_device, param_value, dtype)
98
+ mtq.calibrate(
99
+ module, self.quantization_config.modelopt_config["algorithm"], self.quantization_config.forward_loop
100
+ )
101
+ mtq.compress(module)
102
+ module.weight.requires_grad = False
103
+
104
+ def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | str]:
105
+ max_memory = {key: val * 0.90 for key, val in max_memory.items()}
106
+ return max_memory
107
+
108
+ def adjust_target_dtype(self, target_dtype: "torch.dtype") -> "torch.dtype":
109
+ if self.quantization_config.quant_type == "FP8":
110
+ target_dtype = torch.float8_e4m3fn
111
+ return target_dtype
112
+
113
+ def update_torch_dtype(self, torch_dtype: "torch.dtype" = None) -> "torch.dtype":
114
+ if torch_dtype is None:
115
+ logger.info("You did not specify `torch_dtype` in `from_pretrained`. Setting it to `torch.float32`.")
116
+ torch_dtype = torch.float32
117
+ return torch_dtype
118
+
119
+ def get_conv_param_names(self, model: "ModelMixin") -> list[str]:
120
+ """
121
+ Get parameter names for all convolutional layers in a HuggingFace ModelMixin. Includes Conv1d/2d/3d and
122
+ ConvTranspose1d/2d/3d.
123
+ """
124
+ conv_types = (
125
+ nn.Conv1d,
126
+ nn.Conv2d,
127
+ nn.Conv3d,
128
+ nn.ConvTranspose1d,
129
+ nn.ConvTranspose2d,
130
+ nn.ConvTranspose3d,
131
+ )
132
+
133
+ conv_param_names = []
134
+ for name, module in model.named_modules():
135
+ if isinstance(module, conv_types):
136
+ for param_name, _ in module.named_parameters(recurse=False):
137
+ conv_param_names.append(f"{name}.{param_name}")
138
+
139
+ return conv_param_names
140
+
141
+ def _process_model_before_weight_loading(
142
+ self,
143
+ model: "ModelMixin",
144
+ device_map,
145
+ keep_in_fp32_modules: list[str] = [],
146
+ **kwargs,
147
+ ):
148
+ # ModelOpt imports diffusers internally. This is here to prevent circular imports
149
+ import modelopt.torch.opt as mto
150
+
151
+ if self.pre_quantized:
152
+ return
153
+
154
+ modules_to_not_convert = self.quantization_config.modules_to_not_convert
155
+
156
+ if modules_to_not_convert is None:
157
+ modules_to_not_convert = []
158
+ if isinstance(modules_to_not_convert, str):
159
+ modules_to_not_convert = [modules_to_not_convert]
160
+ modules_to_not_convert.extend(keep_in_fp32_modules)
161
+ if self.quantization_config.disable_conv_quantization:
162
+ modules_to_not_convert.extend(self.get_conv_param_names(model))
163
+
164
+ for module in modules_to_not_convert:
165
+ self.quantization_config.modelopt_config["quant_cfg"]["*" + module + "*"] = {"enable": False}
166
+ self.quantization_config.modules_to_not_convert = modules_to_not_convert
167
+ mto.apply_mode(model, mode=[("quantize", self.quantization_config.modelopt_config)])
168
+ model.config.quantization_config = self.quantization_config
169
+
170
+ def _process_model_after_weight_loading(self, model, **kwargs):
171
+ # ModelOpt imports diffusers internally. This is here to prevent circular imports
172
+ from modelopt.torch.opt import ModeloptStateManager
173
+
174
+ if self.pre_quantized:
175
+ return model
176
+
177
+ for _, m in model.named_modules():
178
+ if hasattr(m, ModeloptStateManager._state_key) and m is not model:
179
+ ModeloptStateManager.remove_state(m)
180
+
181
+ return model
182
+
183
+ @property
184
+ def is_trainable(self):
185
+ return True
186
+
187
+ @property
188
+ def is_serializable(self):
189
+ self.quantization_config.check_model_patching(operation="saving")
190
+ return True