Text-to-Image
Diffusers
Safetensors
English
StableDiffusionPipeline
image-to-image
stable-diffusion
stable-diffusion-diffusers
art
Instructions to use Cyanex/D.R.E.A.M with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use Cyanex/D.R.E.A.M with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("Cyanex/D.R.E.A.M", dtype=torch.bfloat16, device_map="cuda") prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" image = pipe(prompt).images[0] - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- Draw Things
- DiffusionBee
Rename app.py to pipeline.py
Browse files- app.py +0 -115
- pipeline.py +664 -0
app.py
DELETED
|
@@ -1,115 +0,0 @@
|
|
| 1 |
-
import gradio as gr
|
| 2 |
-
import torch
|
| 3 |
-
from PIL.ImageDraw import Draw
|
| 4 |
-
from diffusers import StableDiffusionPipeline
|
| 5 |
-
from PIL import Image, ImageOps
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
# Load pipeline once
|
| 9 |
-
model_id = 'Deci/DeciDiffusion-v1-0'
|
| 10 |
-
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 11 |
-
pipe = StableDiffusionPipeline.from_pretrained(model_id, custom_pipeline=model_id, torch_dtype=torch.float32)
|
| 12 |
-
pipe.unet = pipe.unet.from_pretrained(model_id, subfolder='flexible_unet', torch_dtype=torch.float32)
|
| 13 |
-
pipe = pipe.to(device)
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
def read_content(file_path: str) -> str:
|
| 17 |
-
"""read the content of target file
|
| 18 |
-
"""
|
| 19 |
-
with open(file_path, 'r', encoding='utf-8') as f:
|
| 20 |
-
content = f.read()
|
| 21 |
-
|
| 22 |
-
return content
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
def predict(_prompt: str, _steps: int = 30, _seed: int = 42, _guidance_scale: float = 7.5, _negative_prompt: str = ""):
|
| 26 |
-
_negative_prompt = [_negative_prompt] if _negative_prompt else None
|
| 27 |
-
|
| 28 |
-
output = pipe(prompt=[_prompt],
|
| 29 |
-
negative_prompt=_negative_prompt,
|
| 30 |
-
num_inference_steps=int(_steps),
|
| 31 |
-
guidance_scale=_guidance_scale,
|
| 32 |
-
generator=torch.Generator(device).manual_seed(_seed),
|
| 33 |
-
)
|
| 34 |
-
output_image = output.images[0]
|
| 35 |
-
|
| 36 |
-
# Add border beneath the image with Deci logo + prompt
|
| 37 |
-
if len(_prompt) > 52:
|
| 38 |
-
_prompt = _prompt[:52] + "..."
|
| 39 |
-
|
| 40 |
-
original_image_height = output_image.size[1]
|
| 41 |
-
output_image = ImageOps.expand(output_image, border=(0, 0, 0, 64), fill='white')
|
| 42 |
-
deci_logo = Image.open('./deci_logo_white.png')
|
| 43 |
-
output_image.paste(deci_logo, (0, original_image_height))
|
| 44 |
-
Draw(output_image).text((deci_logo.size[0], original_image_height + 26), _prompt, (127, 127, 127))
|
| 45 |
-
return output_image
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
css = '''
|
| 49 |
-
.gradio-container {
|
| 50 |
-
max-width: 1100px !important;
|
| 51 |
-
background-image: url(https://huggingface.co/spaces/Deci/Deci-DeciDiffusionClean/resolve/main/background-image.png);
|
| 52 |
-
background-size: cover;
|
| 53 |
-
background-position: center center;
|
| 54 |
-
background-repeat: no-repeat;
|
| 55 |
-
}
|
| 56 |
-
|
| 57 |
-
.footer {margin-bottom: 45px;margin-top: 35px !important;text-align: center;border-bottom: 1px solid #e5e5e5}
|
| 58 |
-
.footer>p {font-size: .8rem; display: inline-block; padding: 0 10px;transform: translateY(10px);background: white}
|
| 59 |
-
.dark .footer {border-color: #303030}
|
| 60 |
-
.dark .footer>p {background: #0b0f19}
|
| 61 |
-
.acknowledgments h4{margin: 1.25em 0 .25em 0;font-weight: bold;font-size: 115%}
|
| 62 |
-
@keyframes spin {
|
| 63 |
-
from {
|
| 64 |
-
transform: rotate(0deg);
|
| 65 |
-
}
|
| 66 |
-
to {
|
| 67 |
-
transform: rotate(360deg);
|
| 68 |
-
}
|
| 69 |
-
}
|
| 70 |
-
'''
|
| 71 |
-
|
| 72 |
-
demo = gr.Blocks(css=css, elem_id="total-container")
|
| 73 |
-
with demo:
|
| 74 |
-
gr.HTML(read_content("header.html"))
|
| 75 |
-
with gr.Row():
|
| 76 |
-
with gr.Column():
|
| 77 |
-
with gr.Row(mobile_collapse=False, equal_height=True):
|
| 78 |
-
prompt = gr.Textbox(placeholder="Your prompt", show_label=False, elem_id="prompt", autofocus=True, lines=3, )
|
| 79 |
-
|
| 80 |
-
with gr.Accordion(label="Advanced Settings", open=False):
|
| 81 |
-
with gr.Row(mobile_collapse=False, equal_height=True):
|
| 82 |
-
steps = gr.Slider(value=30, minimum=15, maximum=50, step=1, label="steps", interactive=True)
|
| 83 |
-
seed = gr.Slider(value=42, minimum=1, maximum=100, step=1, label="seed", interactive=True)
|
| 84 |
-
guidance_scale = gr.Slider(value=7.5, minimum=1, maximum=15, step=0.1, label='guidance_scale', interactive=True)
|
| 85 |
-
|
| 86 |
-
with gr.Row(mobile_collapse=False, equal_height=True):
|
| 87 |
-
negative_prompt = gr.Textbox(label="negative_prompt", placeholder="Your negative prompt",
|
| 88 |
-
info="what you don't want to see in the image", lines=3)
|
| 89 |
-
with gr.Row():
|
| 90 |
-
btn = gr.Button(value="Generate!", elem_id="run_button")
|
| 91 |
-
|
| 92 |
-
with gr.Column():
|
| 93 |
-
image_out = gr.Image(label="Output", elem_id="output-img", height=400)
|
| 94 |
-
|
| 95 |
-
btn.click(fn=predict,
|
| 96 |
-
inputs=[prompt, steps, seed, guidance_scale, negative_prompt],
|
| 97 |
-
outputs=[image_out],
|
| 98 |
-
api_name='run')
|
| 99 |
-
|
| 100 |
-
gr.HTML(
|
| 101 |
-
"""
|
| 102 |
-
<div class="footer">
|
| 103 |
-
<p>Model by <a href="https://deci.ai" style="text-decoration: underline;" target="_blank">Deci.ai</a> - Gradio Demo by 🤗 Hugging Face
|
| 104 |
-
</p>
|
| 105 |
-
</div>
|
| 106 |
-
<div class="acknowledgments">
|
| 107 |
-
<p><h4>LICENSE</h4>
|
| 108 |
-
The model is licensed with a <a href="https://huggingface.co/Deci/DeciDiffusion-v1-0/blob/main/LICENSE-WEIGHTS.md" style="text-decoration: underline;" target="_blank">CreativeML Open RAIL-M</a> license. The authors claim no rights on the outputs you generate, you are free to use them and are accountable for their use which must not go against the provisions set in this license. The license forbids you from sharing any content that violates any laws, produce any harm to a person, disseminate any personal information that would be meant for harm, spread misinformation and target vulnerable groups. For the full list of restrictions please <a href="https://huggingface.co/Deci/DeciDiffusion-v1-0/blob/main/LICENSE-WEIGHTS.md" target="_blank" style="text-decoration: underline;" target="_blank">read the license</a></p>
|
| 109 |
-
<p><h4>Biases and content acknowledgment</h4>
|
| 110 |
-
Despite how impressive being able to turn text into image is, beware to the fact that this model may output content that reinforces or exacerbates societal biases, as well as realistic faces, pornography and violence. The model was trained on the <a href="https://laion.ai/blog/laion-5b/" style="text-decoration: underline;" target="_blank">LAION-5B dataset</a>, which scraped non-curated image-text-pairs from the internet (the exception being the removal of illegal content) and is meant for research purposes. You can read more in the <a href="https://huggingface.co/Deci/DeciDiffusion-v1-0" style="text-decoration: underline;" target="_blank">model card</a></p>
|
| 111 |
-
</div>
|
| 112 |
-
"""
|
| 113 |
-
)
|
| 114 |
-
|
| 115 |
-
demo.queue(max_size=50).launch()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
pipeline.py
ADDED
|
@@ -0,0 +1,664 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import itertools
|
| 2 |
+
from typing import Any, Optional, Dict, Tuple
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
from diffusers import StableDiffusionPipeline, AutoencoderKL
|
| 6 |
+
from diffusers import Transformer2DModel, ModelMixin, ConfigMixin
|
| 7 |
+
from diffusers import UNet2DConditionModel
|
| 8 |
+
from diffusers.configuration_utils import register_to_config
|
| 9 |
+
from diffusers.models.attention import BasicTransformerBlock
|
| 10 |
+
from diffusers.models.resnet import ResnetBlock2D, Downsample2D, Upsample2D
|
| 11 |
+
from diffusers.models.transformer_2d import Transformer2DModelOutput
|
| 12 |
+
from diffusers.pipelines.stable_diffusion import StableDiffusionSafetyChecker
|
| 13 |
+
from diffusers.schedulers import KarrasDiffusionSchedulers
|
| 14 |
+
from torch import nn
|
| 15 |
+
from transformers import CLIPTextModel, CLIPTokenizer, CLIPImageProcessor
|
| 16 |
+
|
| 17 |
+
FlexibleUnetConfigurations = {
|
| 18 |
+
# General parameters for all blocks
|
| 19 |
+
'sample_size': 64,
|
| 20 |
+
'temb_dim': 320 * 4,
|
| 21 |
+
'resnet_eps': 1e-5,
|
| 22 |
+
'resnet_act_fn': 'silu',
|
| 23 |
+
'num_attention_heads': 8,
|
| 24 |
+
'cross_attention_dim': 768,
|
| 25 |
+
|
| 26 |
+
# Controls modules execute order in unet's forward
|
| 27 |
+
'mix_block_in_forward': True,
|
| 28 |
+
# Down blocks parameters
|
| 29 |
+
'down_blocks_in_channels': [320, 320, 640],
|
| 30 |
+
'down_blocks_out_channels': [320, 640, 1280],
|
| 31 |
+
'down_blocks_num_attentions': [0, 1, 3],
|
| 32 |
+
'down_blocks_num_resnets': [2, 2, 1],
|
| 33 |
+
'add_downsample': [True, True, True],
|
| 34 |
+
|
| 35 |
+
# Middle block parameters
|
| 36 |
+
'add_upsample_mid_block': True,
|
| 37 |
+
'mid_num_resnets': 4,
|
| 38 |
+
'mid_num_attentions': 2,
|
| 39 |
+
|
| 40 |
+
# Up block parameters
|
| 41 |
+
'prev_output_channels': [1280, 1280, 640],
|
| 42 |
+
'up_blocks_num_attentions': [6, 3, 0],
|
| 43 |
+
'up_blocks_num_resnets': [2, 3, 3],
|
| 44 |
+
'add_upsample': [True, True, False],
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def custom_sort_order(obj):
|
| 49 |
+
"""
|
| 50 |
+
Key function for sorting order of execution in forward methods
|
| 51 |
+
"""
|
| 52 |
+
return {ResnetBlock2D: 0, Transformer2DModel: 1, FlexibleTransformer2DModel: 1}.get(obj.__class__)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class FlexibleUNet2DConditionModel(UNet2DConditionModel, ModelMixin):
|
| 56 |
+
configurations = FlexibleUnetConfigurations
|
| 57 |
+
|
| 58 |
+
@register_to_config
|
| 59 |
+
def __init__(self):
|
| 60 |
+
super().__init__(sample_size=self.configurations.get('sample_size',
|
| 61 |
+
FlexibleUnetConfigurations['sample_size']),
|
| 62 |
+
cross_attention_dim=self.configurations.get("cross_attention_dim",
|
| 63 |
+
FlexibleUnetConfigurations['cross_attention_dim']))
|
| 64 |
+
|
| 65 |
+
num_attention_heads = self.configurations.get("num_attention_heads")
|
| 66 |
+
cross_attention_dim = self.configurations.get("cross_attention_dim")
|
| 67 |
+
mix_block_in_forward = self.configurations.get("mix_block_in_forward")
|
| 68 |
+
resnet_act_fn = self.configurations.get("resnet_act_fn")
|
| 69 |
+
resnet_eps = self.configurations.get("resnet_eps")
|
| 70 |
+
temb_dim = self.configurations.get("temb_dim")
|
| 71 |
+
|
| 72 |
+
###############
|
| 73 |
+
# Down blocks #
|
| 74 |
+
###############
|
| 75 |
+
down_blocks_num_attentions = self.configurations.get("down_blocks_num_attentions")
|
| 76 |
+
down_blocks_out_channels = self.configurations.get("down_blocks_out_channels")
|
| 77 |
+
down_blocks_in_channels = self.configurations.get("down_blocks_in_channels")
|
| 78 |
+
down_blocks_num_resnets = self.configurations.get("down_blocks_num_resnets")
|
| 79 |
+
add_downsample = self.configurations.get("add_downsample")
|
| 80 |
+
|
| 81 |
+
self.down_blocks = nn.ModuleList()
|
| 82 |
+
|
| 83 |
+
for i, (in_c, out_c, n_res, n_att, add_down) in enumerate(zip(down_blocks_in_channels, down_blocks_out_channels,
|
| 84 |
+
down_blocks_num_resnets,
|
| 85 |
+
down_blocks_num_attentions,
|
| 86 |
+
add_downsample)):
|
| 87 |
+
last_block = i == len(down_blocks_in_channels) - 1
|
| 88 |
+
self.down_blocks.append(FlexibleCrossAttnDownBlock2D(in_channels=in_c,
|
| 89 |
+
out_channels=out_c,
|
| 90 |
+
temb_channels=temb_dim,
|
| 91 |
+
num_resnets=n_res,
|
| 92 |
+
num_attentions=n_att,
|
| 93 |
+
resnet_eps=resnet_eps,
|
| 94 |
+
resnet_act_fn=resnet_act_fn,
|
| 95 |
+
num_attention_heads=num_attention_heads,
|
| 96 |
+
cross_attention_dim=cross_attention_dim,
|
| 97 |
+
add_downsample=add_down,
|
| 98 |
+
last_block=last_block,
|
| 99 |
+
mix_block_in_forward=mix_block_in_forward))
|
| 100 |
+
|
| 101 |
+
###############
|
| 102 |
+
# Mid blocks #
|
| 103 |
+
###############
|
| 104 |
+
|
| 105 |
+
mid_block_add_upsample = self.configurations.get("add_upsample_mid_block")
|
| 106 |
+
mid_num_attentions = self.configurations.get("mid_num_attentions")
|
| 107 |
+
mid_num_resnets = self.configurations.get("mid_num_resnets")
|
| 108 |
+
|
| 109 |
+
self.mid_block = FlexibleUNetMidBlock2DCrossAttn(in_channels=down_blocks_out_channels[-1],
|
| 110 |
+
temb_channels=temb_dim,
|
| 111 |
+
resnet_act_fn=resnet_act_fn,
|
| 112 |
+
resnet_eps=resnet_eps,
|
| 113 |
+
cross_attention_dim=cross_attention_dim,
|
| 114 |
+
num_attention_heads=num_attention_heads,
|
| 115 |
+
num_resnets=mid_num_resnets,
|
| 116 |
+
num_attentions=mid_num_attentions,
|
| 117 |
+
mix_block_in_forward=mix_block_in_forward,
|
| 118 |
+
add_upsample=mid_block_add_upsample
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
###############
|
| 122 |
+
# Up blocks #
|
| 123 |
+
###############
|
| 124 |
+
|
| 125 |
+
up_blocks_num_attentions = self.configurations.get("up_blocks_num_attentions")
|
| 126 |
+
up_blocks_num_resnets = self.configurations.get("up_blocks_num_resnets")
|
| 127 |
+
prev_output_channels = self.configurations.get("prev_output_channels")
|
| 128 |
+
up_upsample = self.configurations.get("add_upsample")
|
| 129 |
+
|
| 130 |
+
self.up_blocks = nn.ModuleList()
|
| 131 |
+
for in_c, out_c, prev_out, n_res, n_att, add_up in zip(reversed(down_blocks_in_channels),
|
| 132 |
+
reversed(down_blocks_out_channels),
|
| 133 |
+
prev_output_channels,
|
| 134 |
+
up_blocks_num_resnets, up_blocks_num_attentions,
|
| 135 |
+
up_upsample):
|
| 136 |
+
self.up_blocks.append(FlexibleCrossAttnUpBlock2D(in_channels=in_c,
|
| 137 |
+
out_channels=out_c,
|
| 138 |
+
prev_output_channel=prev_out,
|
| 139 |
+
temb_channels=temb_dim,
|
| 140 |
+
num_resnets=n_res,
|
| 141 |
+
num_attentions=n_att,
|
| 142 |
+
resnet_eps=resnet_eps,
|
| 143 |
+
resnet_act_fn=resnet_act_fn,
|
| 144 |
+
num_attention_heads=num_attention_heads,
|
| 145 |
+
cross_attention_dim=cross_attention_dim,
|
| 146 |
+
add_upsample=add_up,
|
| 147 |
+
mix_block_in_forward=mix_block_in_forward
|
| 148 |
+
))
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
class FlexibleCrossAttnDownBlock2D(nn.Module):
|
| 152 |
+
def __init__(
|
| 153 |
+
self,
|
| 154 |
+
in_channels: int,
|
| 155 |
+
out_channels: int,
|
| 156 |
+
temb_channels: int,
|
| 157 |
+
dropout: float = 0.0,
|
| 158 |
+
num_resnets: int = 1,
|
| 159 |
+
num_attentions: int = 1,
|
| 160 |
+
transformer_layers_per_block: int = 1,
|
| 161 |
+
resnet_eps: float = 1e-6,
|
| 162 |
+
resnet_time_scale_shift: str = "default",
|
| 163 |
+
resnet_act_fn: str = "swish",
|
| 164 |
+
resnet_groups: int = 32,
|
| 165 |
+
resnet_pre_norm: bool = True,
|
| 166 |
+
num_attention_heads: int = 1,
|
| 167 |
+
cross_attention_dim: int = 1280,
|
| 168 |
+
output_scale_factor: float = 1.0,
|
| 169 |
+
downsample_padding: int = 1,
|
| 170 |
+
add_downsample: bool = True,
|
| 171 |
+
use_linear_projection: bool = False,
|
| 172 |
+
only_cross_attention: bool = False,
|
| 173 |
+
upcast_attention: bool = False,
|
| 174 |
+
last_block: bool = False,
|
| 175 |
+
mix_block_in_forward: bool = True,
|
| 176 |
+
):
|
| 177 |
+
super().__init__()
|
| 178 |
+
|
| 179 |
+
self.last_block = last_block
|
| 180 |
+
self.mix_block_in_forward = mix_block_in_forward
|
| 181 |
+
self.has_cross_attention = True
|
| 182 |
+
self.num_attention_heads = num_attention_heads
|
| 183 |
+
|
| 184 |
+
modules = []
|
| 185 |
+
|
| 186 |
+
add_resnets = [True] * num_resnets
|
| 187 |
+
add_cross_attentions = [True] * num_attentions
|
| 188 |
+
for i, (add_resnet, add_cross_attention) in enumerate(
|
| 189 |
+
itertools.zip_longest(add_resnets, add_cross_attentions, fillvalue=False)):
|
| 190 |
+
in_channels = in_channels if i == 0 else out_channels
|
| 191 |
+
if add_resnet:
|
| 192 |
+
modules.append(
|
| 193 |
+
ResnetBlock2D(
|
| 194 |
+
in_channels=in_channels,
|
| 195 |
+
out_channels=out_channels,
|
| 196 |
+
temb_channels=temb_channels,
|
| 197 |
+
eps=resnet_eps,
|
| 198 |
+
groups=resnet_groups,
|
| 199 |
+
dropout=dropout,
|
| 200 |
+
time_embedding_norm=resnet_time_scale_shift,
|
| 201 |
+
non_linearity=resnet_act_fn,
|
| 202 |
+
output_scale_factor=output_scale_factor,
|
| 203 |
+
pre_norm=resnet_pre_norm,
|
| 204 |
+
)
|
| 205 |
+
)
|
| 206 |
+
if add_cross_attention:
|
| 207 |
+
modules.append(
|
| 208 |
+
FlexibleTransformer2DModel(
|
| 209 |
+
num_attention_heads=num_attention_heads,
|
| 210 |
+
attention_head_dim=out_channels // num_attention_heads,
|
| 211 |
+
in_channels=out_channels,
|
| 212 |
+
num_layers=transformer_layers_per_block,
|
| 213 |
+
cross_attention_dim=cross_attention_dim,
|
| 214 |
+
norm_num_groups=resnet_groups,
|
| 215 |
+
use_linear_projection=use_linear_projection,
|
| 216 |
+
only_cross_attention=only_cross_attention,
|
| 217 |
+
upcast_attention=upcast_attention,
|
| 218 |
+
)
|
| 219 |
+
)
|
| 220 |
+
|
| 221 |
+
if not mix_block_in_forward:
|
| 222 |
+
modules = sorted(modules, key=custom_sort_order)
|
| 223 |
+
|
| 224 |
+
self.modules_list = nn.ModuleList(modules)
|
| 225 |
+
|
| 226 |
+
if add_downsample:
|
| 227 |
+
self.downsamplers = nn.ModuleList(
|
| 228 |
+
[
|
| 229 |
+
Downsample2D(
|
| 230 |
+
out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op"
|
| 231 |
+
)
|
| 232 |
+
]
|
| 233 |
+
)
|
| 234 |
+
else:
|
| 235 |
+
self.downsamplers = None
|
| 236 |
+
|
| 237 |
+
self.gradient_checkpointing = False
|
| 238 |
+
|
| 239 |
+
def forward(
|
| 240 |
+
self,
|
| 241 |
+
hidden_states: torch.FloatTensor,
|
| 242 |
+
temb: Optional[torch.FloatTensor] = None,
|
| 243 |
+
encoder_hidden_states: Optional[torch.FloatTensor] = None,
|
| 244 |
+
attention_mask: Optional[torch.FloatTensor] = None,
|
| 245 |
+
cross_attention_kwargs: Optional[Dict[str, Any]] = None,
|
| 246 |
+
encoder_attention_mask: Optional[torch.FloatTensor] = None,
|
| 247 |
+
):
|
| 248 |
+
output_states = ()
|
| 249 |
+
|
| 250 |
+
for module in self.modules_list:
|
| 251 |
+
if isinstance(module, ResnetBlock2D):
|
| 252 |
+
hidden_states = module(hidden_states, temb)
|
| 253 |
+
elif isinstance(module, (FlexibleTransformer2DModel, Transformer2DModel)):
|
| 254 |
+
hidden_states = module(
|
| 255 |
+
hidden_states,
|
| 256 |
+
encoder_hidden_states=encoder_hidden_states,
|
| 257 |
+
cross_attention_kwargs=cross_attention_kwargs,
|
| 258 |
+
attention_mask=attention_mask,
|
| 259 |
+
encoder_attention_mask=encoder_attention_mask,
|
| 260 |
+
return_dict=False,
|
| 261 |
+
)[0]
|
| 262 |
+
else:
|
| 263 |
+
raise ValueError(f'Got an unexpected module in modules list! {type(module)}')
|
| 264 |
+
if isinstance(module, ResnetBlock2D):
|
| 265 |
+
output_states = output_states + (hidden_states,)
|
| 266 |
+
|
| 267 |
+
if self.downsamplers is not None:
|
| 268 |
+
for downsampler in self.downsamplers:
|
| 269 |
+
hidden_states = downsampler(hidden_states)
|
| 270 |
+
|
| 271 |
+
if not self.last_block:
|
| 272 |
+
output_states = output_states + (hidden_states,)
|
| 273 |
+
|
| 274 |
+
return hidden_states, output_states
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
class FlexibleCrossAttnUpBlock2D(nn.Module):
|
| 278 |
+
def __init__(
|
| 279 |
+
self,
|
| 280 |
+
in_channels: int,
|
| 281 |
+
out_channels: int,
|
| 282 |
+
prev_output_channel: int,
|
| 283 |
+
temb_channels: int,
|
| 284 |
+
dropout: float = 0.0,
|
| 285 |
+
num_resnets: int = 1,
|
| 286 |
+
num_attentions: int = 1,
|
| 287 |
+
transformer_layers_per_block: int = 1,
|
| 288 |
+
resnet_eps: float = 1e-6,
|
| 289 |
+
resnet_time_scale_shift: str = "default",
|
| 290 |
+
resnet_act_fn: str = "swish",
|
| 291 |
+
resnet_groups: int = 32,
|
| 292 |
+
resnet_pre_norm: bool = True,
|
| 293 |
+
num_attention_heads: int = 1,
|
| 294 |
+
cross_attention_dim: int = 1280,
|
| 295 |
+
output_scale_factor: float = 1.0,
|
| 296 |
+
add_upsample: bool = True,
|
| 297 |
+
use_linear_projection: bool = False,
|
| 298 |
+
only_cross_attention: bool = False,
|
| 299 |
+
upcast_attention: bool = False,
|
| 300 |
+
mix_block_in_forward: bool = True
|
| 301 |
+
):
|
| 302 |
+
super().__init__()
|
| 303 |
+
modules = []
|
| 304 |
+
|
| 305 |
+
# WARNING: This parameter is filled with number of resnets and used within StableDiffusionPipeline
|
| 306 |
+
self.resnets = []
|
| 307 |
+
|
| 308 |
+
self.has_cross_attention = True
|
| 309 |
+
self.num_attention_heads = num_attention_heads
|
| 310 |
+
|
| 311 |
+
add_resnets = [True] * num_resnets
|
| 312 |
+
add_cross_attentions = [True] * num_attentions
|
| 313 |
+
for i, (add_resnet, add_cross_attention) in enumerate(
|
| 314 |
+
itertools.zip_longest(add_resnets, add_cross_attentions, fillvalue=False)):
|
| 315 |
+
res_skip_channels = in_channels if (i == len(add_resnets) - 1) else out_channels
|
| 316 |
+
resnet_in_channels = prev_output_channel if i == 0 else out_channels
|
| 317 |
+
|
| 318 |
+
if add_resnet:
|
| 319 |
+
self.resnets += [True]
|
| 320 |
+
modules.append(
|
| 321 |
+
ResnetBlock2D(
|
| 322 |
+
in_channels=resnet_in_channels + res_skip_channels,
|
| 323 |
+
out_channels=out_channels,
|
| 324 |
+
temb_channels=temb_channels,
|
| 325 |
+
eps=resnet_eps,
|
| 326 |
+
groups=resnet_groups,
|
| 327 |
+
dropout=dropout,
|
| 328 |
+
time_embedding_norm=resnet_time_scale_shift,
|
| 329 |
+
non_linearity=resnet_act_fn,
|
| 330 |
+
output_scale_factor=output_scale_factor,
|
| 331 |
+
pre_norm=resnet_pre_norm,
|
| 332 |
+
)
|
| 333 |
+
)
|
| 334 |
+
if add_cross_attention:
|
| 335 |
+
modules.append(
|
| 336 |
+
FlexibleTransformer2DModel(
|
| 337 |
+
num_attention_heads,
|
| 338 |
+
out_channels // num_attention_heads,
|
| 339 |
+
in_channels=out_channels,
|
| 340 |
+
num_layers=transformer_layers_per_block,
|
| 341 |
+
cross_attention_dim=cross_attention_dim,
|
| 342 |
+
norm_num_groups=resnet_groups,
|
| 343 |
+
use_linear_projection=use_linear_projection,
|
| 344 |
+
only_cross_attention=only_cross_attention,
|
| 345 |
+
upcast_attention=upcast_attention,
|
| 346 |
+
)
|
| 347 |
+
)
|
| 348 |
+
|
| 349 |
+
if not mix_block_in_forward:
|
| 350 |
+
modules = sorted(modules, key=custom_sort_order)
|
| 351 |
+
|
| 352 |
+
self.modules_list = nn.ModuleList(modules)
|
| 353 |
+
|
| 354 |
+
self.upsamplers = None
|
| 355 |
+
if add_upsample:
|
| 356 |
+
self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)])
|
| 357 |
+
|
| 358 |
+
self.gradient_checkpointing = False
|
| 359 |
+
|
| 360 |
+
def forward(
|
| 361 |
+
self,
|
| 362 |
+
hidden_states: torch.FloatTensor,
|
| 363 |
+
res_hidden_states_tuple: Tuple[torch.FloatTensor, ...],
|
| 364 |
+
temb: Optional[torch.FloatTensor] = None,
|
| 365 |
+
encoder_hidden_states: Optional[torch.FloatTensor] = None,
|
| 366 |
+
cross_attention_kwargs: Optional[Dict[str, Any]] = None,
|
| 367 |
+
upsample_size: Optional[int] = None,
|
| 368 |
+
attention_mask: Optional[torch.FloatTensor] = None,
|
| 369 |
+
encoder_attention_mask: Optional[torch.FloatTensor] = None,
|
| 370 |
+
):
|
| 371 |
+
|
| 372 |
+
for module in self.modules_list:
|
| 373 |
+
if isinstance(module, ResnetBlock2D):
|
| 374 |
+
res_hidden_states = res_hidden_states_tuple[-1]
|
| 375 |
+
res_hidden_states_tuple = res_hidden_states_tuple[:-1]
|
| 376 |
+
hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)
|
| 377 |
+
hidden_states = module(hidden_states, temb)
|
| 378 |
+
if isinstance(module, (FlexibleTransformer2DModel, Transformer2DModel)):
|
| 379 |
+
hidden_states = module(
|
| 380 |
+
hidden_states,
|
| 381 |
+
encoder_hidden_states=encoder_hidden_states,
|
| 382 |
+
cross_attention_kwargs=cross_attention_kwargs,
|
| 383 |
+
attention_mask=attention_mask,
|
| 384 |
+
encoder_attention_mask=encoder_attention_mask,
|
| 385 |
+
return_dict=False,
|
| 386 |
+
)[0]
|
| 387 |
+
|
| 388 |
+
if self.upsamplers is not None:
|
| 389 |
+
for upsampler in self.upsamplers:
|
| 390 |
+
hidden_states = upsampler(hidden_states, upsample_size)
|
| 391 |
+
|
| 392 |
+
return hidden_states
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
class FlexibleUNetMidBlock2DCrossAttn(nn.Module):
|
| 396 |
+
def __init__(
|
| 397 |
+
self,
|
| 398 |
+
in_channels: int,
|
| 399 |
+
temb_channels: int,
|
| 400 |
+
dropout: float = 0.0,
|
| 401 |
+
num_resnets: int = 1,
|
| 402 |
+
num_attentions: int = 1,
|
| 403 |
+
transformer_layers_per_block: int = 1,
|
| 404 |
+
resnet_eps: float = 1e-6,
|
| 405 |
+
resnet_time_scale_shift: str = "default",
|
| 406 |
+
resnet_act_fn: str = "swish",
|
| 407 |
+
resnet_groups: int = 32,
|
| 408 |
+
resnet_pre_norm: bool = True,
|
| 409 |
+
num_attention_heads: int = 1,
|
| 410 |
+
output_scale_factor: float = 1.0,
|
| 411 |
+
cross_attention_dim: int = 1280,
|
| 412 |
+
use_linear_projection: bool = False,
|
| 413 |
+
upcast_attention: bool = False,
|
| 414 |
+
mix_block_in_forward: bool = True,
|
| 415 |
+
add_upsample: bool = True,
|
| 416 |
+
):
|
| 417 |
+
super().__init__()
|
| 418 |
+
|
| 419 |
+
self.has_cross_attention = True
|
| 420 |
+
self.num_attention_heads = num_attention_heads
|
| 421 |
+
resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32)
|
| 422 |
+
# There is always at least one resnet
|
| 423 |
+
modules = [ResnetBlock2D(
|
| 424 |
+
in_channels=in_channels,
|
| 425 |
+
out_channels=in_channels,
|
| 426 |
+
temb_channels=temb_channels,
|
| 427 |
+
eps=resnet_eps,
|
| 428 |
+
groups=resnet_groups,
|
| 429 |
+
dropout=dropout,
|
| 430 |
+
time_embedding_norm=resnet_time_scale_shift,
|
| 431 |
+
non_linearity=resnet_act_fn,
|
| 432 |
+
output_scale_factor=output_scale_factor,
|
| 433 |
+
pre_norm=resnet_pre_norm,
|
| 434 |
+
)]
|
| 435 |
+
|
| 436 |
+
add_resnets = [True] * num_resnets
|
| 437 |
+
add_cross_attentions = [True] * num_attentions
|
| 438 |
+
for i, (add_resnet, add_cross_attention) in enumerate(
|
| 439 |
+
itertools.zip_longest(add_resnets, add_cross_attentions, fillvalue=False)):
|
| 440 |
+
if add_cross_attention:
|
| 441 |
+
modules.append(
|
| 442 |
+
FlexibleTransformer2DModel(
|
| 443 |
+
num_attention_heads,
|
| 444 |
+
in_channels // num_attention_heads,
|
| 445 |
+
in_channels=in_channels,
|
| 446 |
+
num_layers=transformer_layers_per_block,
|
| 447 |
+
cross_attention_dim=cross_attention_dim,
|
| 448 |
+
norm_num_groups=resnet_groups,
|
| 449 |
+
use_linear_projection=use_linear_projection,
|
| 450 |
+
upcast_attention=upcast_attention,
|
| 451 |
+
)
|
| 452 |
+
)
|
| 453 |
+
|
| 454 |
+
if add_resnet:
|
| 455 |
+
modules.append(
|
| 456 |
+
ResnetBlock2D(
|
| 457 |
+
in_channels=in_channels,
|
| 458 |
+
out_channels=in_channels,
|
| 459 |
+
temb_channels=temb_channels,
|
| 460 |
+
eps=resnet_eps,
|
| 461 |
+
groups=resnet_groups,
|
| 462 |
+
dropout=dropout,
|
| 463 |
+
time_embedding_norm=resnet_time_scale_shift,
|
| 464 |
+
non_linearity=resnet_act_fn,
|
| 465 |
+
output_scale_factor=output_scale_factor,
|
| 466 |
+
pre_norm=resnet_pre_norm,
|
| 467 |
+
)
|
| 468 |
+
)
|
| 469 |
+
if not mix_block_in_forward:
|
| 470 |
+
modules = sorted(modules, key=custom_sort_order)
|
| 471 |
+
|
| 472 |
+
self.modules_list = nn.ModuleList(modules)
|
| 473 |
+
|
| 474 |
+
self.upsamplers = nn.ModuleList([nn.Identity()])
|
| 475 |
+
if add_upsample:
|
| 476 |
+
self.upsamplers = nn.ModuleList([Upsample2D(in_channels, use_conv=True, out_channels=in_channels)])
|
| 477 |
+
|
| 478 |
+
def forward(
|
| 479 |
+
self,
|
| 480 |
+
hidden_states: torch.FloatTensor,
|
| 481 |
+
temb: Optional[torch.FloatTensor] = None,
|
| 482 |
+
encoder_hidden_states: Optional[torch.FloatTensor] = None,
|
| 483 |
+
attention_mask: Optional[torch.FloatTensor] = None,
|
| 484 |
+
cross_attention_kwargs: Optional[Dict[str, Any]] = None,
|
| 485 |
+
encoder_attention_mask: Optional[torch.FloatTensor] = None,
|
| 486 |
+
) -> torch.FloatTensor:
|
| 487 |
+
hidden_states = self.modules_list[0](hidden_states, temb)
|
| 488 |
+
|
| 489 |
+
for module in self.modules_list:
|
| 490 |
+
if isinstance(module, (FlexibleTransformer2DModel, Transformer2DModel)):
|
| 491 |
+
hidden_states = module(
|
| 492 |
+
hidden_states,
|
| 493 |
+
encoder_hidden_states=encoder_hidden_states,
|
| 494 |
+
cross_attention_kwargs=cross_attention_kwargs,
|
| 495 |
+
attention_mask=attention_mask,
|
| 496 |
+
encoder_attention_mask=encoder_attention_mask,
|
| 497 |
+
return_dict=False,
|
| 498 |
+
)[0]
|
| 499 |
+
elif isinstance(module, ResnetBlock2D):
|
| 500 |
+
hidden_states = module(hidden_states, temb)
|
| 501 |
+
|
| 502 |
+
for upsampler in self.upsamplers:
|
| 503 |
+
hidden_states = upsampler(hidden_states)
|
| 504 |
+
|
| 505 |
+
return hidden_states
|
| 506 |
+
|
| 507 |
+
|
| 508 |
+
class FlexibleTransformer2DModel(ModelMixin, ConfigMixin):
|
| 509 |
+
@register_to_config
|
| 510 |
+
def __init__(
|
| 511 |
+
self,
|
| 512 |
+
num_attention_heads: int = 16,
|
| 513 |
+
attention_head_dim: int = 88,
|
| 514 |
+
in_channels: Optional[int] = None,
|
| 515 |
+
out_channels: Optional[int] = None,
|
| 516 |
+
num_layers: int = 1,
|
| 517 |
+
dropout: float = 0.0,
|
| 518 |
+
norm_num_groups: int = 32,
|
| 519 |
+
cross_attention_dim: Optional[int] = None,
|
| 520 |
+
attention_bias: bool = False,
|
| 521 |
+
activation_fn: str = "geglu",
|
| 522 |
+
num_embeds_ada_norm: Optional[int] = None,
|
| 523 |
+
only_cross_attention: bool = False,
|
| 524 |
+
use_linear_projection: bool = False,
|
| 525 |
+
upcast_attention: bool = False,
|
| 526 |
+
norm_type: str = "layer_norm",
|
| 527 |
+
norm_elementwise_affine: bool = True,
|
| 528 |
+
):
|
| 529 |
+
super().__init__()
|
| 530 |
+
self.num_attention_heads = num_attention_heads
|
| 531 |
+
self.attention_head_dim = attention_head_dim
|
| 532 |
+
self.in_channels = in_channels
|
| 533 |
+
inner_dim = num_attention_heads * attention_head_dim
|
| 534 |
+
|
| 535 |
+
# Define input layers
|
| 536 |
+
self.norm = torch.nn.GroupNorm(num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True)
|
| 537 |
+
self.use_linear_projection = use_linear_projection
|
| 538 |
+
if self.use_linear_projection:
|
| 539 |
+
self.proj_in = nn.Linear(in_channels, inner_dim)
|
| 540 |
+
else:
|
| 541 |
+
self.proj_in = nn.Conv2d(in_channels, inner_dim, kernel_size=1, stride=1, padding=0)
|
| 542 |
+
|
| 543 |
+
# Define transformers blocks
|
| 544 |
+
self.transformer_blocks = nn.ModuleList(
|
| 545 |
+
[
|
| 546 |
+
BasicTransformerBlock(
|
| 547 |
+
inner_dim,
|
| 548 |
+
num_attention_heads,
|
| 549 |
+
attention_head_dim,
|
| 550 |
+
dropout=dropout,
|
| 551 |
+
cross_attention_dim=cross_attention_dim,
|
| 552 |
+
activation_fn=activation_fn,
|
| 553 |
+
num_embeds_ada_norm=num_embeds_ada_norm,
|
| 554 |
+
attention_bias=attention_bias,
|
| 555 |
+
only_cross_attention=only_cross_attention,
|
| 556 |
+
upcast_attention=upcast_attention,
|
| 557 |
+
norm_type=norm_type,
|
| 558 |
+
norm_elementwise_affine=norm_elementwise_affine,
|
| 559 |
+
)
|
| 560 |
+
for _ in range(num_layers)
|
| 561 |
+
]
|
| 562 |
+
)
|
| 563 |
+
|
| 564 |
+
# Define output layers
|
| 565 |
+
self.out_channels = in_channels if out_channels is None else out_channels
|
| 566 |
+
if self.use_linear_projection:
|
| 567 |
+
self.proj_out = nn.Linear(inner_dim, in_channels)
|
| 568 |
+
else:
|
| 569 |
+
self.proj_out = nn.Conv2d(inner_dim, in_channels, kernel_size=1, stride=1, padding=0)
|
| 570 |
+
|
| 571 |
+
def forward(
|
| 572 |
+
self,
|
| 573 |
+
hidden_states: torch.Tensor,
|
| 574 |
+
encoder_hidden_states: Optional[torch.Tensor] = None,
|
| 575 |
+
timestep: Optional[torch.LongTensor] = None,
|
| 576 |
+
class_labels: Optional[torch.LongTensor] = None,
|
| 577 |
+
cross_attention_kwargs: Dict[str, Any] = None,
|
| 578 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 579 |
+
encoder_attention_mask: Optional[torch.Tensor] = None,
|
| 580 |
+
return_dict: bool = False
|
| 581 |
+
):
|
| 582 |
+
# 1. Input
|
| 583 |
+
batch, _, height, width = hidden_states.shape
|
| 584 |
+
residual = hidden_states
|
| 585 |
+
|
| 586 |
+
hidden_states = self.norm(hidden_states)
|
| 587 |
+
if not self.use_linear_projection:
|
| 588 |
+
hidden_states = self.proj_in(hidden_states)
|
| 589 |
+
inner_dim = hidden_states.shape[1]
|
| 590 |
+
hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * width, inner_dim)
|
| 591 |
+
else:
|
| 592 |
+
inner_dim = hidden_states.shape[1]
|
| 593 |
+
hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * width, inner_dim)
|
| 594 |
+
hidden_states = self.proj_in(hidden_states)
|
| 595 |
+
|
| 596 |
+
# 2. Blocks
|
| 597 |
+
for block in self.transformer_blocks:
|
| 598 |
+
hidden_states = block(
|
| 599 |
+
hidden_states,
|
| 600 |
+
attention_mask=attention_mask,
|
| 601 |
+
encoder_hidden_states=encoder_hidden_states,
|
| 602 |
+
encoder_attention_mask=encoder_attention_mask,
|
| 603 |
+
timestep=timestep,
|
| 604 |
+
cross_attention_kwargs=cross_attention_kwargs,
|
| 605 |
+
class_labels=class_labels,
|
| 606 |
+
)
|
| 607 |
+
|
| 608 |
+
# 3. Output
|
| 609 |
+
if not self.use_linear_projection:
|
| 610 |
+
hidden_states = hidden_states.reshape(batch, height, width, inner_dim).permute(0, 3, 1, 2).contiguous()
|
| 611 |
+
hidden_states = self.proj_out(hidden_states)
|
| 612 |
+
else:
|
| 613 |
+
hidden_states = self.proj_out(hidden_states)
|
| 614 |
+
hidden_states = hidden_states.reshape(batch, height, width, inner_dim).permute(0, 3, 1, 2).contiguous()
|
| 615 |
+
|
| 616 |
+
output = hidden_states + residual
|
| 617 |
+
if return_dict:
|
| 618 |
+
return (output,)
|
| 619 |
+
return Transformer2DModelOutput(sample=output)
|
| 620 |
+
|
| 621 |
+
|
| 622 |
+
class DeciDiffusionPipeline(StableDiffusionPipeline):
|
| 623 |
+
deci_default_number_of_iterations = 30
|
| 624 |
+
deci_default_guidance_rescale = 0.7
|
| 625 |
+
|
| 626 |
+
def __init__(self,
|
| 627 |
+
vae: AutoencoderKL,
|
| 628 |
+
text_encoder: CLIPTextModel,
|
| 629 |
+
tokenizer: CLIPTokenizer,
|
| 630 |
+
unet: UNet2DConditionModel,
|
| 631 |
+
scheduler: KarrasDiffusionSchedulers,
|
| 632 |
+
safety_checker: StableDiffusionSafetyChecker,
|
| 633 |
+
feature_extractor: CLIPImageProcessor,
|
| 634 |
+
requires_safety_checker: bool = True
|
| 635 |
+
):
|
| 636 |
+
# Replace UNet with Deci`s unet
|
| 637 |
+
del unet
|
| 638 |
+
unet = FlexibleUNet2DConditionModel()
|
| 639 |
+
|
| 640 |
+
super().__init__(vae=vae,
|
| 641 |
+
text_encoder=text_encoder,
|
| 642 |
+
tokenizer=tokenizer,
|
| 643 |
+
unet=unet,
|
| 644 |
+
scheduler=scheduler,
|
| 645 |
+
safety_checker=safety_checker,
|
| 646 |
+
feature_extractor=feature_extractor,
|
| 647 |
+
requires_safety_checker=requires_safety_checker
|
| 648 |
+
)
|
| 649 |
+
|
| 650 |
+
self.register_modules(vae=vae,
|
| 651 |
+
text_encoder=text_encoder,
|
| 652 |
+
tokenizer=tokenizer,
|
| 653 |
+
unet=unet,
|
| 654 |
+
scheduler=scheduler,
|
| 655 |
+
safety_checker=safety_checker,
|
| 656 |
+
feature_extractor=feature_extractor)
|
| 657 |
+
|
| 658 |
+
def __call__(self, *args, **kwargs):
|
| 659 |
+
# Set up default training parameters (if not given by user specifically)
|
| 660 |
+
if "guidance_rescale" not in kwargs:
|
| 661 |
+
kwargs.update({'guidance_rescale': self.deci_default_guidance_rescale})
|
| 662 |
+
if "num_inference_steps" not in kwargs:
|
| 663 |
+
kwargs.update({'num_inference_steps': self.deci_default_number_of_iterations})
|
| 664 |
+
return super().__call__(*args, **kwargs)
|