Cyanex commited on
Commit
20a7079
·
1 Parent(s): 0b1df6d

Rename app.py to pipeline.py

Browse files
Files changed (2) hide show
  1. app.py +0 -115
  2. pipeline.py +664 -0
app.py DELETED
@@ -1,115 +0,0 @@
1
- import gradio as gr
2
- import torch
3
- from PIL.ImageDraw import Draw
4
- from diffusers import StableDiffusionPipeline
5
- from PIL import Image, ImageOps
6
-
7
-
8
- # Load pipeline once
9
- model_id = 'Deci/DeciDiffusion-v1-0'
10
- device = "cuda" if torch.cuda.is_available() else "cpu"
11
- pipe = StableDiffusionPipeline.from_pretrained(model_id, custom_pipeline=model_id, torch_dtype=torch.float32)
12
- pipe.unet = pipe.unet.from_pretrained(model_id, subfolder='flexible_unet', torch_dtype=torch.float32)
13
- pipe = pipe.to(device)
14
-
15
-
16
- def read_content(file_path: str) -> str:
17
- """read the content of target file
18
- """
19
- with open(file_path, 'r', encoding='utf-8') as f:
20
- content = f.read()
21
-
22
- return content
23
-
24
-
25
- def predict(_prompt: str, _steps: int = 30, _seed: int = 42, _guidance_scale: float = 7.5, _negative_prompt: str = ""):
26
- _negative_prompt = [_negative_prompt] if _negative_prompt else None
27
-
28
- output = pipe(prompt=[_prompt],
29
- negative_prompt=_negative_prompt,
30
- num_inference_steps=int(_steps),
31
- guidance_scale=_guidance_scale,
32
- generator=torch.Generator(device).manual_seed(_seed),
33
- )
34
- output_image = output.images[0]
35
-
36
- # Add border beneath the image with Deci logo + prompt
37
- if len(_prompt) > 52:
38
- _prompt = _prompt[:52] + "..."
39
-
40
- original_image_height = output_image.size[1]
41
- output_image = ImageOps.expand(output_image, border=(0, 0, 0, 64), fill='white')
42
- deci_logo = Image.open('./deci_logo_white.png')
43
- output_image.paste(deci_logo, (0, original_image_height))
44
- Draw(output_image).text((deci_logo.size[0], original_image_height + 26), _prompt, (127, 127, 127))
45
- return output_image
46
-
47
-
48
- css = '''
49
- .gradio-container {
50
- max-width: 1100px !important;
51
- background-image: url(https://huggingface.co/spaces/Deci/Deci-DeciDiffusionClean/resolve/main/background-image.png);
52
- background-size: cover;
53
- background-position: center center;
54
- background-repeat: no-repeat;
55
- }
56
-
57
- .footer {margin-bottom: 45px;margin-top: 35px !important;text-align: center;border-bottom: 1px solid #e5e5e5}
58
- .footer>p {font-size: .8rem; display: inline-block; padding: 0 10px;transform: translateY(10px);background: white}
59
- .dark .footer {border-color: #303030}
60
- .dark .footer>p {background: #0b0f19}
61
- .acknowledgments h4{margin: 1.25em 0 .25em 0;font-weight: bold;font-size: 115%}
62
- @keyframes spin {
63
- from {
64
- transform: rotate(0deg);
65
- }
66
- to {
67
- transform: rotate(360deg);
68
- }
69
- }
70
- '''
71
-
72
- demo = gr.Blocks(css=css, elem_id="total-container")
73
- with demo:
74
- gr.HTML(read_content("header.html"))
75
- with gr.Row():
76
- with gr.Column():
77
- with gr.Row(mobile_collapse=False, equal_height=True):
78
- prompt = gr.Textbox(placeholder="Your prompt", show_label=False, elem_id="prompt", autofocus=True, lines=3, )
79
-
80
- with gr.Accordion(label="Advanced Settings", open=False):
81
- with gr.Row(mobile_collapse=False, equal_height=True):
82
- steps = gr.Slider(value=30, minimum=15, maximum=50, step=1, label="steps", interactive=True)
83
- seed = gr.Slider(value=42, minimum=1, maximum=100, step=1, label="seed", interactive=True)
84
- guidance_scale = gr.Slider(value=7.5, minimum=1, maximum=15, step=0.1, label='guidance_scale', interactive=True)
85
-
86
- with gr.Row(mobile_collapse=False, equal_height=True):
87
- negative_prompt = gr.Textbox(label="negative_prompt", placeholder="Your negative prompt",
88
- info="what you don't want to see in the image", lines=3)
89
- with gr.Row():
90
- btn = gr.Button(value="Generate!", elem_id="run_button")
91
-
92
- with gr.Column():
93
- image_out = gr.Image(label="Output", elem_id="output-img", height=400)
94
-
95
- btn.click(fn=predict,
96
- inputs=[prompt, steps, seed, guidance_scale, negative_prompt],
97
- outputs=[image_out],
98
- api_name='run')
99
-
100
- gr.HTML(
101
- """
102
- <div class="footer">
103
- <p>Model by <a href="https://deci.ai" style="text-decoration: underline;" target="_blank">Deci.ai</a> - Gradio Demo by 🤗 Hugging Face
104
- </p>
105
- </div>
106
- <div class="acknowledgments">
107
- <p><h4>LICENSE</h4>
108
- The model is licensed with a <a href="https://huggingface.co/Deci/DeciDiffusion-v1-0/blob/main/LICENSE-WEIGHTS.md" style="text-decoration: underline;" target="_blank">CreativeML Open RAIL-M</a> license. The authors claim no rights on the outputs you generate, you are free to use them and are accountable for their use which must not go against the provisions set in this license. The license forbids you from sharing any content that violates any laws, produce any harm to a person, disseminate any personal information that would be meant for harm, spread misinformation and target vulnerable groups. For the full list of restrictions please <a href="https://huggingface.co/Deci/DeciDiffusion-v1-0/blob/main/LICENSE-WEIGHTS.md" target="_blank" style="text-decoration: underline;" target="_blank">read the license</a></p>
109
- <p><h4>Biases and content acknowledgment</h4>
110
- Despite how impressive being able to turn text into image is, beware to the fact that this model may output content that reinforces or exacerbates societal biases, as well as realistic faces, pornography and violence. The model was trained on the <a href="https://laion.ai/blog/laion-5b/" style="text-decoration: underline;" target="_blank">LAION-5B dataset</a>, which scraped non-curated image-text-pairs from the internet (the exception being the removal of illegal content) and is meant for research purposes. You can read more in the <a href="https://huggingface.co/Deci/DeciDiffusion-v1-0" style="text-decoration: underline;" target="_blank">model card</a></p>
111
- </div>
112
- """
113
- )
114
-
115
- demo.queue(max_size=50).launch()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
pipeline.py ADDED
@@ -0,0 +1,664 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import itertools
2
+ from typing import Any, Optional, Dict, Tuple
3
+
4
+ import torch
5
+ from diffusers import StableDiffusionPipeline, AutoencoderKL
6
+ from diffusers import Transformer2DModel, ModelMixin, ConfigMixin
7
+ from diffusers import UNet2DConditionModel
8
+ from diffusers.configuration_utils import register_to_config
9
+ from diffusers.models.attention import BasicTransformerBlock
10
+ from diffusers.models.resnet import ResnetBlock2D, Downsample2D, Upsample2D
11
+ from diffusers.models.transformer_2d import Transformer2DModelOutput
12
+ from diffusers.pipelines.stable_diffusion import StableDiffusionSafetyChecker
13
+ from diffusers.schedulers import KarrasDiffusionSchedulers
14
+ from torch import nn
15
+ from transformers import CLIPTextModel, CLIPTokenizer, CLIPImageProcessor
16
+
17
+ FlexibleUnetConfigurations = {
18
+ # General parameters for all blocks
19
+ 'sample_size': 64,
20
+ 'temb_dim': 320 * 4,
21
+ 'resnet_eps': 1e-5,
22
+ 'resnet_act_fn': 'silu',
23
+ 'num_attention_heads': 8,
24
+ 'cross_attention_dim': 768,
25
+
26
+ # Controls modules execute order in unet's forward
27
+ 'mix_block_in_forward': True,
28
+ # Down blocks parameters
29
+ 'down_blocks_in_channels': [320, 320, 640],
30
+ 'down_blocks_out_channels': [320, 640, 1280],
31
+ 'down_blocks_num_attentions': [0, 1, 3],
32
+ 'down_blocks_num_resnets': [2, 2, 1],
33
+ 'add_downsample': [True, True, True],
34
+
35
+ # Middle block parameters
36
+ 'add_upsample_mid_block': True,
37
+ 'mid_num_resnets': 4,
38
+ 'mid_num_attentions': 2,
39
+
40
+ # Up block parameters
41
+ 'prev_output_channels': [1280, 1280, 640],
42
+ 'up_blocks_num_attentions': [6, 3, 0],
43
+ 'up_blocks_num_resnets': [2, 3, 3],
44
+ 'add_upsample': [True, True, False],
45
+ }
46
+
47
+
48
+ def custom_sort_order(obj):
49
+ """
50
+ Key function for sorting order of execution in forward methods
51
+ """
52
+ return {ResnetBlock2D: 0, Transformer2DModel: 1, FlexibleTransformer2DModel: 1}.get(obj.__class__)
53
+
54
+
55
+ class FlexibleUNet2DConditionModel(UNet2DConditionModel, ModelMixin):
56
+ configurations = FlexibleUnetConfigurations
57
+
58
+ @register_to_config
59
+ def __init__(self):
60
+ super().__init__(sample_size=self.configurations.get('sample_size',
61
+ FlexibleUnetConfigurations['sample_size']),
62
+ cross_attention_dim=self.configurations.get("cross_attention_dim",
63
+ FlexibleUnetConfigurations['cross_attention_dim']))
64
+
65
+ num_attention_heads = self.configurations.get("num_attention_heads")
66
+ cross_attention_dim = self.configurations.get("cross_attention_dim")
67
+ mix_block_in_forward = self.configurations.get("mix_block_in_forward")
68
+ resnet_act_fn = self.configurations.get("resnet_act_fn")
69
+ resnet_eps = self.configurations.get("resnet_eps")
70
+ temb_dim = self.configurations.get("temb_dim")
71
+
72
+ ###############
73
+ # Down blocks #
74
+ ###############
75
+ down_blocks_num_attentions = self.configurations.get("down_blocks_num_attentions")
76
+ down_blocks_out_channels = self.configurations.get("down_blocks_out_channels")
77
+ down_blocks_in_channels = self.configurations.get("down_blocks_in_channels")
78
+ down_blocks_num_resnets = self.configurations.get("down_blocks_num_resnets")
79
+ add_downsample = self.configurations.get("add_downsample")
80
+
81
+ self.down_blocks = nn.ModuleList()
82
+
83
+ for i, (in_c, out_c, n_res, n_att, add_down) in enumerate(zip(down_blocks_in_channels, down_blocks_out_channels,
84
+ down_blocks_num_resnets,
85
+ down_blocks_num_attentions,
86
+ add_downsample)):
87
+ last_block = i == len(down_blocks_in_channels) - 1
88
+ self.down_blocks.append(FlexibleCrossAttnDownBlock2D(in_channels=in_c,
89
+ out_channels=out_c,
90
+ temb_channels=temb_dim,
91
+ num_resnets=n_res,
92
+ num_attentions=n_att,
93
+ resnet_eps=resnet_eps,
94
+ resnet_act_fn=resnet_act_fn,
95
+ num_attention_heads=num_attention_heads,
96
+ cross_attention_dim=cross_attention_dim,
97
+ add_downsample=add_down,
98
+ last_block=last_block,
99
+ mix_block_in_forward=mix_block_in_forward))
100
+
101
+ ###############
102
+ # Mid blocks #
103
+ ###############
104
+
105
+ mid_block_add_upsample = self.configurations.get("add_upsample_mid_block")
106
+ mid_num_attentions = self.configurations.get("mid_num_attentions")
107
+ mid_num_resnets = self.configurations.get("mid_num_resnets")
108
+
109
+ self.mid_block = FlexibleUNetMidBlock2DCrossAttn(in_channels=down_blocks_out_channels[-1],
110
+ temb_channels=temb_dim,
111
+ resnet_act_fn=resnet_act_fn,
112
+ resnet_eps=resnet_eps,
113
+ cross_attention_dim=cross_attention_dim,
114
+ num_attention_heads=num_attention_heads,
115
+ num_resnets=mid_num_resnets,
116
+ num_attentions=mid_num_attentions,
117
+ mix_block_in_forward=mix_block_in_forward,
118
+ add_upsample=mid_block_add_upsample
119
+ )
120
+
121
+ ###############
122
+ # Up blocks #
123
+ ###############
124
+
125
+ up_blocks_num_attentions = self.configurations.get("up_blocks_num_attentions")
126
+ up_blocks_num_resnets = self.configurations.get("up_blocks_num_resnets")
127
+ prev_output_channels = self.configurations.get("prev_output_channels")
128
+ up_upsample = self.configurations.get("add_upsample")
129
+
130
+ self.up_blocks = nn.ModuleList()
131
+ for in_c, out_c, prev_out, n_res, n_att, add_up in zip(reversed(down_blocks_in_channels),
132
+ reversed(down_blocks_out_channels),
133
+ prev_output_channels,
134
+ up_blocks_num_resnets, up_blocks_num_attentions,
135
+ up_upsample):
136
+ self.up_blocks.append(FlexibleCrossAttnUpBlock2D(in_channels=in_c,
137
+ out_channels=out_c,
138
+ prev_output_channel=prev_out,
139
+ temb_channels=temb_dim,
140
+ num_resnets=n_res,
141
+ num_attentions=n_att,
142
+ resnet_eps=resnet_eps,
143
+ resnet_act_fn=resnet_act_fn,
144
+ num_attention_heads=num_attention_heads,
145
+ cross_attention_dim=cross_attention_dim,
146
+ add_upsample=add_up,
147
+ mix_block_in_forward=mix_block_in_forward
148
+ ))
149
+
150
+
151
+ class FlexibleCrossAttnDownBlock2D(nn.Module):
152
+ def __init__(
153
+ self,
154
+ in_channels: int,
155
+ out_channels: int,
156
+ temb_channels: int,
157
+ dropout: float = 0.0,
158
+ num_resnets: int = 1,
159
+ num_attentions: int = 1,
160
+ transformer_layers_per_block: int = 1,
161
+ resnet_eps: float = 1e-6,
162
+ resnet_time_scale_shift: str = "default",
163
+ resnet_act_fn: str = "swish",
164
+ resnet_groups: int = 32,
165
+ resnet_pre_norm: bool = True,
166
+ num_attention_heads: int = 1,
167
+ cross_attention_dim: int = 1280,
168
+ output_scale_factor: float = 1.0,
169
+ downsample_padding: int = 1,
170
+ add_downsample: bool = True,
171
+ use_linear_projection: bool = False,
172
+ only_cross_attention: bool = False,
173
+ upcast_attention: bool = False,
174
+ last_block: bool = False,
175
+ mix_block_in_forward: bool = True,
176
+ ):
177
+ super().__init__()
178
+
179
+ self.last_block = last_block
180
+ self.mix_block_in_forward = mix_block_in_forward
181
+ self.has_cross_attention = True
182
+ self.num_attention_heads = num_attention_heads
183
+
184
+ modules = []
185
+
186
+ add_resnets = [True] * num_resnets
187
+ add_cross_attentions = [True] * num_attentions
188
+ for i, (add_resnet, add_cross_attention) in enumerate(
189
+ itertools.zip_longest(add_resnets, add_cross_attentions, fillvalue=False)):
190
+ in_channels = in_channels if i == 0 else out_channels
191
+ if add_resnet:
192
+ modules.append(
193
+ ResnetBlock2D(
194
+ in_channels=in_channels,
195
+ out_channels=out_channels,
196
+ temb_channels=temb_channels,
197
+ eps=resnet_eps,
198
+ groups=resnet_groups,
199
+ dropout=dropout,
200
+ time_embedding_norm=resnet_time_scale_shift,
201
+ non_linearity=resnet_act_fn,
202
+ output_scale_factor=output_scale_factor,
203
+ pre_norm=resnet_pre_norm,
204
+ )
205
+ )
206
+ if add_cross_attention:
207
+ modules.append(
208
+ FlexibleTransformer2DModel(
209
+ num_attention_heads=num_attention_heads,
210
+ attention_head_dim=out_channels // num_attention_heads,
211
+ in_channels=out_channels,
212
+ num_layers=transformer_layers_per_block,
213
+ cross_attention_dim=cross_attention_dim,
214
+ norm_num_groups=resnet_groups,
215
+ use_linear_projection=use_linear_projection,
216
+ only_cross_attention=only_cross_attention,
217
+ upcast_attention=upcast_attention,
218
+ )
219
+ )
220
+
221
+ if not mix_block_in_forward:
222
+ modules = sorted(modules, key=custom_sort_order)
223
+
224
+ self.modules_list = nn.ModuleList(modules)
225
+
226
+ if add_downsample:
227
+ self.downsamplers = nn.ModuleList(
228
+ [
229
+ Downsample2D(
230
+ out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op"
231
+ )
232
+ ]
233
+ )
234
+ else:
235
+ self.downsamplers = None
236
+
237
+ self.gradient_checkpointing = False
238
+
239
+ def forward(
240
+ self,
241
+ hidden_states: torch.FloatTensor,
242
+ temb: Optional[torch.FloatTensor] = None,
243
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
244
+ attention_mask: Optional[torch.FloatTensor] = None,
245
+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,
246
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
247
+ ):
248
+ output_states = ()
249
+
250
+ for module in self.modules_list:
251
+ if isinstance(module, ResnetBlock2D):
252
+ hidden_states = module(hidden_states, temb)
253
+ elif isinstance(module, (FlexibleTransformer2DModel, Transformer2DModel)):
254
+ hidden_states = module(
255
+ hidden_states,
256
+ encoder_hidden_states=encoder_hidden_states,
257
+ cross_attention_kwargs=cross_attention_kwargs,
258
+ attention_mask=attention_mask,
259
+ encoder_attention_mask=encoder_attention_mask,
260
+ return_dict=False,
261
+ )[0]
262
+ else:
263
+ raise ValueError(f'Got an unexpected module in modules list! {type(module)}')
264
+ if isinstance(module, ResnetBlock2D):
265
+ output_states = output_states + (hidden_states,)
266
+
267
+ if self.downsamplers is not None:
268
+ for downsampler in self.downsamplers:
269
+ hidden_states = downsampler(hidden_states)
270
+
271
+ if not self.last_block:
272
+ output_states = output_states + (hidden_states,)
273
+
274
+ return hidden_states, output_states
275
+
276
+
277
+ class FlexibleCrossAttnUpBlock2D(nn.Module):
278
+ def __init__(
279
+ self,
280
+ in_channels: int,
281
+ out_channels: int,
282
+ prev_output_channel: int,
283
+ temb_channels: int,
284
+ dropout: float = 0.0,
285
+ num_resnets: int = 1,
286
+ num_attentions: int = 1,
287
+ transformer_layers_per_block: int = 1,
288
+ resnet_eps: float = 1e-6,
289
+ resnet_time_scale_shift: str = "default",
290
+ resnet_act_fn: str = "swish",
291
+ resnet_groups: int = 32,
292
+ resnet_pre_norm: bool = True,
293
+ num_attention_heads: int = 1,
294
+ cross_attention_dim: int = 1280,
295
+ output_scale_factor: float = 1.0,
296
+ add_upsample: bool = True,
297
+ use_linear_projection: bool = False,
298
+ only_cross_attention: bool = False,
299
+ upcast_attention: bool = False,
300
+ mix_block_in_forward: bool = True
301
+ ):
302
+ super().__init__()
303
+ modules = []
304
+
305
+ # WARNING: This parameter is filled with number of resnets and used within StableDiffusionPipeline
306
+ self.resnets = []
307
+
308
+ self.has_cross_attention = True
309
+ self.num_attention_heads = num_attention_heads
310
+
311
+ add_resnets = [True] * num_resnets
312
+ add_cross_attentions = [True] * num_attentions
313
+ for i, (add_resnet, add_cross_attention) in enumerate(
314
+ itertools.zip_longest(add_resnets, add_cross_attentions, fillvalue=False)):
315
+ res_skip_channels = in_channels if (i == len(add_resnets) - 1) else out_channels
316
+ resnet_in_channels = prev_output_channel if i == 0 else out_channels
317
+
318
+ if add_resnet:
319
+ self.resnets += [True]
320
+ modules.append(
321
+ ResnetBlock2D(
322
+ in_channels=resnet_in_channels + res_skip_channels,
323
+ out_channels=out_channels,
324
+ temb_channels=temb_channels,
325
+ eps=resnet_eps,
326
+ groups=resnet_groups,
327
+ dropout=dropout,
328
+ time_embedding_norm=resnet_time_scale_shift,
329
+ non_linearity=resnet_act_fn,
330
+ output_scale_factor=output_scale_factor,
331
+ pre_norm=resnet_pre_norm,
332
+ )
333
+ )
334
+ if add_cross_attention:
335
+ modules.append(
336
+ FlexibleTransformer2DModel(
337
+ num_attention_heads,
338
+ out_channels // num_attention_heads,
339
+ in_channels=out_channels,
340
+ num_layers=transformer_layers_per_block,
341
+ cross_attention_dim=cross_attention_dim,
342
+ norm_num_groups=resnet_groups,
343
+ use_linear_projection=use_linear_projection,
344
+ only_cross_attention=only_cross_attention,
345
+ upcast_attention=upcast_attention,
346
+ )
347
+ )
348
+
349
+ if not mix_block_in_forward:
350
+ modules = sorted(modules, key=custom_sort_order)
351
+
352
+ self.modules_list = nn.ModuleList(modules)
353
+
354
+ self.upsamplers = None
355
+ if add_upsample:
356
+ self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)])
357
+
358
+ self.gradient_checkpointing = False
359
+
360
+ def forward(
361
+ self,
362
+ hidden_states: torch.FloatTensor,
363
+ res_hidden_states_tuple: Tuple[torch.FloatTensor, ...],
364
+ temb: Optional[torch.FloatTensor] = None,
365
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
366
+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,
367
+ upsample_size: Optional[int] = None,
368
+ attention_mask: Optional[torch.FloatTensor] = None,
369
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
370
+ ):
371
+
372
+ for module in self.modules_list:
373
+ if isinstance(module, ResnetBlock2D):
374
+ res_hidden_states = res_hidden_states_tuple[-1]
375
+ res_hidden_states_tuple = res_hidden_states_tuple[:-1]
376
+ hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)
377
+ hidden_states = module(hidden_states, temb)
378
+ if isinstance(module, (FlexibleTransformer2DModel, Transformer2DModel)):
379
+ hidden_states = module(
380
+ hidden_states,
381
+ encoder_hidden_states=encoder_hidden_states,
382
+ cross_attention_kwargs=cross_attention_kwargs,
383
+ attention_mask=attention_mask,
384
+ encoder_attention_mask=encoder_attention_mask,
385
+ return_dict=False,
386
+ )[0]
387
+
388
+ if self.upsamplers is not None:
389
+ for upsampler in self.upsamplers:
390
+ hidden_states = upsampler(hidden_states, upsample_size)
391
+
392
+ return hidden_states
393
+
394
+
395
+ class FlexibleUNetMidBlock2DCrossAttn(nn.Module):
396
+ def __init__(
397
+ self,
398
+ in_channels: int,
399
+ temb_channels: int,
400
+ dropout: float = 0.0,
401
+ num_resnets: int = 1,
402
+ num_attentions: int = 1,
403
+ transformer_layers_per_block: int = 1,
404
+ resnet_eps: float = 1e-6,
405
+ resnet_time_scale_shift: str = "default",
406
+ resnet_act_fn: str = "swish",
407
+ resnet_groups: int = 32,
408
+ resnet_pre_norm: bool = True,
409
+ num_attention_heads: int = 1,
410
+ output_scale_factor: float = 1.0,
411
+ cross_attention_dim: int = 1280,
412
+ use_linear_projection: bool = False,
413
+ upcast_attention: bool = False,
414
+ mix_block_in_forward: bool = True,
415
+ add_upsample: bool = True,
416
+ ):
417
+ super().__init__()
418
+
419
+ self.has_cross_attention = True
420
+ self.num_attention_heads = num_attention_heads
421
+ resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32)
422
+ # There is always at least one resnet
423
+ modules = [ResnetBlock2D(
424
+ in_channels=in_channels,
425
+ out_channels=in_channels,
426
+ temb_channels=temb_channels,
427
+ eps=resnet_eps,
428
+ groups=resnet_groups,
429
+ dropout=dropout,
430
+ time_embedding_norm=resnet_time_scale_shift,
431
+ non_linearity=resnet_act_fn,
432
+ output_scale_factor=output_scale_factor,
433
+ pre_norm=resnet_pre_norm,
434
+ )]
435
+
436
+ add_resnets = [True] * num_resnets
437
+ add_cross_attentions = [True] * num_attentions
438
+ for i, (add_resnet, add_cross_attention) in enumerate(
439
+ itertools.zip_longest(add_resnets, add_cross_attentions, fillvalue=False)):
440
+ if add_cross_attention:
441
+ modules.append(
442
+ FlexibleTransformer2DModel(
443
+ num_attention_heads,
444
+ in_channels // num_attention_heads,
445
+ in_channels=in_channels,
446
+ num_layers=transformer_layers_per_block,
447
+ cross_attention_dim=cross_attention_dim,
448
+ norm_num_groups=resnet_groups,
449
+ use_linear_projection=use_linear_projection,
450
+ upcast_attention=upcast_attention,
451
+ )
452
+ )
453
+
454
+ if add_resnet:
455
+ modules.append(
456
+ ResnetBlock2D(
457
+ in_channels=in_channels,
458
+ out_channels=in_channels,
459
+ temb_channels=temb_channels,
460
+ eps=resnet_eps,
461
+ groups=resnet_groups,
462
+ dropout=dropout,
463
+ time_embedding_norm=resnet_time_scale_shift,
464
+ non_linearity=resnet_act_fn,
465
+ output_scale_factor=output_scale_factor,
466
+ pre_norm=resnet_pre_norm,
467
+ )
468
+ )
469
+ if not mix_block_in_forward:
470
+ modules = sorted(modules, key=custom_sort_order)
471
+
472
+ self.modules_list = nn.ModuleList(modules)
473
+
474
+ self.upsamplers = nn.ModuleList([nn.Identity()])
475
+ if add_upsample:
476
+ self.upsamplers = nn.ModuleList([Upsample2D(in_channels, use_conv=True, out_channels=in_channels)])
477
+
478
+ def forward(
479
+ self,
480
+ hidden_states: torch.FloatTensor,
481
+ temb: Optional[torch.FloatTensor] = None,
482
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
483
+ attention_mask: Optional[torch.FloatTensor] = None,
484
+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,
485
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
486
+ ) -> torch.FloatTensor:
487
+ hidden_states = self.modules_list[0](hidden_states, temb)
488
+
489
+ for module in self.modules_list:
490
+ if isinstance(module, (FlexibleTransformer2DModel, Transformer2DModel)):
491
+ hidden_states = module(
492
+ hidden_states,
493
+ encoder_hidden_states=encoder_hidden_states,
494
+ cross_attention_kwargs=cross_attention_kwargs,
495
+ attention_mask=attention_mask,
496
+ encoder_attention_mask=encoder_attention_mask,
497
+ return_dict=False,
498
+ )[0]
499
+ elif isinstance(module, ResnetBlock2D):
500
+ hidden_states = module(hidden_states, temb)
501
+
502
+ for upsampler in self.upsamplers:
503
+ hidden_states = upsampler(hidden_states)
504
+
505
+ return hidden_states
506
+
507
+
508
+ class FlexibleTransformer2DModel(ModelMixin, ConfigMixin):
509
+ @register_to_config
510
+ def __init__(
511
+ self,
512
+ num_attention_heads: int = 16,
513
+ attention_head_dim: int = 88,
514
+ in_channels: Optional[int] = None,
515
+ out_channels: Optional[int] = None,
516
+ num_layers: int = 1,
517
+ dropout: float = 0.0,
518
+ norm_num_groups: int = 32,
519
+ cross_attention_dim: Optional[int] = None,
520
+ attention_bias: bool = False,
521
+ activation_fn: str = "geglu",
522
+ num_embeds_ada_norm: Optional[int] = None,
523
+ only_cross_attention: bool = False,
524
+ use_linear_projection: bool = False,
525
+ upcast_attention: bool = False,
526
+ norm_type: str = "layer_norm",
527
+ norm_elementwise_affine: bool = True,
528
+ ):
529
+ super().__init__()
530
+ self.num_attention_heads = num_attention_heads
531
+ self.attention_head_dim = attention_head_dim
532
+ self.in_channels = in_channels
533
+ inner_dim = num_attention_heads * attention_head_dim
534
+
535
+ # Define input layers
536
+ self.norm = torch.nn.GroupNorm(num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True)
537
+ self.use_linear_projection = use_linear_projection
538
+ if self.use_linear_projection:
539
+ self.proj_in = nn.Linear(in_channels, inner_dim)
540
+ else:
541
+ self.proj_in = nn.Conv2d(in_channels, inner_dim, kernel_size=1, stride=1, padding=0)
542
+
543
+ # Define transformers blocks
544
+ self.transformer_blocks = nn.ModuleList(
545
+ [
546
+ BasicTransformerBlock(
547
+ inner_dim,
548
+ num_attention_heads,
549
+ attention_head_dim,
550
+ dropout=dropout,
551
+ cross_attention_dim=cross_attention_dim,
552
+ activation_fn=activation_fn,
553
+ num_embeds_ada_norm=num_embeds_ada_norm,
554
+ attention_bias=attention_bias,
555
+ only_cross_attention=only_cross_attention,
556
+ upcast_attention=upcast_attention,
557
+ norm_type=norm_type,
558
+ norm_elementwise_affine=norm_elementwise_affine,
559
+ )
560
+ for _ in range(num_layers)
561
+ ]
562
+ )
563
+
564
+ # Define output layers
565
+ self.out_channels = in_channels if out_channels is None else out_channels
566
+ if self.use_linear_projection:
567
+ self.proj_out = nn.Linear(inner_dim, in_channels)
568
+ else:
569
+ self.proj_out = nn.Conv2d(inner_dim, in_channels, kernel_size=1, stride=1, padding=0)
570
+
571
+ def forward(
572
+ self,
573
+ hidden_states: torch.Tensor,
574
+ encoder_hidden_states: Optional[torch.Tensor] = None,
575
+ timestep: Optional[torch.LongTensor] = None,
576
+ class_labels: Optional[torch.LongTensor] = None,
577
+ cross_attention_kwargs: Dict[str, Any] = None,
578
+ attention_mask: Optional[torch.Tensor] = None,
579
+ encoder_attention_mask: Optional[torch.Tensor] = None,
580
+ return_dict: bool = False
581
+ ):
582
+ # 1. Input
583
+ batch, _, height, width = hidden_states.shape
584
+ residual = hidden_states
585
+
586
+ hidden_states = self.norm(hidden_states)
587
+ if not self.use_linear_projection:
588
+ hidden_states = self.proj_in(hidden_states)
589
+ inner_dim = hidden_states.shape[1]
590
+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * width, inner_dim)
591
+ else:
592
+ inner_dim = hidden_states.shape[1]
593
+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * width, inner_dim)
594
+ hidden_states = self.proj_in(hidden_states)
595
+
596
+ # 2. Blocks
597
+ for block in self.transformer_blocks:
598
+ hidden_states = block(
599
+ hidden_states,
600
+ attention_mask=attention_mask,
601
+ encoder_hidden_states=encoder_hidden_states,
602
+ encoder_attention_mask=encoder_attention_mask,
603
+ timestep=timestep,
604
+ cross_attention_kwargs=cross_attention_kwargs,
605
+ class_labels=class_labels,
606
+ )
607
+
608
+ # 3. Output
609
+ if not self.use_linear_projection:
610
+ hidden_states = hidden_states.reshape(batch, height, width, inner_dim).permute(0, 3, 1, 2).contiguous()
611
+ hidden_states = self.proj_out(hidden_states)
612
+ else:
613
+ hidden_states = self.proj_out(hidden_states)
614
+ hidden_states = hidden_states.reshape(batch, height, width, inner_dim).permute(0, 3, 1, 2).contiguous()
615
+
616
+ output = hidden_states + residual
617
+ if return_dict:
618
+ return (output,)
619
+ return Transformer2DModelOutput(sample=output)
620
+
621
+
622
+ class DeciDiffusionPipeline(StableDiffusionPipeline):
623
+ deci_default_number_of_iterations = 30
624
+ deci_default_guidance_rescale = 0.7
625
+
626
+ def __init__(self,
627
+ vae: AutoencoderKL,
628
+ text_encoder: CLIPTextModel,
629
+ tokenizer: CLIPTokenizer,
630
+ unet: UNet2DConditionModel,
631
+ scheduler: KarrasDiffusionSchedulers,
632
+ safety_checker: StableDiffusionSafetyChecker,
633
+ feature_extractor: CLIPImageProcessor,
634
+ requires_safety_checker: bool = True
635
+ ):
636
+ # Replace UNet with Deci`s unet
637
+ del unet
638
+ unet = FlexibleUNet2DConditionModel()
639
+
640
+ super().__init__(vae=vae,
641
+ text_encoder=text_encoder,
642
+ tokenizer=tokenizer,
643
+ unet=unet,
644
+ scheduler=scheduler,
645
+ safety_checker=safety_checker,
646
+ feature_extractor=feature_extractor,
647
+ requires_safety_checker=requires_safety_checker
648
+ )
649
+
650
+ self.register_modules(vae=vae,
651
+ text_encoder=text_encoder,
652
+ tokenizer=tokenizer,
653
+ unet=unet,
654
+ scheduler=scheduler,
655
+ safety_checker=safety_checker,
656
+ feature_extractor=feature_extractor)
657
+
658
+ def __call__(self, *args, **kwargs):
659
+ # Set up default training parameters (if not given by user specifically)
660
+ if "guidance_rescale" not in kwargs:
661
+ kwargs.update({'guidance_rescale': self.deci_default_guidance_rescale})
662
+ if "num_inference_steps" not in kwargs:
663
+ kwargs.update({'num_inference_steps': self.deci_default_number_of_iterations})
664
+ return super().__call__(*args, **kwargs)