Cyanex commited on
Commit
eb17f2f
·
1 Parent(s): 20a7079

Delete pipeline.py

Browse files
Files changed (1) hide show
  1. pipeline.py +0 -664
pipeline.py DELETED
@@ -1,664 +0,0 @@
1
- import itertools
2
- from typing import Any, Optional, Dict, Tuple
3
-
4
- import torch
5
- from diffusers import StableDiffusionPipeline, AutoencoderKL
6
- from diffusers import Transformer2DModel, ModelMixin, ConfigMixin
7
- from diffusers import UNet2DConditionModel
8
- from diffusers.configuration_utils import register_to_config
9
- from diffusers.models.attention import BasicTransformerBlock
10
- from diffusers.models.resnet import ResnetBlock2D, Downsample2D, Upsample2D
11
- from diffusers.models.transformer_2d import Transformer2DModelOutput
12
- from diffusers.pipelines.stable_diffusion import StableDiffusionSafetyChecker
13
- from diffusers.schedulers import KarrasDiffusionSchedulers
14
- from torch import nn
15
- from transformers import CLIPTextModel, CLIPTokenizer, CLIPImageProcessor
16
-
17
- FlexibleUnetConfigurations = {
18
- # General parameters for all blocks
19
- 'sample_size': 64,
20
- 'temb_dim': 320 * 4,
21
- 'resnet_eps': 1e-5,
22
- 'resnet_act_fn': 'silu',
23
- 'num_attention_heads': 8,
24
- 'cross_attention_dim': 768,
25
-
26
- # Controls modules execute order in unet's forward
27
- 'mix_block_in_forward': True,
28
- # Down blocks parameters
29
- 'down_blocks_in_channels': [320, 320, 640],
30
- 'down_blocks_out_channels': [320, 640, 1280],
31
- 'down_blocks_num_attentions': [0, 1, 3],
32
- 'down_blocks_num_resnets': [2, 2, 1],
33
- 'add_downsample': [True, True, True],
34
-
35
- # Middle block parameters
36
- 'add_upsample_mid_block': True,
37
- 'mid_num_resnets': 4,
38
- 'mid_num_attentions': 2,
39
-
40
- # Up block parameters
41
- 'prev_output_channels': [1280, 1280, 640],
42
- 'up_blocks_num_attentions': [6, 3, 0],
43
- 'up_blocks_num_resnets': [2, 3, 3],
44
- 'add_upsample': [True, True, False],
45
- }
46
-
47
-
48
- def custom_sort_order(obj):
49
- """
50
- Key function for sorting order of execution in forward methods
51
- """
52
- return {ResnetBlock2D: 0, Transformer2DModel: 1, FlexibleTransformer2DModel: 1}.get(obj.__class__)
53
-
54
-
55
- class FlexibleUNet2DConditionModel(UNet2DConditionModel, ModelMixin):
56
- configurations = FlexibleUnetConfigurations
57
-
58
- @register_to_config
59
- def __init__(self):
60
- super().__init__(sample_size=self.configurations.get('sample_size',
61
- FlexibleUnetConfigurations['sample_size']),
62
- cross_attention_dim=self.configurations.get("cross_attention_dim",
63
- FlexibleUnetConfigurations['cross_attention_dim']))
64
-
65
- num_attention_heads = self.configurations.get("num_attention_heads")
66
- cross_attention_dim = self.configurations.get("cross_attention_dim")
67
- mix_block_in_forward = self.configurations.get("mix_block_in_forward")
68
- resnet_act_fn = self.configurations.get("resnet_act_fn")
69
- resnet_eps = self.configurations.get("resnet_eps")
70
- temb_dim = self.configurations.get("temb_dim")
71
-
72
- ###############
73
- # Down blocks #
74
- ###############
75
- down_blocks_num_attentions = self.configurations.get("down_blocks_num_attentions")
76
- down_blocks_out_channels = self.configurations.get("down_blocks_out_channels")
77
- down_blocks_in_channels = self.configurations.get("down_blocks_in_channels")
78
- down_blocks_num_resnets = self.configurations.get("down_blocks_num_resnets")
79
- add_downsample = self.configurations.get("add_downsample")
80
-
81
- self.down_blocks = nn.ModuleList()
82
-
83
- for i, (in_c, out_c, n_res, n_att, add_down) in enumerate(zip(down_blocks_in_channels, down_blocks_out_channels,
84
- down_blocks_num_resnets,
85
- down_blocks_num_attentions,
86
- add_downsample)):
87
- last_block = i == len(down_blocks_in_channels) - 1
88
- self.down_blocks.append(FlexibleCrossAttnDownBlock2D(in_channels=in_c,
89
- out_channels=out_c,
90
- temb_channels=temb_dim,
91
- num_resnets=n_res,
92
- num_attentions=n_att,
93
- resnet_eps=resnet_eps,
94
- resnet_act_fn=resnet_act_fn,
95
- num_attention_heads=num_attention_heads,
96
- cross_attention_dim=cross_attention_dim,
97
- add_downsample=add_down,
98
- last_block=last_block,
99
- mix_block_in_forward=mix_block_in_forward))
100
-
101
- ###############
102
- # Mid blocks #
103
- ###############
104
-
105
- mid_block_add_upsample = self.configurations.get("add_upsample_mid_block")
106
- mid_num_attentions = self.configurations.get("mid_num_attentions")
107
- mid_num_resnets = self.configurations.get("mid_num_resnets")
108
-
109
- self.mid_block = FlexibleUNetMidBlock2DCrossAttn(in_channels=down_blocks_out_channels[-1],
110
- temb_channels=temb_dim,
111
- resnet_act_fn=resnet_act_fn,
112
- resnet_eps=resnet_eps,
113
- cross_attention_dim=cross_attention_dim,
114
- num_attention_heads=num_attention_heads,
115
- num_resnets=mid_num_resnets,
116
- num_attentions=mid_num_attentions,
117
- mix_block_in_forward=mix_block_in_forward,
118
- add_upsample=mid_block_add_upsample
119
- )
120
-
121
- ###############
122
- # Up blocks #
123
- ###############
124
-
125
- up_blocks_num_attentions = self.configurations.get("up_blocks_num_attentions")
126
- up_blocks_num_resnets = self.configurations.get("up_blocks_num_resnets")
127
- prev_output_channels = self.configurations.get("prev_output_channels")
128
- up_upsample = self.configurations.get("add_upsample")
129
-
130
- self.up_blocks = nn.ModuleList()
131
- for in_c, out_c, prev_out, n_res, n_att, add_up in zip(reversed(down_blocks_in_channels),
132
- reversed(down_blocks_out_channels),
133
- prev_output_channels,
134
- up_blocks_num_resnets, up_blocks_num_attentions,
135
- up_upsample):
136
- self.up_blocks.append(FlexibleCrossAttnUpBlock2D(in_channels=in_c,
137
- out_channels=out_c,
138
- prev_output_channel=prev_out,
139
- temb_channels=temb_dim,
140
- num_resnets=n_res,
141
- num_attentions=n_att,
142
- resnet_eps=resnet_eps,
143
- resnet_act_fn=resnet_act_fn,
144
- num_attention_heads=num_attention_heads,
145
- cross_attention_dim=cross_attention_dim,
146
- add_upsample=add_up,
147
- mix_block_in_forward=mix_block_in_forward
148
- ))
149
-
150
-
151
- class FlexibleCrossAttnDownBlock2D(nn.Module):
152
- def __init__(
153
- self,
154
- in_channels: int,
155
- out_channels: int,
156
- temb_channels: int,
157
- dropout: float = 0.0,
158
- num_resnets: int = 1,
159
- num_attentions: int = 1,
160
- transformer_layers_per_block: int = 1,
161
- resnet_eps: float = 1e-6,
162
- resnet_time_scale_shift: str = "default",
163
- resnet_act_fn: str = "swish",
164
- resnet_groups: int = 32,
165
- resnet_pre_norm: bool = True,
166
- num_attention_heads: int = 1,
167
- cross_attention_dim: int = 1280,
168
- output_scale_factor: float = 1.0,
169
- downsample_padding: int = 1,
170
- add_downsample: bool = True,
171
- use_linear_projection: bool = False,
172
- only_cross_attention: bool = False,
173
- upcast_attention: bool = False,
174
- last_block: bool = False,
175
- mix_block_in_forward: bool = True,
176
- ):
177
- super().__init__()
178
-
179
- self.last_block = last_block
180
- self.mix_block_in_forward = mix_block_in_forward
181
- self.has_cross_attention = True
182
- self.num_attention_heads = num_attention_heads
183
-
184
- modules = []
185
-
186
- add_resnets = [True] * num_resnets
187
- add_cross_attentions = [True] * num_attentions
188
- for i, (add_resnet, add_cross_attention) in enumerate(
189
- itertools.zip_longest(add_resnets, add_cross_attentions, fillvalue=False)):
190
- in_channels = in_channels if i == 0 else out_channels
191
- if add_resnet:
192
- modules.append(
193
- ResnetBlock2D(
194
- in_channels=in_channels,
195
- out_channels=out_channels,
196
- temb_channels=temb_channels,
197
- eps=resnet_eps,
198
- groups=resnet_groups,
199
- dropout=dropout,
200
- time_embedding_norm=resnet_time_scale_shift,
201
- non_linearity=resnet_act_fn,
202
- output_scale_factor=output_scale_factor,
203
- pre_norm=resnet_pre_norm,
204
- )
205
- )
206
- if add_cross_attention:
207
- modules.append(
208
- FlexibleTransformer2DModel(
209
- num_attention_heads=num_attention_heads,
210
- attention_head_dim=out_channels // num_attention_heads,
211
- in_channels=out_channels,
212
- num_layers=transformer_layers_per_block,
213
- cross_attention_dim=cross_attention_dim,
214
- norm_num_groups=resnet_groups,
215
- use_linear_projection=use_linear_projection,
216
- only_cross_attention=only_cross_attention,
217
- upcast_attention=upcast_attention,
218
- )
219
- )
220
-
221
- if not mix_block_in_forward:
222
- modules = sorted(modules, key=custom_sort_order)
223
-
224
- self.modules_list = nn.ModuleList(modules)
225
-
226
- if add_downsample:
227
- self.downsamplers = nn.ModuleList(
228
- [
229
- Downsample2D(
230
- out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op"
231
- )
232
- ]
233
- )
234
- else:
235
- self.downsamplers = None
236
-
237
- self.gradient_checkpointing = False
238
-
239
- def forward(
240
- self,
241
- hidden_states: torch.FloatTensor,
242
- temb: Optional[torch.FloatTensor] = None,
243
- encoder_hidden_states: Optional[torch.FloatTensor] = None,
244
- attention_mask: Optional[torch.FloatTensor] = None,
245
- cross_attention_kwargs: Optional[Dict[str, Any]] = None,
246
- encoder_attention_mask: Optional[torch.FloatTensor] = None,
247
- ):
248
- output_states = ()
249
-
250
- for module in self.modules_list:
251
- if isinstance(module, ResnetBlock2D):
252
- hidden_states = module(hidden_states, temb)
253
- elif isinstance(module, (FlexibleTransformer2DModel, Transformer2DModel)):
254
- hidden_states = module(
255
- hidden_states,
256
- encoder_hidden_states=encoder_hidden_states,
257
- cross_attention_kwargs=cross_attention_kwargs,
258
- attention_mask=attention_mask,
259
- encoder_attention_mask=encoder_attention_mask,
260
- return_dict=False,
261
- )[0]
262
- else:
263
- raise ValueError(f'Got an unexpected module in modules list! {type(module)}')
264
- if isinstance(module, ResnetBlock2D):
265
- output_states = output_states + (hidden_states,)
266
-
267
- if self.downsamplers is not None:
268
- for downsampler in self.downsamplers:
269
- hidden_states = downsampler(hidden_states)
270
-
271
- if not self.last_block:
272
- output_states = output_states + (hidden_states,)
273
-
274
- return hidden_states, output_states
275
-
276
-
277
- class FlexibleCrossAttnUpBlock2D(nn.Module):
278
- def __init__(
279
- self,
280
- in_channels: int,
281
- out_channels: int,
282
- prev_output_channel: int,
283
- temb_channels: int,
284
- dropout: float = 0.0,
285
- num_resnets: int = 1,
286
- num_attentions: int = 1,
287
- transformer_layers_per_block: int = 1,
288
- resnet_eps: float = 1e-6,
289
- resnet_time_scale_shift: str = "default",
290
- resnet_act_fn: str = "swish",
291
- resnet_groups: int = 32,
292
- resnet_pre_norm: bool = True,
293
- num_attention_heads: int = 1,
294
- cross_attention_dim: int = 1280,
295
- output_scale_factor: float = 1.0,
296
- add_upsample: bool = True,
297
- use_linear_projection: bool = False,
298
- only_cross_attention: bool = False,
299
- upcast_attention: bool = False,
300
- mix_block_in_forward: bool = True
301
- ):
302
- super().__init__()
303
- modules = []
304
-
305
- # WARNING: This parameter is filled with number of resnets and used within StableDiffusionPipeline
306
- self.resnets = []
307
-
308
- self.has_cross_attention = True
309
- self.num_attention_heads = num_attention_heads
310
-
311
- add_resnets = [True] * num_resnets
312
- add_cross_attentions = [True] * num_attentions
313
- for i, (add_resnet, add_cross_attention) in enumerate(
314
- itertools.zip_longest(add_resnets, add_cross_attentions, fillvalue=False)):
315
- res_skip_channels = in_channels if (i == len(add_resnets) - 1) else out_channels
316
- resnet_in_channels = prev_output_channel if i == 0 else out_channels
317
-
318
- if add_resnet:
319
- self.resnets += [True]
320
- modules.append(
321
- ResnetBlock2D(
322
- in_channels=resnet_in_channels + res_skip_channels,
323
- out_channels=out_channels,
324
- temb_channels=temb_channels,
325
- eps=resnet_eps,
326
- groups=resnet_groups,
327
- dropout=dropout,
328
- time_embedding_norm=resnet_time_scale_shift,
329
- non_linearity=resnet_act_fn,
330
- output_scale_factor=output_scale_factor,
331
- pre_norm=resnet_pre_norm,
332
- )
333
- )
334
- if add_cross_attention:
335
- modules.append(
336
- FlexibleTransformer2DModel(
337
- num_attention_heads,
338
- out_channels // num_attention_heads,
339
- in_channels=out_channels,
340
- num_layers=transformer_layers_per_block,
341
- cross_attention_dim=cross_attention_dim,
342
- norm_num_groups=resnet_groups,
343
- use_linear_projection=use_linear_projection,
344
- only_cross_attention=only_cross_attention,
345
- upcast_attention=upcast_attention,
346
- )
347
- )
348
-
349
- if not mix_block_in_forward:
350
- modules = sorted(modules, key=custom_sort_order)
351
-
352
- self.modules_list = nn.ModuleList(modules)
353
-
354
- self.upsamplers = None
355
- if add_upsample:
356
- self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)])
357
-
358
- self.gradient_checkpointing = False
359
-
360
- def forward(
361
- self,
362
- hidden_states: torch.FloatTensor,
363
- res_hidden_states_tuple: Tuple[torch.FloatTensor, ...],
364
- temb: Optional[torch.FloatTensor] = None,
365
- encoder_hidden_states: Optional[torch.FloatTensor] = None,
366
- cross_attention_kwargs: Optional[Dict[str, Any]] = None,
367
- upsample_size: Optional[int] = None,
368
- attention_mask: Optional[torch.FloatTensor] = None,
369
- encoder_attention_mask: Optional[torch.FloatTensor] = None,
370
- ):
371
-
372
- for module in self.modules_list:
373
- if isinstance(module, ResnetBlock2D):
374
- res_hidden_states = res_hidden_states_tuple[-1]
375
- res_hidden_states_tuple = res_hidden_states_tuple[:-1]
376
- hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)
377
- hidden_states = module(hidden_states, temb)
378
- if isinstance(module, (FlexibleTransformer2DModel, Transformer2DModel)):
379
- hidden_states = module(
380
- hidden_states,
381
- encoder_hidden_states=encoder_hidden_states,
382
- cross_attention_kwargs=cross_attention_kwargs,
383
- attention_mask=attention_mask,
384
- encoder_attention_mask=encoder_attention_mask,
385
- return_dict=False,
386
- )[0]
387
-
388
- if self.upsamplers is not None:
389
- for upsampler in self.upsamplers:
390
- hidden_states = upsampler(hidden_states, upsample_size)
391
-
392
- return hidden_states
393
-
394
-
395
- class FlexibleUNetMidBlock2DCrossAttn(nn.Module):
396
- def __init__(
397
- self,
398
- in_channels: int,
399
- temb_channels: int,
400
- dropout: float = 0.0,
401
- num_resnets: int = 1,
402
- num_attentions: int = 1,
403
- transformer_layers_per_block: int = 1,
404
- resnet_eps: float = 1e-6,
405
- resnet_time_scale_shift: str = "default",
406
- resnet_act_fn: str = "swish",
407
- resnet_groups: int = 32,
408
- resnet_pre_norm: bool = True,
409
- num_attention_heads: int = 1,
410
- output_scale_factor: float = 1.0,
411
- cross_attention_dim: int = 1280,
412
- use_linear_projection: bool = False,
413
- upcast_attention: bool = False,
414
- mix_block_in_forward: bool = True,
415
- add_upsample: bool = True,
416
- ):
417
- super().__init__()
418
-
419
- self.has_cross_attention = True
420
- self.num_attention_heads = num_attention_heads
421
- resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32)
422
- # There is always at least one resnet
423
- modules = [ResnetBlock2D(
424
- in_channels=in_channels,
425
- out_channels=in_channels,
426
- temb_channels=temb_channels,
427
- eps=resnet_eps,
428
- groups=resnet_groups,
429
- dropout=dropout,
430
- time_embedding_norm=resnet_time_scale_shift,
431
- non_linearity=resnet_act_fn,
432
- output_scale_factor=output_scale_factor,
433
- pre_norm=resnet_pre_norm,
434
- )]
435
-
436
- add_resnets = [True] * num_resnets
437
- add_cross_attentions = [True] * num_attentions
438
- for i, (add_resnet, add_cross_attention) in enumerate(
439
- itertools.zip_longest(add_resnets, add_cross_attentions, fillvalue=False)):
440
- if add_cross_attention:
441
- modules.append(
442
- FlexibleTransformer2DModel(
443
- num_attention_heads,
444
- in_channels // num_attention_heads,
445
- in_channels=in_channels,
446
- num_layers=transformer_layers_per_block,
447
- cross_attention_dim=cross_attention_dim,
448
- norm_num_groups=resnet_groups,
449
- use_linear_projection=use_linear_projection,
450
- upcast_attention=upcast_attention,
451
- )
452
- )
453
-
454
- if add_resnet:
455
- modules.append(
456
- ResnetBlock2D(
457
- in_channels=in_channels,
458
- out_channels=in_channels,
459
- temb_channels=temb_channels,
460
- eps=resnet_eps,
461
- groups=resnet_groups,
462
- dropout=dropout,
463
- time_embedding_norm=resnet_time_scale_shift,
464
- non_linearity=resnet_act_fn,
465
- output_scale_factor=output_scale_factor,
466
- pre_norm=resnet_pre_norm,
467
- )
468
- )
469
- if not mix_block_in_forward:
470
- modules = sorted(modules, key=custom_sort_order)
471
-
472
- self.modules_list = nn.ModuleList(modules)
473
-
474
- self.upsamplers = nn.ModuleList([nn.Identity()])
475
- if add_upsample:
476
- self.upsamplers = nn.ModuleList([Upsample2D(in_channels, use_conv=True, out_channels=in_channels)])
477
-
478
- def forward(
479
- self,
480
- hidden_states: torch.FloatTensor,
481
- temb: Optional[torch.FloatTensor] = None,
482
- encoder_hidden_states: Optional[torch.FloatTensor] = None,
483
- attention_mask: Optional[torch.FloatTensor] = None,
484
- cross_attention_kwargs: Optional[Dict[str, Any]] = None,
485
- encoder_attention_mask: Optional[torch.FloatTensor] = None,
486
- ) -> torch.FloatTensor:
487
- hidden_states = self.modules_list[0](hidden_states, temb)
488
-
489
- for module in self.modules_list:
490
- if isinstance(module, (FlexibleTransformer2DModel, Transformer2DModel)):
491
- hidden_states = module(
492
- hidden_states,
493
- encoder_hidden_states=encoder_hidden_states,
494
- cross_attention_kwargs=cross_attention_kwargs,
495
- attention_mask=attention_mask,
496
- encoder_attention_mask=encoder_attention_mask,
497
- return_dict=False,
498
- )[0]
499
- elif isinstance(module, ResnetBlock2D):
500
- hidden_states = module(hidden_states, temb)
501
-
502
- for upsampler in self.upsamplers:
503
- hidden_states = upsampler(hidden_states)
504
-
505
- return hidden_states
506
-
507
-
508
- class FlexibleTransformer2DModel(ModelMixin, ConfigMixin):
509
- @register_to_config
510
- def __init__(
511
- self,
512
- num_attention_heads: int = 16,
513
- attention_head_dim: int = 88,
514
- in_channels: Optional[int] = None,
515
- out_channels: Optional[int] = None,
516
- num_layers: int = 1,
517
- dropout: float = 0.0,
518
- norm_num_groups: int = 32,
519
- cross_attention_dim: Optional[int] = None,
520
- attention_bias: bool = False,
521
- activation_fn: str = "geglu",
522
- num_embeds_ada_norm: Optional[int] = None,
523
- only_cross_attention: bool = False,
524
- use_linear_projection: bool = False,
525
- upcast_attention: bool = False,
526
- norm_type: str = "layer_norm",
527
- norm_elementwise_affine: bool = True,
528
- ):
529
- super().__init__()
530
- self.num_attention_heads = num_attention_heads
531
- self.attention_head_dim = attention_head_dim
532
- self.in_channels = in_channels
533
- inner_dim = num_attention_heads * attention_head_dim
534
-
535
- # Define input layers
536
- self.norm = torch.nn.GroupNorm(num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True)
537
- self.use_linear_projection = use_linear_projection
538
- if self.use_linear_projection:
539
- self.proj_in = nn.Linear(in_channels, inner_dim)
540
- else:
541
- self.proj_in = nn.Conv2d(in_channels, inner_dim, kernel_size=1, stride=1, padding=0)
542
-
543
- # Define transformers blocks
544
- self.transformer_blocks = nn.ModuleList(
545
- [
546
- BasicTransformerBlock(
547
- inner_dim,
548
- num_attention_heads,
549
- attention_head_dim,
550
- dropout=dropout,
551
- cross_attention_dim=cross_attention_dim,
552
- activation_fn=activation_fn,
553
- num_embeds_ada_norm=num_embeds_ada_norm,
554
- attention_bias=attention_bias,
555
- only_cross_attention=only_cross_attention,
556
- upcast_attention=upcast_attention,
557
- norm_type=norm_type,
558
- norm_elementwise_affine=norm_elementwise_affine,
559
- )
560
- for _ in range(num_layers)
561
- ]
562
- )
563
-
564
- # Define output layers
565
- self.out_channels = in_channels if out_channels is None else out_channels
566
- if self.use_linear_projection:
567
- self.proj_out = nn.Linear(inner_dim, in_channels)
568
- else:
569
- self.proj_out = nn.Conv2d(inner_dim, in_channels, kernel_size=1, stride=1, padding=0)
570
-
571
- def forward(
572
- self,
573
- hidden_states: torch.Tensor,
574
- encoder_hidden_states: Optional[torch.Tensor] = None,
575
- timestep: Optional[torch.LongTensor] = None,
576
- class_labels: Optional[torch.LongTensor] = None,
577
- cross_attention_kwargs: Dict[str, Any] = None,
578
- attention_mask: Optional[torch.Tensor] = None,
579
- encoder_attention_mask: Optional[torch.Tensor] = None,
580
- return_dict: bool = False
581
- ):
582
- # 1. Input
583
- batch, _, height, width = hidden_states.shape
584
- residual = hidden_states
585
-
586
- hidden_states = self.norm(hidden_states)
587
- if not self.use_linear_projection:
588
- hidden_states = self.proj_in(hidden_states)
589
- inner_dim = hidden_states.shape[1]
590
- hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * width, inner_dim)
591
- else:
592
- inner_dim = hidden_states.shape[1]
593
- hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * width, inner_dim)
594
- hidden_states = self.proj_in(hidden_states)
595
-
596
- # 2. Blocks
597
- for block in self.transformer_blocks:
598
- hidden_states = block(
599
- hidden_states,
600
- attention_mask=attention_mask,
601
- encoder_hidden_states=encoder_hidden_states,
602
- encoder_attention_mask=encoder_attention_mask,
603
- timestep=timestep,
604
- cross_attention_kwargs=cross_attention_kwargs,
605
- class_labels=class_labels,
606
- )
607
-
608
- # 3. Output
609
- if not self.use_linear_projection:
610
- hidden_states = hidden_states.reshape(batch, height, width, inner_dim).permute(0, 3, 1, 2).contiguous()
611
- hidden_states = self.proj_out(hidden_states)
612
- else:
613
- hidden_states = self.proj_out(hidden_states)
614
- hidden_states = hidden_states.reshape(batch, height, width, inner_dim).permute(0, 3, 1, 2).contiguous()
615
-
616
- output = hidden_states + residual
617
- if return_dict:
618
- return (output,)
619
- return Transformer2DModelOutput(sample=output)
620
-
621
-
622
- class DeciDiffusionPipeline(StableDiffusionPipeline):
623
- deci_default_number_of_iterations = 30
624
- deci_default_guidance_rescale = 0.7
625
-
626
- def __init__(self,
627
- vae: AutoencoderKL,
628
- text_encoder: CLIPTextModel,
629
- tokenizer: CLIPTokenizer,
630
- unet: UNet2DConditionModel,
631
- scheduler: KarrasDiffusionSchedulers,
632
- safety_checker: StableDiffusionSafetyChecker,
633
- feature_extractor: CLIPImageProcessor,
634
- requires_safety_checker: bool = True
635
- ):
636
- # Replace UNet with Deci`s unet
637
- del unet
638
- unet = FlexibleUNet2DConditionModel()
639
-
640
- super().__init__(vae=vae,
641
- text_encoder=text_encoder,
642
- tokenizer=tokenizer,
643
- unet=unet,
644
- scheduler=scheduler,
645
- safety_checker=safety_checker,
646
- feature_extractor=feature_extractor,
647
- requires_safety_checker=requires_safety_checker
648
- )
649
-
650
- self.register_modules(vae=vae,
651
- text_encoder=text_encoder,
652
- tokenizer=tokenizer,
653
- unet=unet,
654
- scheduler=scheduler,
655
- safety_checker=safety_checker,
656
- feature_extractor=feature_extractor)
657
-
658
- def __call__(self, *args, **kwargs):
659
- # Set up default training parameters (if not given by user specifically)
660
- if "guidance_rescale" not in kwargs:
661
- kwargs.update({'guidance_rescale': self.deci_default_guidance_rescale})
662
- if "num_inference_steps" not in kwargs:
663
- kwargs.update({'num_inference_steps': self.deci_default_number_of_iterations})
664
- return super().__call__(*args, **kwargs)