Edison commited on
Commit
eb05df9
·
1 Parent(s): 22d1749

Refactor: Update app.py to use diffusers_xl_canny_full and lllyasviel/sd-controlnet-canny models for multi-step generation with MultiControlNet

Browse files
Files changed (1) hide show
  1. app.py +101 -14
app.py CHANGED
@@ -85,10 +85,89 @@
85
  # if __name__ == "__main__":
86
  # interface.launch()
87
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88
  import gradio as gr
89
  import torch
90
- from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
91
- from diffusers.utils import load_image
92
  import cv2
93
  import numpy as np
94
  import logging
@@ -97,21 +176,27 @@ from PIL import Image
97
  # 配置日志记录
98
  logging.basicConfig(level=logging.DEBUG)
99
 
100
- # 加载模型
101
- model_id = "runwayml/stable-diffusion-v1-5"
102
- controlnet_id = "lllyasviel/sd-controlnet-canny"
103
  device = "cpu"
 
 
 
 
 
 
104
 
105
  # 加载 ControlNet 模型
106
- controlnet = ControlNetModel.from_pretrained(controlnet_id, torch_dtype=torch.float32)
 
107
 
108
  # 加载 Stable Diffusion 管道
109
  pipe = StableDiffusionControlNetPipeline.from_pretrained(
110
  model_id,
111
- controlnet=controlnet,
112
  use_safetensors=True,
113
- torch_dtype=torch.float32
114
  )
 
115
  pipe = pipe.to(device)
116
 
117
  def canny_edge_detection(image):
@@ -126,14 +211,16 @@ def generate_image(prompt, input_image):
126
  # 进行 Canny 边缘检测
127
  canny_image = canny_edge_detection(input_image)
128
 
129
- # 使用 ControlNet 和 Stable Diffusion 模型生成图像
130
  with torch.no_grad():
131
  generated_images = pipe(
132
- prompt=prompt,
133
- image=canny_image,
 
 
 
134
  height=500,
135
  width=750,
136
- num_inference_steps=30
137
  ).images
138
 
139
  if not generated_images or not isinstance(generated_images[0], Image.Image):
@@ -157,8 +244,8 @@ interface = gr.Interface(
157
  gr.components.Image(type="pil", label="Input Image")
158
  ],
159
  outputs=gr.components.Image(type="pil", label="Generated Image"),
160
- title="Stable Diffusion with ControlNet Canny",
161
- description="Generate images using a prompt and an input image with Stable Diffusion and ControlNet Canny on CPU."
162
  )
163
 
164
  if __name__ == "__main__":
 
85
  # if __name__ == "__main__":
86
  # interface.launch()
87
 
88
+ # import gradio as gr
89
+ # import torch
90
+ # from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
91
+ # from diffusers.utils import load_image
92
+ # import cv2
93
+ # import numpy as np
94
+ # import logging
95
+ # from PIL import Image
96
+
97
+ # # 配置日志记录
98
+ # logging.basicConfig(level=logging.DEBUG)
99
+
100
+ # # 加载模型
101
+ # # model_id = "runwayml/stable-diffusion-v1-5"
102
+ # model_id = "digiplay/Cetus-Mix-Codaedition_diffusers"
103
+ # controlnet_id = "lllyasviel/sd-controlnet-canny"
104
+ # device = "cpu"
105
+
106
+ # # 加载 ControlNet 模型
107
+ # controlnet = ControlNetModel.from_pretrained(controlnet_id, torch_dtype=torch.float32)
108
+
109
+ # # 加载 Stable Diffusion 管道
110
+ # pipe = StableDiffusionControlNetPipeline.from_pretrained(
111
+ # model_id,
112
+ # controlnet=controlnet,
113
+ # use_safetensors=True,
114
+ # torch_dtype=torch.float32
115
+ # )
116
+ # pipe = pipe.to(device)
117
+
118
+ # def canny_edge_detection(image):
119
+ # image = np.array(image)
120
+ # edges = cv2.Canny(image, 100, 200)
121
+ # return Image.fromarray(edges)
122
+
123
+ # def generate_image(prompt, input_image):
124
+ # try:
125
+ # logging.debug("Starting image generation process...")
126
+
127
+ # # 进行 Canny 边缘检测
128
+ # canny_image = canny_edge_detection(input_image)
129
+
130
+ # # 使用 ControlNet 和 Stable Diffusion 模型生成图像
131
+ # with torch.no_grad():
132
+ # generated_images = pipe(
133
+ # prompt=prompt,
134
+ # image=canny_image,
135
+ # height=500,
136
+ # width=750,
137
+ # num_inference_steps=20
138
+ # ).images
139
+
140
+ # if not generated_images or not isinstance(generated_images[0], Image.Image):
141
+ # error_message = "Error: Generated image is not valid."
142
+ # logging.error(error_message)
143
+ # return error_message
144
+
145
+ # final_image = generated_images[0]
146
+ # logging.debug("Image generation process completed successfully.")
147
+ # return final_image
148
+ # except Exception as e:
149
+ # error_message = f"Error: {str(e)}"
150
+ # logging.error(error_message)
151
+ # return error_message
152
+
153
+ # # 定义 Gradio 接口
154
+ # interface = gr.Interface(
155
+ # fn=generate_image,
156
+ # inputs=[
157
+ # gr.components.Textbox(label="Prompt"),
158
+ # gr.components.Image(type="pil", label="Input Image")
159
+ # ],
160
+ # outputs=gr.components.Image(type="pil", label="Generated Image"),
161
+ # title="Stable Diffusion with ControlNet Canny",
162
+ # description="Generate images using a prompt and an input image with Stable Diffusion and ControlNet Canny on CPU."
163
+ # )
164
+
165
+ # if __name__ == "__main__":
166
+ # interface.launch()
167
+
168
  import gradio as gr
169
  import torch
170
+ from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
 
171
  import cv2
172
  import numpy as np
173
  import logging
 
176
  # 配置日志记录
177
  logging.basicConfig(level=logging.DEBUG)
178
 
179
+ # 设置设备
 
 
180
  device = "cpu"
181
+ torch_dtype = torch.float32
182
+
183
+ # 加载模型
184
+ model_id = "digiplay/Cetus-Mix-Codaedition_diffusers"
185
+ canny_controlnet_id = "lllyasviel/sd-controlnet-canny"
186
+ reference_controlnet_id = "lllyasviel/control_v11f1e_sd15_tile"
187
 
188
  # 加载 ControlNet 模型
189
+ canny_controlnet = ControlNetModel.from_pretrained(canny_controlnet_id, torch_dtype=torch_dtype)
190
+ reference_controlnet = ControlNetModel.from_pretrained(reference_controlnet_id, torch_dtype=torch_dtype)
191
 
192
  # 加载 Stable Diffusion 管道
193
  pipe = StableDiffusionControlNetPipeline.from_pretrained(
194
  model_id,
195
+ controlnet=[canny_controlnet, reference_controlnet],
196
  use_safetensors=True,
197
+ torch_dtype=torch_dtype
198
  )
199
+ pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
200
  pipe = pipe.to(device)
201
 
202
  def canny_edge_detection(image):
 
211
  # 进行 Canny 边缘检测
212
  canny_image = canny_edge_detection(input_image)
213
 
214
+ # 使用 MultiControlNet 和 Stable Diffusion 模型生成图像
215
  with torch.no_grad():
216
  generated_images = pipe(
217
+ prompt,
218
+ image=[canny_image, input_image], # Canny 图像和原始输入图像
219
+ controlnet_conditioning_scale=[1.0, 0.75], # Canny 和 reference 的权重
220
+ guess_mode=False, # 对 reference 使用 False
221
+ num_inference_steps=20,
222
  height=500,
223
  width=750,
 
224
  ).images
225
 
226
  if not generated_images or not isinstance(generated_images[0], Image.Image):
 
244
  gr.components.Image(type="pil", label="Input Image")
245
  ],
246
  outputs=gr.components.Image(type="pil", label="Generated Image"),
247
+ title="Stable Diffusion with MultiControlNet (Canny and Reference)",
248
+ description="Generate images using a prompt and an input image with Stable Diffusion and MultiControlNet (Canny and Reference) on CPU."
249
  )
250
 
251
  if __name__ == "__main__":