Edison commited on
Commit
2fd927b
·
1 Parent(s): c6a3dc4

Refactor: Refactor app.py to use diffusers_xl_canny_full and lllyasviel/sd-controlnet-canny models for multi-step generation

Browse files
Files changed (1) hide show
  1. app.py +8 -27
app.py CHANGED
@@ -2,50 +2,31 @@ import gradio as gr
2
  import torch
3
  from io import BytesIO
4
  from PIL import Image
5
- import requests
6
  from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
7
 
8
- # 加載模型和ControlNet
9
  model_id = "CompVis/stable-diffusion-v1-4"
10
  controlnet_id = "lllyasviel/sd-controlnet-canny"
11
 
12
  pipe = StableDiffusionControlNetPipeline.from_pretrained(model_id, controlnet=ControlNetModel.from_pretrained(controlnet_id))
13
  pipe = pipe.to("cuda" if torch.cuda.is_available() else "cpu")
14
 
15
- # 定义函数来调用CLIP-Interrogator-2
16
- def get_clip_interrogator_prompt(image):
17
- url = "https://huggingface.co/spaces/fffiloni/CLIP-Interrogator-2/run/predict"
18
- buffered = BytesIO()
19
- image.save(buffered, format="JPEG")
20
- img_str = buffered.getvalue()
21
- try:
22
- response = requests.post(url, files={"file": img_str}, timeout=120)
23
- response.raise_for_status() # 检查请求是否成功
24
- prompt = response.json()["data"][0]["caption"]
25
- return prompt
26
- except requests.exceptions.RequestException as e:
27
- return f"Error: {str(e)}"
28
-
29
  def generate_image(control_image, additional_prompt, reference_image1, reference_image2):
30
- # 第一步:使用 canny 模型生成初始图像
31
- base_prompt = get_clip_interrogator_prompt(control_image)
32
- if base_prompt.startswith("Error:"):
33
- return base_prompt
34
- combined_prompt = base_prompt + " " + additional_prompt
35
  try:
36
- initial_image = pipe(prompt=combined_prompt, control_image=control_image).images[0]
 
37
  except Exception as e:
38
  return f"Error in Step 1: {str(e)}"
39
 
40
- # 第二步:使用第一张参考图像进行 reference_adain+attn 生成
41
  try:
42
- step2_image = pipe(prompt=combined_prompt, control_image=reference_image1).images[0]
 
43
  except Exception as e:
44
  return f"Error in Step 2: {str(e)}"
45
 
46
- # 第三步:使用第二张参考图像进行 reference_adain+attn 生成
47
  try:
48
- final_image = pipe(prompt=combined_prompt, control_image=reference_image2).images[0]
 
49
  return final_image
50
  except Exception as e:
51
  return f"Error in Step 3: {str(e)}"
@@ -61,7 +42,7 @@ interface = gr.Interface(
61
  ],
62
  outputs=gr.components.Image(type="pil", label="Generated Image"),
63
  title="Stable Diffusion with Multi-Step ControlNet",
64
- description="Generate images using a multi-step process with Stable Diffusion, ControlNet, and CLIP Interrogator."
65
  )
66
 
67
  if __name__ == "__main__":
 
2
  import torch
3
  from io import BytesIO
4
  from PIL import Image
 
5
  from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
6
 
7
+ # 加载模型和ControlNet
8
  model_id = "CompVis/stable-diffusion-v1-4"
9
  controlnet_id = "lllyasviel/sd-controlnet-canny"
10
 
11
  pipe = StableDiffusionControlNetPipeline.from_pretrained(model_id, controlnet=ControlNetModel.from_pretrained(controlnet_id))
12
  pipe = pipe.to("cuda" if torch.cuda.is_available() else "cpu")
13
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
  def generate_image(control_image, additional_prompt, reference_image1, reference_image2):
 
 
 
 
 
15
  try:
16
+ # 第一步:使用 canny 模型生成初始图像
17
+ initial_image = pipe(prompt=additional_prompt, control_image=control_image).images[0]
18
  except Exception as e:
19
  return f"Error in Step 1: {str(e)}"
20
 
 
21
  try:
22
+ # 第二步:使用第一张参考图像进行 reference_adain+attn 生成
23
+ step2_image = pipe(prompt=additional_prompt, control_image=reference_image1).images[0]
24
  except Exception as e:
25
  return f"Error in Step 2: {str(e)}"
26
 
 
27
  try:
28
+ # 第三步:使用第二张参考图像进行 reference_adain+attn 生成
29
+ final_image = pipe(prompt=additional_prompt, control_image=reference_image2).images[0]
30
  return final_image
31
  except Exception as e:
32
  return f"Error in Step 3: {str(e)}"
 
42
  ],
43
  outputs=gr.components.Image(type="pil", label="Generated Image"),
44
  title="Stable Diffusion with Multi-Step ControlNet",
45
+ description="Generate images using a multi-step process with Stable Diffusion and ControlNet."
46
  )
47
 
48
  if __name__ == "__main__":