Spaces:
Build error
Build error
Refactor: Refactor app.py to use diffusers_xl_canny_full and lllyasviel/sd-controlnet-canny models for multi-step generation
Browse files
app.py
CHANGED
|
@@ -2,50 +2,31 @@ import gradio as gr
|
|
| 2 |
import torch
|
| 3 |
from io import BytesIO
|
| 4 |
from PIL import Image
|
| 5 |
-
import requests
|
| 6 |
from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
|
| 7 |
|
| 8 |
-
# 加
|
| 9 |
model_id = "CompVis/stable-diffusion-v1-4"
|
| 10 |
controlnet_id = "lllyasviel/sd-controlnet-canny"
|
| 11 |
|
| 12 |
pipe = StableDiffusionControlNetPipeline.from_pretrained(model_id, controlnet=ControlNetModel.from_pretrained(controlnet_id))
|
| 13 |
pipe = pipe.to("cuda" if torch.cuda.is_available() else "cpu")
|
| 14 |
|
| 15 |
-
# 定义函数来调用CLIP-Interrogator-2
|
| 16 |
-
def get_clip_interrogator_prompt(image):
|
| 17 |
-
url = "https://huggingface.co/spaces/fffiloni/CLIP-Interrogator-2/run/predict"
|
| 18 |
-
buffered = BytesIO()
|
| 19 |
-
image.save(buffered, format="JPEG")
|
| 20 |
-
img_str = buffered.getvalue()
|
| 21 |
-
try:
|
| 22 |
-
response = requests.post(url, files={"file": img_str}, timeout=120)
|
| 23 |
-
response.raise_for_status() # 检查请求是否成功
|
| 24 |
-
prompt = response.json()["data"][0]["caption"]
|
| 25 |
-
return prompt
|
| 26 |
-
except requests.exceptions.RequestException as e:
|
| 27 |
-
return f"Error: {str(e)}"
|
| 28 |
-
|
| 29 |
def generate_image(control_image, additional_prompt, reference_image1, reference_image2):
|
| 30 |
-
# 第一步:使用 canny 模型生成初始图像
|
| 31 |
-
base_prompt = get_clip_interrogator_prompt(control_image)
|
| 32 |
-
if base_prompt.startswith("Error:"):
|
| 33 |
-
return base_prompt
|
| 34 |
-
combined_prompt = base_prompt + " " + additional_prompt
|
| 35 |
try:
|
| 36 |
-
|
|
|
|
| 37 |
except Exception as e:
|
| 38 |
return f"Error in Step 1: {str(e)}"
|
| 39 |
|
| 40 |
-
# 第二步:使用第一张参考图像进行 reference_adain+attn 生成
|
| 41 |
try:
|
| 42 |
-
|
|
|
|
| 43 |
except Exception as e:
|
| 44 |
return f"Error in Step 2: {str(e)}"
|
| 45 |
|
| 46 |
-
# 第三步:使用第二张参考图像进行 reference_adain+attn 生成
|
| 47 |
try:
|
| 48 |
-
|
|
|
|
| 49 |
return final_image
|
| 50 |
except Exception as e:
|
| 51 |
return f"Error in Step 3: {str(e)}"
|
|
@@ -61,7 +42,7 @@ interface = gr.Interface(
|
|
| 61 |
],
|
| 62 |
outputs=gr.components.Image(type="pil", label="Generated Image"),
|
| 63 |
title="Stable Diffusion with Multi-Step ControlNet",
|
| 64 |
-
description="Generate images using a multi-step process with Stable Diffusion
|
| 65 |
)
|
| 66 |
|
| 67 |
if __name__ == "__main__":
|
|
|
|
| 2 |
import torch
|
| 3 |
from io import BytesIO
|
| 4 |
from PIL import Image
|
|
|
|
| 5 |
from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
|
| 6 |
|
| 7 |
+
# 加载模型和ControlNet
|
| 8 |
model_id = "CompVis/stable-diffusion-v1-4"
|
| 9 |
controlnet_id = "lllyasviel/sd-controlnet-canny"
|
| 10 |
|
| 11 |
pipe = StableDiffusionControlNetPipeline.from_pretrained(model_id, controlnet=ControlNetModel.from_pretrained(controlnet_id))
|
| 12 |
pipe = pipe.to("cuda" if torch.cuda.is_available() else "cpu")
|
| 13 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
def generate_image(control_image, additional_prompt, reference_image1, reference_image2):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
try:
|
| 16 |
+
# 第一步:使用 canny 模型生成初始图像
|
| 17 |
+
initial_image = pipe(prompt=additional_prompt, control_image=control_image).images[0]
|
| 18 |
except Exception as e:
|
| 19 |
return f"Error in Step 1: {str(e)}"
|
| 20 |
|
|
|
|
| 21 |
try:
|
| 22 |
+
# 第二步:使用第一张参考图像进行 reference_adain+attn 生成
|
| 23 |
+
step2_image = pipe(prompt=additional_prompt, control_image=reference_image1).images[0]
|
| 24 |
except Exception as e:
|
| 25 |
return f"Error in Step 2: {str(e)}"
|
| 26 |
|
|
|
|
| 27 |
try:
|
| 28 |
+
# 第三步:使用第二张参考图像进行 reference_adain+attn 生成
|
| 29 |
+
final_image = pipe(prompt=additional_prompt, control_image=reference_image2).images[0]
|
| 30 |
return final_image
|
| 31 |
except Exception as e:
|
| 32 |
return f"Error in Step 3: {str(e)}"
|
|
|
|
| 42 |
],
|
| 43 |
outputs=gr.components.Image(type="pil", label="Generated Image"),
|
| 44 |
title="Stable Diffusion with Multi-Step ControlNet",
|
| 45 |
+
description="Generate images using a multi-step process with Stable Diffusion and ControlNet."
|
| 46 |
)
|
| 47 |
|
| 48 |
if __name__ == "__main__":
|