sdxl_controlnet_hf / handler.py
sakhan2's picture
Update handler.py
e7a0e0b
Raw History Blame Contribute Delete
4.8 kB
from typing import Any
import requests
from PIL import Image
from io import BytesIO
import torch
import json
from diffusers import StableDiffusionXLControlNetPipeline, ControlNetModel, AutoencoderKL, DDPMParallelScheduler
import numpy as np
import cv2
import base64
# Set device
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
if device.type != 'cuda':
raise ValueError("The model requires a GPU for processing.")
dtype = torch.float16
class EndpointHandler():
def __init__(self, path=""):
self.controlnet = ControlNetModel.from_pretrained("diffusers/controlnet-canny-sdxl-1.0", torch_dtype=dtype).to(device)
self.vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=dtype).to(device)
# Remove the .to(device) if DDPMParallelScheduler does not need to be moved to a device
self.scheduler = DDPMParallelScheduler.from_pretrained("nyxia/mysterious-xl", subfolder="scheduler")
self.stable_diffusion_id = "nyxia/mysterious-xl"
self.pipe = StableDiffusionXLControlNetPipeline.from_pretrained(self.stable_diffusion_id,
controlnet=self.controlnet,
torch_dtype=dtype,
scheduler=self.scheduler,
vae=self.vae).to(device)
self.generator = torch.Generator(device="cpu").manual_seed(3)
def __call__(self, data: Any):
prompt = data.get("inputs")
base64_image = data.get("image")
if not prompt or not isinstance(prompt, str) or not base64_image or not isinstance(base64_image, str):
return {"error": "Invalid input. Please provide a prompt and a base64 encoded image."}
num_inference_steps = data.get("num_inference_steps", 50)
guidance_scale = data.get("guidance_scale", 8)
negative_prompt = "ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of frame, extra limbs, disfigured, deformed, body out of frame, bad anatomy, watermark, signature, cut off, low contrast, underexposed, overexposed, bad art, beginner, amateur, distorted face, low quality, photorealistic"
num_images_per_prompt = data.get("num_images_per_prompt", 1)
controlnet_conditioning_scale = data.get("controlnet_conditioning_scale", 0.5)
try:
image = self.decode_base64_image(base64_image)
image = self.resize_image(image, max_size=1024)
except Exception as e:
return json.dumps({"error": str(e)})
width, height = image.size
control_image = self.get_canny_image(image)
out_images = self.pipe(prompt=prompt,
negative_prompt=negative_prompt,
image=control_image,
num_inference_steps=num_inference_steps,
guidance_scale=guidance_scale,
num_images_per_prompt=num_images_per_prompt,
height=height,
width=width,
control_guidance_end=0.7,
controlnet_conditioning_scale=controlnet_conditioning_scale,
generator=self.generator).images
final_images = [self.encode_image(img) for img in out_images]
return json.dumps(final_images)
def decode_base64_image(self, image_string: str) -> Image:
try:
padding_needed = len(image_string) % 4
if padding_needed: # padding_needed should be 0, 1, 2, or 3
image_string += "=" * (4 - padding_needed) # Add the padding if needed
base64_image = base64.b64decode(image_string)
buffer = BytesIO(base64_image)
return Image.open(buffer)
except Exception as e:
raise ValueError(f"Unable to decode the base64 image: {e}")
def encode_image(self, image: Image) -> str:
img_byte_arr = BytesIO()
image.save(img_byte_arr, format='PNG')
return base64.b64encode(img_byte_arr.getvalue()).decode()
def get_canny_image(self, base_image: Image) -> Image:
image = np.array(base_image)
image = cv2.Canny(image, 125, 150)
image = np.stack([image] * 3, axis=2)
return Image.fromarray(image)
def resize_image(self, image: Image, max_size: int) -> Image:
scale = max_size / min(image.size)
new_size = tuple([int(x * scale) for x in image.size])
image = image.resize(new_size, Image.LANCZOS)
return image