File size: 4,802 Bytes
0a37e19 bf725bf c8afad4 bf725bf 0a37e19 bf725bf 0a37e19 bf725bf 0a37e19 bf725bf e7a0e0b 0a37e19 9ae0ffb 0a37e19 e7a0e0b 627e110 bf725bf 75c790f 0a37e19 627e110 0a37e19 e5949be 627e110 0a37e19 bf725bf 0a37e19 bf725bf 582a46a 0a37e19 a40e6d3 0a37e19 c8afad4 bf725bf 0a37e19 bf725bf e5949be 0a37e19 bf725bf 627e110 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | from typing import Any
import requests
from PIL import Image
from io import BytesIO
import torch
import json
from diffusers import StableDiffusionXLControlNetPipeline, ControlNetModel, AutoencoderKL, DDPMParallelScheduler
import numpy as np
import cv2
import base64
# Set device
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
if device.type != 'cuda':
raise ValueError("The model requires a GPU for processing.")
dtype = torch.float16
class EndpointHandler():
def __init__(self, path=""):
self.controlnet = ControlNetModel.from_pretrained("diffusers/controlnet-canny-sdxl-1.0", torch_dtype=dtype).to(device)
self.vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=dtype).to(device)
# Remove the .to(device) if DDPMParallelScheduler does not need to be moved to a device
self.scheduler = DDPMParallelScheduler.from_pretrained("nyxia/mysterious-xl", subfolder="scheduler")
self.stable_diffusion_id = "nyxia/mysterious-xl"
self.pipe = StableDiffusionXLControlNetPipeline.from_pretrained(self.stable_diffusion_id,
controlnet=self.controlnet,
torch_dtype=dtype,
scheduler=self.scheduler,
vae=self.vae).to(device)
self.generator = torch.Generator(device="cpu").manual_seed(3)
def __call__(self, data: Any):
prompt = data.get("inputs")
base64_image = data.get("image")
if not prompt or not isinstance(prompt, str) or not base64_image or not isinstance(base64_image, str):
return {"error": "Invalid input. Please provide a prompt and a base64 encoded image."}
num_inference_steps = data.get("num_inference_steps", 50)
guidance_scale = data.get("guidance_scale", 8)
negative_prompt = "ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of frame, extra limbs, disfigured, deformed, body out of frame, bad anatomy, watermark, signature, cut off, low contrast, underexposed, overexposed, bad art, beginner, amateur, distorted face, low quality, photorealistic"
num_images_per_prompt = data.get("num_images_per_prompt", 1)
controlnet_conditioning_scale = data.get("controlnet_conditioning_scale", 0.5)
try:
image = self.decode_base64_image(base64_image)
image = self.resize_image(image, max_size=1024)
except Exception as e:
return json.dumps({"error": str(e)})
width, height = image.size
control_image = self.get_canny_image(image)
out_images = self.pipe(prompt=prompt,
negative_prompt=negative_prompt,
image=control_image,
num_inference_steps=num_inference_steps,
guidance_scale=guidance_scale,
num_images_per_prompt=num_images_per_prompt,
height=height,
width=width,
control_guidance_end=0.7,
controlnet_conditioning_scale=controlnet_conditioning_scale,
generator=self.generator).images
final_images = [self.encode_image(img) for img in out_images]
return json.dumps(final_images)
def decode_base64_image(self, image_string: str) -> Image:
try:
padding_needed = len(image_string) % 4
if padding_needed: # padding_needed should be 0, 1, 2, or 3
image_string += "=" * (4 - padding_needed) # Add the padding if needed
base64_image = base64.b64decode(image_string)
buffer = BytesIO(base64_image)
return Image.open(buffer)
except Exception as e:
raise ValueError(f"Unable to decode the base64 image: {e}")
def encode_image(self, image: Image) -> str:
img_byte_arr = BytesIO()
image.save(img_byte_arr, format='PNG')
return base64.b64encode(img_byte_arr.getvalue()).decode()
def get_canny_image(self, base_image: Image) -> Image:
image = np.array(base_image)
image = cv2.Canny(image, 125, 150)
image = np.stack([image] * 3, axis=2)
return Image.fromarray(image)
def resize_image(self, image: Image, max_size: int) -> Image:
scale = max_size / min(image.size)
new_size = tuple([int(x * scale) for x in image.size])
image = image.resize(new_size, Image.LANCZOS)
return image
|