File size: 4,802 Bytes
0a37e19
 
bf725bf
 
 
c8afad4
bf725bf
 
 
0a37e19
bf725bf
0a37e19
bf725bf
 
0a37e19
bf725bf
 
 
 
 
e7a0e0b
0a37e19
9ae0ffb
 
0a37e19
 
 
e7a0e0b
 
627e110
bf725bf
 
75c790f
0a37e19
 
627e110
0a37e19
 
 
e5949be
 
 
627e110
 
0a37e19
 
 
 
 
 
bf725bf
0a37e19
bf725bf
582a46a
0a37e19
 
 
 
 
 
 
 
a40e6d3
0a37e19
 
 
 
c8afad4
bf725bf
0a37e19
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bf725bf
e5949be
0a37e19
 
 
 
 
 
 
bf725bf
627e110
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
from typing import Any
import requests
from PIL import Image
from io import BytesIO
import torch
import json
from diffusers import StableDiffusionXLControlNetPipeline, ControlNetModel, AutoencoderKL, DDPMParallelScheduler
import numpy as np
import cv2
import base64

# Set device
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
if device.type != 'cuda':
    raise ValueError("The model requires a GPU for processing.")
dtype = torch.float16

class EndpointHandler():
    def __init__(self, path=""):
        self.controlnet = ControlNetModel.from_pretrained("diffusers/controlnet-canny-sdxl-1.0", torch_dtype=dtype).to(device)
        self.vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=dtype).to(device)
        # Remove the .to(device) if DDPMParallelScheduler does not need to be moved to a device
        self.scheduler = DDPMParallelScheduler.from_pretrained("nyxia/mysterious-xl", subfolder="scheduler")
        self.stable_diffusion_id = "nyxia/mysterious-xl"
        self.pipe = StableDiffusionXLControlNetPipeline.from_pretrained(self.stable_diffusion_id,
                                                                        controlnet=self.controlnet,
                                                                        torch_dtype=dtype,
                                                                        scheduler=self.scheduler,
                                                                        vae=self.vae).to(device)
        self.generator = torch.Generator(device="cpu").manual_seed(3)


    def __call__(self, data: Any):
        prompt = data.get("inputs")
        base64_image = data.get("image")

        if not prompt or not isinstance(prompt, str) or not base64_image or not isinstance(base64_image, str):
            return {"error": "Invalid input. Please provide a prompt and a base64 encoded image."}

        num_inference_steps = data.get("num_inference_steps", 50)
        guidance_scale = data.get("guidance_scale", 8)
        negative_prompt = "ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of frame, extra limbs, disfigured, deformed, body out of frame, bad anatomy, watermark, signature, cut off, low contrast, underexposed, overexposed, bad art, beginner, amateur, distorted face, low quality, photorealistic"
        num_images_per_prompt = data.get("num_images_per_prompt", 1)
        controlnet_conditioning_scale = data.get("controlnet_conditioning_scale", 0.5)

        try:
            image = self.decode_base64_image(base64_image)
            image = self.resize_image(image, max_size=1024)
        except Exception as e:
            return json.dumps({"error": str(e)})
        
        width, height = image.size
        control_image = self.get_canny_image(image)

        out_images = self.pipe(prompt=prompt,
                               negative_prompt=negative_prompt,
                               image=control_image,
                               num_inference_steps=num_inference_steps,
                               guidance_scale=guidance_scale,
                               num_images_per_prompt=num_images_per_prompt,
                               height=height,
                               width=width,
                               control_guidance_end=0.7,
                               controlnet_conditioning_scale=controlnet_conditioning_scale,
                               generator=self.generator).images

        final_images = [self.encode_image(img) for img in out_images]
        return json.dumps(final_images)

    def decode_base64_image(self, image_string: str) -> Image:
        try:
            padding_needed = len(image_string) % 4
            if padding_needed:  # padding_needed should be 0, 1, 2, or 3
                image_string += "=" * (4 - padding_needed)  # Add the padding if needed

            base64_image = base64.b64decode(image_string)
            buffer = BytesIO(base64_image)
            return Image.open(buffer)
        except Exception as e:
            raise ValueError(f"Unable to decode the base64 image: {e}")

    def encode_image(self, image: Image) -> str:
        img_byte_arr = BytesIO()
        image.save(img_byte_arr, format='PNG')
        return base64.b64encode(img_byte_arr.getvalue()).decode()

    def get_canny_image(self, base_image: Image) -> Image:
        image = np.array(base_image)
        image = cv2.Canny(image, 125, 150)
        image = np.stack([image] * 3, axis=2)
        return Image.fromarray(image)

    def resize_image(self, image: Image, max_size: int) -> Image:
        scale = max_size / min(image.size)
        new_size = tuple([int(x * scale) for x in image.size])
        image = image.resize(new_size, Image.LANCZOS)

        return image