InstantID / app_lcm_cpu.py
Annihilator3000's picture
Upload app_lcm_cpu.py
be503d9 verified
Raw History Blame Contribute Delete
21 kB
## -*- coding: utf-8 -*-
import sys
sys.path.append('./')
import os
import cv2
import math
import torch
import random
import numpy as np
import argparse
import PIL
from PIL import Image
import diffusers
from diffusers import DiffusionPipeline, AutoencoderTiny
from diffusers.utils import load_image
from diffusers.models import ControlNetModel
from diffusers import LCMScheduler, EulerAncestralDiscreteScheduler
from huggingface_hub import hf_hub_download
import insightface
from insightface.app import FaceAnalysis
from style_template import styles
from pipeline_stable_diffusion_xl_instantid import StableDiffusionXLInstantIDPipeline
from model_util import load_models_xl, get_torch_device, torch_gc
import gradio as gr
lcm_lora_path = "./checkpoints/lcm-lora-sdxl-turbo.safetensors"
vega_lora_path = "./checkpoints/pytorch_lora_weights_vega.safetensors"
upscalers_path = [file for file in os.listdir('upscalers') if file.endswith('.pth')]
from RestoreFormer import RestoreFormer
from realesrgan import RealESRGANer
from basicsr.archs.srvgg_arch import SRVGGNetCompact
# Глобальная инициализация моделей RestoreFormer и RealESRGAN
restoreformer_model_path = './checkpoints/RestoreFormer.ckpt'
restoreformerplusplus_model_path = './checkpoints/RestoreFormer_plus_plus.ckpt'
realesr_model_path = './checkpoints/realesr-general-x4v3.pth'
# global variable
device = "cpu"
MAX_SEED = np.iinfo(np.int32).max
def gpu_accel(gpu="on"):
if gpu == "on":
device = "cuda"
else:
device = "cpu"
return device
dtype = torch.float32
STYLE_NAMES = list(styles.keys())
DEFAULT_STYLE_NAME = "sai-comic book"
# Load face encoder
app = FaceAnalysis(name='antelopev2', root='./', providers=['CPUExecutionProvider'])
app.prepare(ctx_id=0, det_size=(640, 640))
# Path to InstantID models
face_adapter = f'./checkpoints/ip-adapter.bin'
controlnet_path = f'./checkpoints/ControlNetModel'
# Load pipeline
controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=dtype)
def main(pretrained_model_name_or_path="wangqixun/YamerMIX_v8", gpu="on"):
if pretrained_model_name_or_path.endswith(
".ckpt"
) or pretrained_model_name_or_path.endswith(".safetensors"):
scheduler_kwargs = hf_hub_download(
repo_id="wangqixun/YamerMIX_v8",
subfolder="scheduler",
filename="scheduler_config.json",
)
(tokenizers, text_encoders, unet, _, vae) = load_models_xl(
pretrained_model_name_or_path=pretrained_model_name_or_path,
scheduler_name=None,
weight_dtype=dtype,
)
scheduler = diffusers.LCMScheduler.from_config(scheduler_kwargs)
pipe = StableDiffusionXLInstantIDPipeline(
vae=vae,
text_encoder=text_encoders[0],
text_encoder_2=text_encoders[1],
tokenizer=tokenizers[0],
tokenizer_2=tokenizers[1],
unet=unet,
torch_dtype=dtype,
scheduler=scheduler,
controlnet=controlnet,
).to(device)
else:
pipe = StableDiffusionXLInstantIDPipeline.from_pretrained(
pretrained_model_name_or_path,
controlnet=controlnet,
torch_dtype=dtype,
use_safetensors=True,
#variant="fp16",
safety_checker=None,
feature_extractor=None,
).to(device)
if pretrained_model_name_or_path == "segmind/Segmind-Vega":
pipe.load_lora_weights(vega_lora_path)
else:
pipe.load_lora_weights(lcm_lora_path)
pipe.fuse_lora()
pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config)
pipe.load_ip_adapter_instantid(face_adapter)
pipe.image_proj_model.to(device)
pipe.unet.to(device)
def randomize_seed_fn(seed: int, randomize_seed: bool) -> int:
if randomize_seed:
seed = random.randint(0, MAX_SEED)
return seed
def swap_to_gallery(images):
return gr.update(value=images, visible=True), gr.update(visible=True), gr.update(visible=False)
def upload_example_to_gallery(images, prompt, style, negative_prompt):
return gr.update(value=images, visible=True), gr.update(visible=True), gr.update(visible=False)
def remove_back_to_files():
return gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)
def remove_tips():
return gr.update(visible=False)
def convert_from_cv2_to_image(img: np.ndarray) -> Image:
return Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
def convert_from_image_to_cv2(img: Image) -> np.ndarray:
return cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
def draw_kps(image_pil, kps, color_list=[(255,0,0), (0,255,0), (0,0,255), (255,255,0), (255,0,255)]):
stickwidth = 4
limbSeq = np.array([[0, 2], [1, 2], [3, 2], [4, 2]])
kps = np.array(kps)
w, h = image_pil.size
out_img = np.zeros([h, w, 3])
for i in range(len(limbSeq)):
index = limbSeq[i]
color = color_list[index[0]]
x = kps[index][:, 0]
y = kps[index][:, 1]
length = ((x[0] - x[1]) ** 2 + (y[0] - y[1]) ** 2) ** 0.5
angle = math.degrees(math.atan2(y[0] - y[1], x[0] - x[1]))
polygon = cv2.ellipse2Poly((int(np.mean(x)), int(np.mean(y))), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
out_img = cv2.fillConvexPoly(out_img.copy(), polygon, color)
out_img = (out_img * 0.6).astype(np.uint8)
for idx_kp, kp in enumerate(kps):
color = color_list[idx_kp]
x, y = kp
out_img = cv2.circle(out_img.copy(), (int(x), int(y)), 10, color, -1)
out_img_pil = Image.fromarray(out_img.astype(np.uint8))
return out_img_pil
def resize_img(input_image, max_side=1280, min_side=1024, size=None,
pad_to_max_side=False, mode=PIL.Image.LANCZOS, base_pixel_number=64):
w, h = input_image.size
if size is not None:
w_resize_new, h_resize_new = size
else:
ratio = min_side / min(h, w)
w, h = round(ratio*w), round(ratio*h)
ratio = max_side / max(h, w)
input_image = input_image.resize([round(ratio*w), round(ratio*h)], mode)
w_resize_new = (round(ratio * w) // base_pixel_number) * base_pixel_number
h_resize_new = (round(ratio * h) // base_pixel_number) * base_pixel_number
input_image = input_image.resize([w_resize_new, h_resize_new], mode)
if pad_to_max_side:
res = np.ones([max_side, max_side, 3], dtype=np.uint8) * 255
offset_x = (max_side - w_resize_new) // 2
offset_y = (max_side - h_resize_new) // 2
res[offset_y:offset_y+h_resize_new, offset_x:offset_x+w_resize_new] = np.array(input_image)
input_image = Image.fromarray(res)
return input_image
def apply_style(style_name: str, positive: str, negative: str = "") -> tuple[str, str]:
p, n = styles.get(style_name, styles[DEFAULT_STYLE_NAME])
return p.replace("{prompt}", positive), n + ' ' + negative
def check_input_image(face_image):
if face_image is None:
raise gr.Error("Не могу найти лицо на изображении! Не слишком ли оно крупное или маленькое? Попробуйте другое фото")
def generate_image(
face_image_path,
pose_image_path,
prompt,
negative_prompt,
style_name,
enhance_face_region,
num_steps,
identitynet_strength_ratio,
adapter_strength_ratio,
guidance_scale,
seed,
upscale,
upscaler_model,
restoreformer,
taesdxl,
attention_slicing,
progress=gr.Progress(track_tqdm=True),
):
if face_image_path is None:
raise gr.Error(f"Не могу найти лицо на изображении! Не слишком ли оно крупное или маленькое? Попробуйте другое фото")
if prompt is None:
prompt = "a person"
# apply the style template
prompt, negative_prompt = apply_style(style_name, prompt, negative_prompt)
face_image = load_image(face_image_path)
face_image = resize_img(face_image)
face_image_cv2 = convert_from_image_to_cv2(face_image)
height, width, _ = face_image_cv2.shape
# Extract face features
face_info = app.get(face_image_cv2)
if len(face_info) == 0:
raise gr.Error(f"Не могу найти лицо на изображении! Не слишком ли оно крупное или маленькое? Попробуйте другое фото")
face_info = sorted(face_info, key=lambda x:(x['bbox'][2]-x['bbox'][0])*x['bbox'][3]-x['bbox'][1])[-1] # only use the maximum face
face_emb = face_info['embedding']
face_kps = draw_kps(convert_from_cv2_to_image(face_image_cv2), face_info['kps'])
if pose_image_path is not None:
pose_image = load_image(pose_image_path)
pose_image = resize_img(pose_image)
pose_image_cv2 = convert_from_image_to_cv2(pose_image)
face_info = app.get(pose_image_cv2)
if len(face_info) == 0:
raise gr.Error(f"Не могу найти лицо на изображении! Не слишком ли оно крупное или маленькое? Попробуйте другое фото")
face_info = face_info[-1]
face_kps = draw_kps(pose_image, face_info['kps'])
width, height = face_kps.size
if enhance_face_region:
control_mask = np.zeros([height, width, 3])
x1, y1, x2, y2 = face_info["bbox"]
x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
control_mask[y1:y2, x1:x2] = 255
control_mask = Image.fromarray(control_mask.astype(np.uint8))
else:
control_mask = None
generator = torch.Generator(device=device).manual_seed(seed)
print("Start inference...")
print(f"[Debug] Prompt: {prompt}, \n[Debug] Neg Prompt: {negative_prompt}")
pipe.set_ip_adapter_scale(adapter_strength_ratio)
if taesdxl:
pipe.vae = AutoencoderTiny.from_pretrained("madebyollin/taesdxl", torch_dtype=torch.float32)
pipe.vae = pipe.vae.to(device)
pipe.enable_vae_tiling()
if attention_slicing:
pipe.enable_attention_slicing(slice_size="auto")
images = pipe(
prompt=prompt,
negative_prompt=negative_prompt,
image_embeds=face_emb,
image=face_kps,
controlnet_conditioning_scale=float(identitynet_strength_ratio),
num_inference_steps=num_steps,
guidance_scale=guidance_scale,
height=height,
width=width,
generator=generator
).images
if restoreformer:
model = SRVGGNetCompact(num_in_ch=3, num_out_ch=3, num_feat=64, num_conv=32, upscale=4, act_type='prelu')
half = True if torch.cuda.is_available() else False
upsampler = RealESRGANer(scale=upscale, model_path=realesr_model_path, model=model, tile=0, tile_pad=10, pre_pad=0)
face_enhancer = RestoreFormer(
model_path=restoreformerplusplus_model_path, # Используйте "+" версию для лучшего качества
upscale=upscale,
bg_upsampler=upsampler,
arch='RestoreFormer++', # Установите в 'cuda' или 'cpu' в зависимости от вашей конфигурации,
device="cpu"
)
# Взять первое изображение из списка и преобразовать его из PIL в формат, который ожидает RestoreFormer
generated_image = images[0]
# Преобразуйте изображение PIL в формат данных OpenCV
cv2_image = np.array(generated_image)
cv2_image = cv2_image[:, :, ::-1].copy() # RGB to BGR
#weight = 100
_, _, output = face_enhancer.enhance(cv2_image, has_aligned=False, only_center_face=False, paste_back=True)
# Обратно преобразовать изображение из OpenCV в PIL для Gradio
images = Image.fromarray(output[:, :, ::-1]) # BGR to RGB
return images, gr.update(visible=True)
else:
return images[0], gr.update(visible=True)
### Description
title = r"""
<h1 align="center">InstantID: Zero-shot Identity-Preserving Generation in Seconds / Portable by Neurogen</h1>
"""
description = r"""
<h2 align="center"><b></b><a href='https://t.me/neurogen_news' target='_blank'><b>Telegram</b></a><br></h2>
Как использовать:<br>
1. Загрузите изображение человека. Если изображений несколько, мы определим только самое близкое лицо. Убедитесь, что лицо не слишком маленькое, не сликшом крупное и не сильно размыто.
2. (По желанию) загрузите другое изображение человека в качестве примерной позы. Если не загружено, мы будем использовать первое изображение человека для извлечения ориентиров.
3. Введите текстовый запрос, как это делается в обычных txt2img моделях.
4. Нажмите кнопку <b>Подвердить</b>, чтобы начать обработку.
"""
article = r"""
---"""
tips = r"""
### Рекомендации по использованию InstantID
1. Если вы не удовлетворены сходством, попробуйте увеличить силу влияния "IdentityNet" и "Image Adapter".
2. Если вы чувствуете, что насыщенность слишком высока, сначала уменьшите силу адаптера. Если она все еще слишком высока, уменьшите силу IdentityNet.
3. Если вы обнаружили, что управление текстом не соответствует ожиданиям, уменьшите силу адаптера.
4. Если вы обнаружили, что реалистичный стиль недостаточно хорош, используйте более реалистичную базовую модель.
"""
css = """
.gradio-container {width: 85% !important}
"""
with gr.Blocks(css=css) as demo:
# description
gr.Markdown(title)
gr.Markdown(description)
with gr.Row():
with gr.Column():
# upload face image
face_file = gr.Image(label="Загрузите фото вашего лица", type="filepath")
# optional: upload a reference pose image
pose_file = gr.Image(label="Загрузите фото, откуда возьмем позу (опционально)", type="filepath")
# prompt
prompt = gr.Textbox(
label="Подсказка (Промпт)",
info="",
placeholder="A photo of a person",
value="A photo of a person",
)
submit = gr.Button("Подвердить", variant="primary")
style = gr.Dropdown(label="Стили", choices=STYLE_NAMES, value=DEFAULT_STYLE_NAME)
# strength
identitynet_strength_ratio = gr.Slider(
label="Сила IdentityNet (для схожести)",
minimum=0,
maximum=1.5,
step=0.05,
value=0.80,
)
adapter_strength_ratio = gr.Slider(
label="Сила Image adapter (для деталей)",
minimum=0,
maximum=1.5,
step=0.05,
value=0.80,
)
with gr.Accordion(open=False, label="Продвинутые опции"):
negative_prompt = gr.Textbox(
label="Negative Prompt",
placeholder="low quality",
value="",
)
num_steps = gr.Slider(
label="Number of sample steps",
minimum=2,
maximum=20,
step=1,
value=4,
)
guidance_scale = gr.Slider(
label="Guidance scale",
minimum=0.0,
maximum=2.0,
step=0.1,
value=0.0,
)
seed = gr.Slider(
label="Seed",
minimum=0,
maximum=MAX_SEED,
step=1,
value=42,
)
upscaler_model= gr.Dropdown(label="Выберите модель апскейлера / энхейсера", choices=upscalers_path, value=upscalers_path[0], visible=False)
randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
enhance_face_region = gr.Checkbox(label="Enhance non-face region", value=True)
restoreformer = gr.Checkbox(label="Enable RestoreFormer++", value=False)
upscale = gr.Slider(
label="Select Upscale multiplier (if RestoreFormer Enabled)",
info="x1 - Upscale off, x4 - Upscale x4",
minimum=1,
maximum=4,
step=1,
value=1,
)
taesdxl = gr.Checkbox(label="Enable TinyVAE", value=True)
attention_slicing = gr.Checkbox(label="Enable Attention Slicing", value=False)
with gr.Column():
output_image = gr.Image(label="Сгенерированное изображение", type="numpy", )
usage_tips = gr.Markdown(label="Usage tips of InstantID", value=tips, visible=False)
submit.click(
fn=remove_tips,
outputs=usage_tips,
queue=False,
api_name=False,
).then(
fn=randomize_seed_fn,
inputs=[seed, randomize_seed],
outputs=seed,
queue=False,
api_name=False,
).then(
fn=check_input_image,
inputs=face_file,
queue=False,
api_name=False,
).success(
fn=generate_image,
inputs=[
face_file,
pose_file,
prompt,
negative_prompt,
style,
enhance_face_region,
num_steps,
identitynet_strength_ratio,
adapter_strength_ratio,
guidance_scale,
seed,
upscale,
upscaler_model,
restoreformer,
taesdxl,
attention_slicing
],
outputs=[output_image, usage_tips],
)
gr.Markdown(article)
demo.launch(inbrowser=True, share=True,)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--pretrained_model_name_or_path", type=str, default="wangqixun/YamerMIX_v8")
parser.add_argument("--gpu", type=str, default="on")
args = parser.parse_args()
main(args.pretrained_model_name_or_path,args.gpu)