## -*- coding: utf-8 -*- import sys sys.path.append('./') import os import cv2 import math import torch import random import numpy as np import argparse import PIL from PIL import Image import diffusers from diffusers import DiffusionPipeline, AutoencoderTiny from diffusers.utils import load_image from diffusers.models import ControlNetModel from diffusers import LCMScheduler, EulerAncestralDiscreteScheduler from huggingface_hub import hf_hub_download import insightface from insightface.app import FaceAnalysis from style_template import styles from pipeline_stable_diffusion_xl_instantid import StableDiffusionXLInstantIDPipeline from model_util import load_models_xl, get_torch_device, torch_gc import gradio as gr lcm_lora_path = "./checkpoints/lcm-lora-sdxl-turbo.safetensors" vega_lora_path = "./checkpoints/pytorch_lora_weights_vega.safetensors" upscalers_path = [file for file in os.listdir('upscalers') if file.endswith('.pth')] from RestoreFormer import RestoreFormer from realesrgan import RealESRGANer from basicsr.archs.srvgg_arch import SRVGGNetCompact # Глобальная инициализация моделей RestoreFormer и RealESRGAN restoreformer_model_path = './checkpoints/RestoreFormer.ckpt' restoreformerplusplus_model_path = './checkpoints/RestoreFormer_plus_plus.ckpt' realesr_model_path = './checkpoints/realesr-general-x4v3.pth' # global variable device = "cpu" MAX_SEED = np.iinfo(np.int32).max def gpu_accel(gpu="on"): if gpu == "on": device = "cuda" else: device = "cpu" return device dtype = torch.float32 STYLE_NAMES = list(styles.keys()) DEFAULT_STYLE_NAME = "sai-comic book" # Load face encoder app = FaceAnalysis(name='antelopev2', root='./', providers=['CPUExecutionProvider']) app.prepare(ctx_id=0, det_size=(640, 640)) # Path to InstantID models face_adapter = f'./checkpoints/ip-adapter.bin' controlnet_path = f'./checkpoints/ControlNetModel' # Load pipeline controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=dtype) def main(pretrained_model_name_or_path="wangqixun/YamerMIX_v8", gpu="on"): if pretrained_model_name_or_path.endswith( ".ckpt" ) or pretrained_model_name_or_path.endswith(".safetensors"): scheduler_kwargs = hf_hub_download( repo_id="wangqixun/YamerMIX_v8", subfolder="scheduler", filename="scheduler_config.json", ) (tokenizers, text_encoders, unet, _, vae) = load_models_xl( pretrained_model_name_or_path=pretrained_model_name_or_path, scheduler_name=None, weight_dtype=dtype, ) scheduler = diffusers.LCMScheduler.from_config(scheduler_kwargs) pipe = StableDiffusionXLInstantIDPipeline( vae=vae, text_encoder=text_encoders[0], text_encoder_2=text_encoders[1], tokenizer=tokenizers[0], tokenizer_2=tokenizers[1], unet=unet, torch_dtype=dtype, scheduler=scheduler, controlnet=controlnet, ).to(device) else: pipe = StableDiffusionXLInstantIDPipeline.from_pretrained( pretrained_model_name_or_path, controlnet=controlnet, torch_dtype=dtype, use_safetensors=True, #variant="fp16", safety_checker=None, feature_extractor=None, ).to(device) if pretrained_model_name_or_path == "segmind/Segmind-Vega": pipe.load_lora_weights(vega_lora_path) else: pipe.load_lora_weights(lcm_lora_path) pipe.fuse_lora() pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) pipe.load_ip_adapter_instantid(face_adapter) pipe.image_proj_model.to(device) pipe.unet.to(device) def randomize_seed_fn(seed: int, randomize_seed: bool) -> int: if randomize_seed: seed = random.randint(0, MAX_SEED) return seed def swap_to_gallery(images): return gr.update(value=images, visible=True), gr.update(visible=True), gr.update(visible=False) def upload_example_to_gallery(images, prompt, style, negative_prompt): return gr.update(value=images, visible=True), gr.update(visible=True), gr.update(visible=False) def remove_back_to_files(): return gr.update(visible=False), gr.update(visible=False), gr.update(visible=True) def remove_tips(): return gr.update(visible=False) def convert_from_cv2_to_image(img: np.ndarray) -> Image: return Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) def convert_from_image_to_cv2(img: Image) -> np.ndarray: return cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR) def draw_kps(image_pil, kps, color_list=[(255,0,0), (0,255,0), (0,0,255), (255,255,0), (255,0,255)]): stickwidth = 4 limbSeq = np.array([[0, 2], [1, 2], [3, 2], [4, 2]]) kps = np.array(kps) w, h = image_pil.size out_img = np.zeros([h, w, 3]) for i in range(len(limbSeq)): index = limbSeq[i] color = color_list[index[0]] x = kps[index][:, 0] y = kps[index][:, 1] length = ((x[0] - x[1]) ** 2 + (y[0] - y[1]) ** 2) ** 0.5 angle = math.degrees(math.atan2(y[0] - y[1], x[0] - x[1])) polygon = cv2.ellipse2Poly((int(np.mean(x)), int(np.mean(y))), (int(length / 2), stickwidth), int(angle), 0, 360, 1) out_img = cv2.fillConvexPoly(out_img.copy(), polygon, color) out_img = (out_img * 0.6).astype(np.uint8) for idx_kp, kp in enumerate(kps): color = color_list[idx_kp] x, y = kp out_img = cv2.circle(out_img.copy(), (int(x), int(y)), 10, color, -1) out_img_pil = Image.fromarray(out_img.astype(np.uint8)) return out_img_pil def resize_img(input_image, max_side=1280, min_side=1024, size=None, pad_to_max_side=False, mode=PIL.Image.LANCZOS, base_pixel_number=64): w, h = input_image.size if size is not None: w_resize_new, h_resize_new = size else: ratio = min_side / min(h, w) w, h = round(ratio*w), round(ratio*h) ratio = max_side / max(h, w) input_image = input_image.resize([round(ratio*w), round(ratio*h)], mode) w_resize_new = (round(ratio * w) // base_pixel_number) * base_pixel_number h_resize_new = (round(ratio * h) // base_pixel_number) * base_pixel_number input_image = input_image.resize([w_resize_new, h_resize_new], mode) if pad_to_max_side: res = np.ones([max_side, max_side, 3], dtype=np.uint8) * 255 offset_x = (max_side - w_resize_new) // 2 offset_y = (max_side - h_resize_new) // 2 res[offset_y:offset_y+h_resize_new, offset_x:offset_x+w_resize_new] = np.array(input_image) input_image = Image.fromarray(res) return input_image def apply_style(style_name: str, positive: str, negative: str = "") -> tuple[str, str]: p, n = styles.get(style_name, styles[DEFAULT_STYLE_NAME]) return p.replace("{prompt}", positive), n + ' ' + negative def check_input_image(face_image): if face_image is None: raise gr.Error("Не могу найти лицо на изображении! Не слишком ли оно крупное или маленькое? Попробуйте другое фото") def generate_image( face_image_path, pose_image_path, prompt, negative_prompt, style_name, enhance_face_region, num_steps, identitynet_strength_ratio, adapter_strength_ratio, guidance_scale, seed, upscale, upscaler_model, restoreformer, taesdxl, attention_slicing, progress=gr.Progress(track_tqdm=True), ): if face_image_path is None: raise gr.Error(f"Не могу найти лицо на изображении! Не слишком ли оно крупное или маленькое? Попробуйте другое фото") if prompt is None: prompt = "a person" # apply the style template prompt, negative_prompt = apply_style(style_name, prompt, negative_prompt) face_image = load_image(face_image_path) face_image = resize_img(face_image) face_image_cv2 = convert_from_image_to_cv2(face_image) height, width, _ = face_image_cv2.shape # Extract face features face_info = app.get(face_image_cv2) if len(face_info) == 0: raise gr.Error(f"Не могу найти лицо на изображении! Не слишком ли оно крупное или маленькое? Попробуйте другое фото") face_info = sorted(face_info, key=lambda x:(x['bbox'][2]-x['bbox'][0])*x['bbox'][3]-x['bbox'][1])[-1] # only use the maximum face face_emb = face_info['embedding'] face_kps = draw_kps(convert_from_cv2_to_image(face_image_cv2), face_info['kps']) if pose_image_path is not None: pose_image = load_image(pose_image_path) pose_image = resize_img(pose_image) pose_image_cv2 = convert_from_image_to_cv2(pose_image) face_info = app.get(pose_image_cv2) if len(face_info) == 0: raise gr.Error(f"Не могу найти лицо на изображении! Не слишком ли оно крупное или маленькое? Попробуйте другое фото") face_info = face_info[-1] face_kps = draw_kps(pose_image, face_info['kps']) width, height = face_kps.size if enhance_face_region: control_mask = np.zeros([height, width, 3]) x1, y1, x2, y2 = face_info["bbox"] x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2) control_mask[y1:y2, x1:x2] = 255 control_mask = Image.fromarray(control_mask.astype(np.uint8)) else: control_mask = None generator = torch.Generator(device=device).manual_seed(seed) print("Start inference...") print(f"[Debug] Prompt: {prompt}, \n[Debug] Neg Prompt: {negative_prompt}") pipe.set_ip_adapter_scale(adapter_strength_ratio) if taesdxl: pipe.vae = AutoencoderTiny.from_pretrained("madebyollin/taesdxl", torch_dtype=torch.float32) pipe.vae = pipe.vae.to(device) pipe.enable_vae_tiling() if attention_slicing: pipe.enable_attention_slicing(slice_size="auto") images = pipe( prompt=prompt, negative_prompt=negative_prompt, image_embeds=face_emb, image=face_kps, controlnet_conditioning_scale=float(identitynet_strength_ratio), num_inference_steps=num_steps, guidance_scale=guidance_scale, height=height, width=width, generator=generator ).images if restoreformer: model = SRVGGNetCompact(num_in_ch=3, num_out_ch=3, num_feat=64, num_conv=32, upscale=4, act_type='prelu') half = True if torch.cuda.is_available() else False upsampler = RealESRGANer(scale=upscale, model_path=realesr_model_path, model=model, tile=0, tile_pad=10, pre_pad=0) face_enhancer = RestoreFormer( model_path=restoreformerplusplus_model_path, # Используйте "+" версию для лучшего качества upscale=upscale, bg_upsampler=upsampler, arch='RestoreFormer++', # Установите в 'cuda' или 'cpu' в зависимости от вашей конфигурации, device="cpu" ) # Взять первое изображение из списка и преобразовать его из PIL в формат, который ожидает RestoreFormer generated_image = images[0] # Преобразуйте изображение PIL в формат данных OpenCV cv2_image = np.array(generated_image) cv2_image = cv2_image[:, :, ::-1].copy() # RGB to BGR #weight = 100 _, _, output = face_enhancer.enhance(cv2_image, has_aligned=False, only_center_face=False, paste_back=True) # Обратно преобразовать изображение из OpenCV в PIL для Gradio images = Image.fromarray(output[:, :, ::-1]) # BGR to RGB return images, gr.update(visible=True) else: return images[0], gr.update(visible=True) ### Description title = r"""

InstantID: Zero-shot Identity-Preserving Generation in Seconds / Portable by Neurogen

""" description = r"""

Telegram

Как использовать:
1. Загрузите изображение человека. Если изображений несколько, мы определим только самое близкое лицо. Убедитесь, что лицо не слишком маленькое, не сликшом крупное и не сильно размыто. 2. (По желанию) загрузите другое изображение человека в качестве примерной позы. Если не загружено, мы будем использовать первое изображение человека для извлечения ориентиров. 3. Введите текстовый запрос, как это делается в обычных txt2img моделях. 4. Нажмите кнопку Подвердить, чтобы начать обработку. """ article = r""" ---""" tips = r""" ### Рекомендации по использованию InstantID 1. Если вы не удовлетворены сходством, попробуйте увеличить силу влияния "IdentityNet" и "Image Adapter". 2. Если вы чувствуете, что насыщенность слишком высока, сначала уменьшите силу адаптера. Если она все еще слишком высока, уменьшите силу IdentityNet. 3. Если вы обнаружили, что управление текстом не соответствует ожиданиям, уменьшите силу адаптера. 4. Если вы обнаружили, что реалистичный стиль недостаточно хорош, используйте более реалистичную базовую модель. """ css = """ .gradio-container {width: 85% !important} """ with gr.Blocks(css=css) as demo: # description gr.Markdown(title) gr.Markdown(description) with gr.Row(): with gr.Column(): # upload face image face_file = gr.Image(label="Загрузите фото вашего лица", type="filepath") # optional: upload a reference pose image pose_file = gr.Image(label="Загрузите фото, откуда возьмем позу (опционально)", type="filepath") # prompt prompt = gr.Textbox( label="Подсказка (Промпт)", info="", placeholder="A photo of a person", value="A photo of a person", ) submit = gr.Button("Подвердить", variant="primary") style = gr.Dropdown(label="Стили", choices=STYLE_NAMES, value=DEFAULT_STYLE_NAME) # strength identitynet_strength_ratio = gr.Slider( label="Сила IdentityNet (для схожести)", minimum=0, maximum=1.5, step=0.05, value=0.80, ) adapter_strength_ratio = gr.Slider( label="Сила Image adapter (для деталей)", minimum=0, maximum=1.5, step=0.05, value=0.80, ) with gr.Accordion(open=False, label="Продвинутые опции"): negative_prompt = gr.Textbox( label="Negative Prompt", placeholder="low quality", value="", ) num_steps = gr.Slider( label="Number of sample steps", minimum=2, maximum=20, step=1, value=4, ) guidance_scale = gr.Slider( label="Guidance scale", minimum=0.0, maximum=2.0, step=0.1, value=0.0, ) seed = gr.Slider( label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=42, ) upscaler_model= gr.Dropdown(label="Выберите модель апскейлера / энхейсера", choices=upscalers_path, value=upscalers_path[0], visible=False) randomize_seed = gr.Checkbox(label="Randomize seed", value=True) enhance_face_region = gr.Checkbox(label="Enhance non-face region", value=True) restoreformer = gr.Checkbox(label="Enable RestoreFormer++", value=False) upscale = gr.Slider( label="Select Upscale multiplier (if RestoreFormer Enabled)", info="x1 - Upscale off, x4 - Upscale x4", minimum=1, maximum=4, step=1, value=1, ) taesdxl = gr.Checkbox(label="Enable TinyVAE", value=True) attention_slicing = gr.Checkbox(label="Enable Attention Slicing", value=False) with gr.Column(): output_image = gr.Image(label="Сгенерированное изображение", type="numpy", ) usage_tips = gr.Markdown(label="Usage tips of InstantID", value=tips, visible=False) submit.click( fn=remove_tips, outputs=usage_tips, queue=False, api_name=False, ).then( fn=randomize_seed_fn, inputs=[seed, randomize_seed], outputs=seed, queue=False, api_name=False, ).then( fn=check_input_image, inputs=face_file, queue=False, api_name=False, ).success( fn=generate_image, inputs=[ face_file, pose_file, prompt, negative_prompt, style, enhance_face_region, num_steps, identitynet_strength_ratio, adapter_strength_ratio, guidance_scale, seed, upscale, upscaler_model, restoreformer, taesdxl, attention_slicing ], outputs=[output_image, usage_tips], ) gr.Markdown(article) demo.launch(inbrowser=True, share=True,) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--pretrained_model_name_or_path", type=str, default="wangqixun/YamerMIX_v8") parser.add_argument("--gpu", type=str, default="on") args = parser.parse_args() main(args.pretrained_model_name_or_path,args.gpu)