app.py

import spaces
import torch

# diffusers 0.29.2 imports FLAX_WEIGHTS_NAME from transformers.utils, removed in
# transformers 5.x. Shim it back before importing diffusers.
import transformers.utils as _tu
if not hasattr(_tu, "FLAX_WEIGHTS_NAME"):
    _tu.FLAX_WEIGHTS_NAME = "flax_model.msgpack"

from diffusers import StableDiffusionPipeline, DDIMScheduler, AutoencoderKL
from transformers import CLIPImageProcessor
from ip_adapter.pipeline_stable_diffusion_extra_cfg import StableDiffusionPipelineCFG
from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker

# transformers 5.x's _finalize_model_loading reads self.all_tied_weights_keys,
# which diffusers 0.29.2's StableDiffusionSafetyChecker (older PreTrainedModel)
# does not define. Provide an empty default so loading succeeds.
if not hasattr(StableDiffusionSafetyChecker, "all_tied_weights_keys"):
    StableDiffusionSafetyChecker.all_tied_weights_keys = {}

from ip_adapter.ip_adapter_instruct import IPAdapterInstruct
from huggingface_hub import hf_hub_download
import gradio as gr
import cv2

base_model_path = "SG161222/Realistic_Vision_V4.0_noVAE"
vae_model_path = "stabilityai/sd-vae-ft-mse"
image_encoder_path = "laion/CLIP-ViT-H-14-laion2B-s32B-b79K"
ip_ckpt = hf_hub_download(repo_id="CiaraRowles/IP-Adapter-Instruct", filename="ip-adapter-instruct-sd15.bin", repo_type="model")

safety_model_id = "CompVis/stable-diffusion-safety-checker"
safety_feature_extractor = CLIPImageProcessor.from_pretrained(safety_model_id)
safety_checker = StableDiffusionSafetyChecker.from_pretrained(safety_model_id)

device = "cuda"

noise_scheduler = DDIMScheduler(
    num_train_timesteps=1000,
    beta_start=0.00085,
    beta_end=0.012,
    beta_schedule="scaled_linear",
    clip_sample=False,
    set_alpha_to_one=False,
    steps_offset=1,
)
vae = AutoencoderKL.from_pretrained(vae_model_path).to(dtype=torch.float16)
pipe = StableDiffusionPipelineCFG.from_pretrained(
    base_model_path,
    scheduler=noise_scheduler,
    vae=vae,
    torch_dtype=torch.float16,
    feature_extractor=safety_feature_extractor,
    safety_checker=safety_checker
).to(device)

#pipe.load_lora_weights("h94/IP-Adapter-FaceID", weight_name="ip-adapter-faceid-plusv2_sd15_lora.safetensors")
#pipe.fuse_lora()

ip_model = IPAdapterInstruct(pipe, image_encoder_path, ip_ckpt, device,dtypein=torch.float16,num_tokens=16)

cv2.setNumThreads(1)

@spaces.GPU
def generate_image(images, prompt, negative_prompt,instruct_query, scale, nfaa_negative_prompt, progress=gr.Progress(track_tqdm=True)):
    faceid_all_embeds = []
    first_iteration = True
    image = images
    yield None
    total_negative_prompt = f"{negative_prompt} {nfaa_negative_prompt}"
    print("Generating normal")

    # Calculate aspect ratio
    aspect_ratio = image.width / image.height

    # Set base_size (you can adjust this value as needed)
    base_size = 512

    # Calculate new width and height
    if aspect_ratio > 1:  # Landscape
        new_width = base_size
        new_height = int(base_size / aspect_ratio)
    else:  # Portrait or square
        new_height = base_size
        new_width = int(base_size * aspect_ratio)

    # Ensure dimensions are multiples of 8 (required by some models)
    new_width = (new_width // 8) * 8
    new_height = (new_height // 8) * 8

    image = ip_model.generate(
        prompt=prompt,
        negative_prompt=total_negative_prompt,
        pil_image=image,
        scale=scale,
        width=new_width,
        height=new_height,
        num_inference_steps=30,
        query=instruct_query
    )
    
    yield image



def swap_to_gallery(images):
    return gr.update(value=images, visible=True), gr.update(visible=True), gr.update(visible=False)

def remove_back_to_files():
    return gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)
css = '''
h1{margin-bottom: 0 !important}
'''
with gr.Blocks(css=css) as demo:
    gr.Markdown("# IP-Adapter-Instruct demo")
    gr.Markdown("Demo for the [CiaraRowles/IP-Adapter-Instruct model](https://huggingface.co/CiaraRowles/IP-Adapter-Instruct)")
    with gr.Row():
        with gr.Column():
            files = gr.Image(
                label="Input image",
                type="pil"
            )
            uploaded_files = gr.Gallery(label="Your image", visible=False, columns=5, rows=1, height=125)
            with gr.Column(visible=False) as clear_button:
                remove_and_reupload = gr.ClearButton(value="Remove and upload new ones", components=files, size="sm")
            prompt = gr.Textbox(label="Prompt",
                       info="Try something like 'a photo of a man/woman/person'",
                       placeholder="A photo of a [man/woman/person]...")
            instruct_query = gr.Dropdown(
                label="Instruct Query",
                choices=[
                    "use everything from the image",
                    "use the style",
                    "use the colour",
                    "use the pose",
                    "use the composition",
                    "use the face"
                ],
                value="use everything from the image"
            )
            negative_prompt = gr.Textbox(label="Negative Prompt", placeholder="low quality")
            submit = gr.Button("Submit")
            with gr.Accordion(open=False, label="Advanced Options"):
                nfaa_negative_prompts = gr.Textbox(label="Appended Negative Prompts", info="Negative prompts to steer generations towards safe for all audiences outputs", value="naked, bikini, skimpy, scanty, bare skin, lingerie, swimsuit, exposed, see-through")    
                scale = gr.Slider(label="Scale", value=0.8, step=0.1, minimum=0, maximum=2)
        with gr.Column():
            gallery = gr.Gallery(label="Generated Images")
        
        submit.click(fn=generate_image,
                    inputs=[files, prompt, negative_prompt,instruct_query, scale, nfaa_negative_prompts],
                    outputs=gallery)
    
    gr.Markdown("This demo includes extra features to mitigate the implicit bias of the model and prevent explicit usage of it to generate content with faces of people, including third parties, that is not safe for all audiences, including naked or semi-naked people.")
    gr.Markdown("based on: https://huggingface.co/spaces/multimodalart/Ip-Adapter-FaceID")

demo.launch()