EchoVision-1.5-Pro / inference.py
Maxilicious20's picture
Upload 5 files
e7c90c5 verified
Raw History Blame Contribute Delete
4.12 kB
"""Generate a 512x512 image with EchoVision 1.5 Pro."""
import argparse
from pathlib import Path
import torch
from diffusers import AutoencoderKL, DDIMScheduler, UNet2DConditionModel
from safetensors.torch import load_file
from transformers import CLIPTextModel, CLIPTokenizer
from PIL import Image
TEXT_ID = "openai/clip-vit-base-patch32"
VAE_ID = "stabilityai/sd-vae-ft-mse"
LATENT_SCALE = 0.18215
@torch.inference_mode()
def main():
parser = argparse.ArgumentParser(description="EchoVision 1.5 Pro inference")
parser.add_argument("--model-dir", default=".", help="Directory with config.json and model.safetensors")
parser.add_argument("--prompt", default="A wooden cabin beside a mountain lake at sunrise")
parser.add_argument("--negative-prompt", default="", help="Optional classifier-free guidance negative prompt")
parser.add_argument("--steps", type=int, default=25)
parser.add_argument("--guidance", type=float, default=7.5)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--output", default="output.png")
args = parser.parse_args()
if not 1 <= args.steps <= 100:
parser.error("--steps must be between 1 and 100")
if not 0 <= args.guidance <= 20:
parser.error("--guidance must be between 0 and 20")
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
dtype = torch.float16 if device.type == "cuda" else torch.float32
root = Path(args.model_dir)
print(f"Loading EchoVision from {root.resolve()} on {device}...")
unet = UNet2DConditionModel.from_config(root / "config.json")
state = load_file(str(root / "model.safetensors"), device="cpu")
unet.load_state_dict(state, strict=True)
del state
unet.to(device=device, dtype=dtype).eval()
tokenizer = CLIPTokenizer.from_pretrained(TEXT_ID)
text_encoder = CLIPTextModel.from_pretrained(TEXT_ID).eval().requires_grad_(False).to("cpu")
vae = AutoencoderKL.from_pretrained(VAE_ID).eval().requires_grad_(False).to(device=device, dtype=dtype)
def embed(text):
tokens = tokenizer([text], padding="max_length", truncation=True, max_length=77, return_tensors="pt")
# Text encoder is kept on CPU, matching the original EchoVision sampling code.
return text_encoder(input_ids=tokens.input_ids, attention_mask=tokens.attention_mask).last_hidden_state.to(device=device, dtype=dtype)
uncond = embed(args.negative_prompt)
cond = embed(args.prompt)
embeddings = torch.cat([uncond, cond], dim=0)
generator = torch.Generator(device=device).manual_seed(args.seed)
latents = torch.randn((1, 4, 64, 64), device=device, dtype=dtype, generator=generator)
scheduler = DDIMScheduler(
num_train_timesteps=1000,
beta_start=0.0001,
beta_end=0.02,
beta_schedule="scaled_linear",
prediction_type="epsilon",
clip_sample=False,
set_alpha_to_one=True,
steps_offset=0,
)
scheduler.set_timesteps(args.steps, device=device)
for timestep in scheduler.timesteps:
model_input = torch.cat([latents, latents], dim=0)
if device.type == "cuda":
with torch.autocast(device_type="cuda", dtype=torch.float16):
prediction = unet(model_input, timestep, encoder_hidden_states=embeddings).sample
else:
prediction = unet(model_input, timestep, encoder_hidden_states=embeddings).sample
prediction_uncond, prediction_cond = prediction.chunk(2)
guided = prediction_uncond + args.guidance * (prediction_cond - prediction_uncond)
latents = scheduler.step(guided, timestep, latents).prev_sample
if device.type == "cuda":
with torch.autocast(device_type="cuda", dtype=torch.float16):
pixels = vae.decode(latents / LATENT_SCALE).sample
else:
pixels = vae.decode(latents / LATENT_SCALE).sample
pixels = ((pixels / 2 + 0.5).clamp(0, 1)[0].permute(1, 2, 0) * 255).round().byte().cpu().numpy()
Image.fromarray(pixels).save(args.output)
print(f"Saved {args.output}")
if __name__ == "__main__":
main()