RotEx / infer_multistep.py

Update RotEx documentation, Apache license, and inference checkpoint paths

0261f7a verified 24 days ago

13.4 kB

import os
import sys
import argparse
import math
import torch
import torch.nn.functional as F
from torchvision import transforms
from torchvision.utils import save_image
from PIL import Image, ImageDraw, ImageFont
import numpy as np
from tqdm import tqdm

sys.path.insert(0, "/workspace/RIFT-9226")

from flux.utils.util import load_ae
from flux.chronflux import ChronFlux
from flux.modules.advanced_encoders import EncoderFactory, DepthEstimator
from config import get_config

PI = np.pi


# ─────────────────────────────────────────────────────────────────────────────
#  Helpers
# ─────────────────────────────────────────────────────────────────────────────

def add_label(tensor_img, text):
    img = (tensor_img.clamp(-1, 1) + 1) / 2
    img = img.squeeze(0).permute(1, 2, 0).cpu().float().numpy()
    img = (img * 255).astype(np.uint8)
    pil_img = Image.fromarray(img)
    draw = ImageDraw.Draw(pil_img)
    try:
        font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 22)
    except Exception:
        font = ImageFont.load_default()
    x, y = 6, 6
    for dx, dy in [(-1, 0), (1, 0), (0, -1), (0, 1)]:
        draw.text((x + dx, y + dy), text, fill="black", font=font)
    draw.text((x, y), text, fill="white", font=font)
    out = np.array(pil_img).astype(np.float32) / 255.0
    out = torch.from_numpy(out).permute(2, 0, 1).unsqueeze(0)
    return out * 2 - 1


def load_model(checkpoint_path, model_config, device, dtype):
    model = ChronFlux(
        in_channels=16,
        hidden_size=1024,
        num_double_layers=8,
        num_single_layers=16,
        num_heads=16,
        style_dim=512,
        use_moe=True,
        use_depth_attn=True,
        use_adain=True,
        moe_params={"num_experts": model_config.moe.num_experts}
    ).to(device).to(dtype)

    state = torch.load(checkpoint_path, map_location="cpu")
    if "model_state_dict" in state:
        state = state["model_state_dict"]
    model.load_state_dict(state, strict=True)
    model.eval()
    return model


def get_cosine_schedule(num_steps):
    return [1.0 - math.cos((i / num_steps) * math.pi / 2.0) for i in range(num_steps)]


# ─────────────────────────────────────────────────────────────────────────────
#  Core denoising loop  (shared by single-seed and multi-seed paths)
# ─────────────────────────────────────────────────────────────────────────────

def denoise(
        model, z_init, content_lat, depth_small, style_vec,
        phi, device, dtype, denoise_steps,
        latent_clamp=20.0,          # clamp during ODE integration
        latent_clamp_final=15.0,    # tighter clamp right before decode
):
    """
    Heun 2nd-order rectified-flow ODE solver.
    Returns the final latent (NOT yet decoded).
    Caller is responsible for ae.decode(z).
    """
    t_schedule = get_cosine_schedule(denoise_steps)
    z = z_init.clone()

    for i in range(denoise_steps):
        t_val = t_schedule[i]
        dt    = (t_schedule[i + 1] - t_schedule[i]) if i < denoise_steps - 1 else (1.0 - t_schedule[i])

        t_cur = torch.tensor([t_val], device=device, dtype=dtype)

        # ── Euler predictor ─────────────────────────────────────────────────
        v1, _ = model(z, content_lat, depth_small, style_vec, t_cur, phi)
        z_mid  = (z + v1 * dt).clamp(-latent_clamp, latent_clamp)

        # ── Heun corrector ───────────────────────────────────────────────────
        t_next_val = t_schedule[i + 1] if i < denoise_steps - 1 else 1.0
        t_next = torch.tensor([t_next_val], device=device, dtype=dtype)
        v2, _ = model(z_mid, content_lat, depth_small, style_vec, t_next, phi)

        z = z + (v1 + v2) * 0.5 * dt
        z = z.clamp(-latent_clamp, latent_clamp)               # keep ODE stable

    # ── Final pre-decode clamp ───────────────────────────────────────────────
    z = z.clamp(-latent_clamp_final, latent_clamp_final)
    return z


# ─────────────────────────────────────────────────────────────────────────────
#  Per-image inference
# ─────────────────────────────────────────────────────────────────────────────

def run_inference(
        model, ae, style_enc, depth_est,
        content_path, style_path,
        device, dtype,
        denoise_steps, image_size, depth_size,
        noise_alpha=0.92,      # [DEPRECATED] kept for signature compatibility
        num_seeds=1,           # >1 → average over multiple trajectories
        hours=None,            # list of hours to render; default = 0..24
):
    """
    Returns a strip tensor:
        [content | style | depth | hour_0 | hour_1 | ... | hour_24]
    """
    if hours is None:
        hours = list(range(25))      # 0 h … 24 h

    transform = transforms.Compose([
        transforms.Resize((image_size, image_size)),
        transforms.ToTensor(),
        transforms.Normalize([0.5], [0.5])
    ])

    content_tensor = transform(Image.open(content_path).convert("RGB")).unsqueeze(0).to(device).to(dtype)
    style_tensor   = transform(Image.open(style_path).convert("RGB")).unsqueeze(0).to(device).to(dtype)

    with torch.no_grad():
        # ── Encode once ─────────────────────────────────────────────────────
        depth_full  = depth_est.predict(content_tensor.float(), image_size).to(dtype)
        depth_small = F.interpolate(depth_full, size=(depth_size, depth_size), mode="nearest")

        content_lat = ae.encode(content_tensor)

        style_vec = style_enc(style_tensor, return_spatial=True)

        depth_vis = (depth_full.repeat(1, 3, 1, 1) - 0.5) / 0.5

        generated = []
        for h in hours:
            phi_val = h * (2 * PI / 24.0)
            phi     = torch.tensor([phi_val], device=device, dtype=dtype)

            if num_seeds == 1:
                # ── Single trajectory ────────────────────────────────────────
                z_init = torch.randn_like(content_lat)

                z_final = denoise(
                    model, z_init, content_lat, depth_small, style_vec,
                    phi, device, dtype, denoise_steps
                )
                out = ae.decode(z_final).clamp(-1, 1)

            else:
                decoded_sum = torch.zeros(1, 3, image_size, image_size, device=device, dtype=torch.float32)
                for _ in range(num_seeds):
                    z_init = torch.randn_like(content_lat)

                    z_final = denoise(
                        model, z_init, content_lat, depth_small, style_vec,
                        phi, device, dtype, denoise_steps
                    )
                    decoded_sum += ae.decode(z_final).clamp(-1, 1).float()
                out = (decoded_sum / num_seeds).clamp(-1, 1)

            generated.append(add_label(out.float(), f"{h}h").to(device))

    content_labeled = add_label(content_tensor.float(), "Content").to(device)
    style_labeled   = add_label(style_tensor.float(),   "Style").to(device)
    depth_labeled   = add_label(depth_vis.float(),      "Depth").to(device)

    strip = torch.cat([content_labeled, style_labeled, depth_labeled] + generated, dim=3)
    return strip


# ─────────────────────────────────────────────────────────────────────────────
#  CLI entry-point
# ─────────────────────────────────────────────────────────────────────────────

def main():
    parser = argparse.ArgumentParser(description="ChronFlux 24-hour inference")
    parser.add_argument("--checkpoint",    type=str,   default="checkpoints/full/best.pt")
    parser.add_argument("--content_dir",   type=str,   default="/workspace/RIFT/rafael_ds_ver2/img_test/cnt")
    parser.add_argument("--style_dir",     type=str,   default="/workspace/RIFT/rafael_ds_ver2/img_test/sty")
    parser.add_argument("--output_dir",    type=str,   default="/workspace/RIFT-9226/inference_outputs")
    parser.add_argument("--denoise_steps", type=int,   default=50,
                        help="More steps = better quality. 50 is a good default; try 75 for maximum quality.")
    parser.add_argument("--image_size",    type=int,   default=384)
    parser.add_argument("--depth_size",    type=int,   default=48)
    parser.add_argument("--noise_alpha",   type=float, default=0.85,
                        help="[DEPRECATED] No longer used. Model requires pure noise init.")
    parser.add_argument("--num_seeds",     type=int,   default=1,
                        help="Average over N random seeds (slow but reduces noise artifacts).")
    parser.add_argument("--device",        type=str,   default="cuda")
    parser.add_argument("--dtype",         type=str,   default="bfloat16")
    args = parser.parse_args()

    device = args.device
    dtype  = torch.bfloat16 if args.dtype == "bfloat16" else torch.float32

    os.makedirs(args.output_dir, exist_ok=True)

    print(f"[INFO] Loading config...")
    model_config = get_config("q1_paper")

    print(f"[INFO] Loading AutoEncoder...")
    ae = load_ae(device).to(device).to(dtype).eval().requires_grad_(False)

    print(f"[INFO] Loading Style Encoder (DINOv2)...")
    style_enc = EncoderFactory.create_encoder(
        "dinov2", device, 512, False, model_size="base"
    ).to(device).to(dtype).eval().requires_grad_(False)

    print(f"[INFO] Loading Depth Estimator...")
    depth_est = DepthEstimator(device)

    print(f"[INFO] Loading ChronFlux from {args.checkpoint}...")
    model = load_model(args.checkpoint, model_config, device, dtype)
    model.requires_grad_(False)

    print(f"[INFO] denoise_steps={args.denoise_steps} | num_seeds={args.num_seeds} | Pure Noise Init: ENABLED")

    content_files = sorted([
        f for f in os.listdir(args.content_dir)
        if f.lower().endswith((".jpg", ".jpeg", ".png"))
    ])
    style_files = sorted([
        f for f in os.listdir(args.style_dir)
        if f.lower().endswith((".jpg", ".jpeg", ".png"))
    ])

    if not content_files:
        print("[ERROR] No content images found.")
        return
    if not style_files:
        print("[ERROR] No style images found.")
        return

    print(f"[INFO] Found {len(content_files)} content images and {len(style_files)} style images.")

    for content_file in tqdm(content_files, desc="Content images"):
        content_path = os.path.join(args.content_dir, content_file)
        content_stem = os.path.splitext(content_file)[0]

        for style_file in style_files:
            style_path = os.path.join(args.style_dir, style_file)
            style_stem = os.path.splitext(style_file)[0]

            out_name = f"{content_stem}__sty_{style_stem}.png"
            out_path = os.path.join(args.output_dir, out_name)

            if os.path.exists(out_path):
                print(f"[SKIP] {out_name} already exists.")
                continue

            try:
                strip = run_inference(
                    model=model,
                    ae=ae,
                    style_enc=style_enc,
                    depth_est=depth_est,
                    content_path=content_path,
                    style_path=style_path,
                    device=device,
                    dtype=dtype,
                    denoise_steps=args.denoise_steps,
                    image_size=args.image_size,
                    depth_size=args.depth_size,
                    noise_alpha=args.noise_alpha,
                    num_seeds=args.num_seeds,
                )

                save_image(strip, out_path, normalize=True, value_range=(-1, 1))
                print(f"[SAVED] {out_path}")

            except Exception as e:
                print(f"[ERROR] Failed on {content_file} x {style_file}: {e}")
                import traceback
                traceback.print_exc()

    print(f"\n[DONE] All outputs saved to: {args.output_dir}")


if __name__ == "__main__":
    main()

	import os
	import sys
	import argparse
	import math
	import torch
	import torch.nn.functional as F
	from torchvision import transforms
	from torchvision.utils import save_image
	from PIL import Image, ImageDraw, ImageFont
	import numpy as np
	from tqdm import tqdm

	sys.path.insert(0, "/workspace/RIFT-9226")

	from flux.utils.util import load_ae
	from flux.chronflux import ChronFlux
	from flux.modules.advanced_encoders import EncoderFactory, DepthEstimator
	from config import get_config

	PI = np.pi


	# ─────────────────────────────────────────────────────────────────────────────
	# Helpers
	# ─────────────────────────────────────────────────────────────────────────────

	def add_label(tensor_img, text):
	img = (tensor_img.clamp(-1, 1) + 1) / 2
	img = img.squeeze(0).permute(1, 2, 0).cpu().float().numpy()
	img = (img * 255).astype(np.uint8)
	pil_img = Image.fromarray(img)
	draw = ImageDraw.Draw(pil_img)
	try:
	font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 22)
	except Exception:
	font = ImageFont.load_default()
	x, y = 6, 6
	for dx, dy in [(-1, 0), (1, 0), (0, -1), (0, 1)]:
	draw.text((x + dx, y + dy), text, fill="black", font=font)
	draw.text((x, y), text, fill="white", font=font)
	out = np.array(pil_img).astype(np.float32) / 255.0
	out = torch.from_numpy(out).permute(2, 0, 1).unsqueeze(0)
	return out * 2 - 1


	def load_model(checkpoint_path, model_config, device, dtype):
	model = ChronFlux(
	in_channels=16,
	hidden_size=1024,
	num_double_layers=8,
	num_single_layers=16,
	num_heads=16,
	style_dim=512,
	use_moe=True,
	use_depth_attn=True,
	use_adain=True,
	moe_params={"num_experts": model_config.moe.num_experts}
	).to(device).to(dtype)

	state = torch.load(checkpoint_path, map_location="cpu")
	if "model_state_dict" in state:
	state = state["model_state_dict"]
	model.load_state_dict(state, strict=True)
	model.eval()
	return model


	def get_cosine_schedule(num_steps):
	return [1.0 - math.cos((i / num_steps) * math.pi / 2.0) for i in range(num_steps)]


	# ─────────────────────────────────────────────────────────────────────────────
	# Core denoising loop (shared by single-seed and multi-seed paths)
	# ─────────────────────────────────────────────────────────────────────────────

	def denoise(
	model, z_init, content_lat, depth_small, style_vec,
	phi, device, dtype, denoise_steps,
	latent_clamp=20.0, # clamp during ODE integration
	latent_clamp_final=15.0, # tighter clamp right before decode
	):
	"""
	Heun 2nd-order rectified-flow ODE solver.

	Returns the final latent (NOT yet decoded).
	Caller is responsible for ae.decode(z).
	"""
	t_schedule = get_cosine_schedule(denoise_steps)
	z = z_init.clone()

	for i in range(denoise_steps):
	t_val = t_schedule[i]
	dt = (t_schedule[i + 1] - t_schedule[i]) if i < denoise_steps - 1 else (1.0 - t_schedule[i])

	t_cur = torch.tensor([t_val], device=device, dtype=dtype)

	# ── Euler predictor ─────────────────────────────────────────────────
	v1, _ = model(z, content_lat, depth_small, style_vec, t_cur, phi)
	z_mid = (z + v1 * dt).clamp(-latent_clamp, latent_clamp)

	# ── Heun corrector ───────────────────────────────────────────────────
	t_next_val = t_schedule[i + 1] if i < denoise_steps - 1 else 1.0
	t_next = torch.tensor([t_next_val], device=device, dtype=dtype)
	v2, _ = model(z_mid, content_lat, depth_small, style_vec, t_next, phi)

	z = z + (v1 + v2) * 0.5 * dt
	z = z.clamp(-latent_clamp, latent_clamp) # keep ODE stable

	# ── Final pre-decode clamp ───────────────────────────────────────────────
	z = z.clamp(-latent_clamp_final, latent_clamp_final)
	return z


	# ─────────────────────────────────────────────────────────────────────────────
	# Per-image inference
	# ─────────────────────────────────────────────────────────────────────────────

	def run_inference(
	model, ae, style_enc, depth_est,
	content_path, style_path,
	device, dtype,
	denoise_steps, image_size, depth_size,
	noise_alpha=0.92, # [DEPRECATED] kept for signature compatibility
	num_seeds=1, # >1 → average over multiple trajectories
	hours=None, # list of hours to render; default = 0..24
	):
	"""
	Returns a strip tensor:
	[content \| style \| depth \| hour_0 \| hour_1 \| ... \| hour_24]
	"""
	if hours is None:
	hours = list(range(25)) # 0 h … 24 h

	transform = transforms.Compose([
	transforms.Resize((image_size, image_size)),
	transforms.ToTensor(),
	transforms.Normalize([0.5], [0.5])
	])

	content_tensor = transform(Image.open(content_path).convert("RGB")).unsqueeze(0).to(device).to(dtype)
	style_tensor = transform(Image.open(style_path).convert("RGB")).unsqueeze(0).to(device).to(dtype)

	with torch.no_grad():
	# ── Encode once ─────────────────────────────────────────────────────
	depth_full = depth_est.predict(content_tensor.float(), image_size).to(dtype)
	depth_small = F.interpolate(depth_full, size=(depth_size, depth_size), mode="nearest")

	content_lat = ae.encode(content_tensor)

	style_vec = style_enc(style_tensor, return_spatial=True)

	depth_vis = (depth_full.repeat(1, 3, 1, 1) - 0.5) / 0.5

	generated = []
	for h in hours:
	phi_val = h * (2 * PI / 24.0)
	phi = torch.tensor([phi_val], device=device, dtype=dtype)

	if num_seeds == 1:
	# ── Single trajectory ────────────────────────────────────────
	z_init = torch.randn_like(content_lat)

	z_final = denoise(
	model, z_init, content_lat, depth_small, style_vec,
	phi, device, dtype, denoise_steps
	)
	out = ae.decode(z_final).clamp(-1, 1)

	else:
	decoded_sum = torch.zeros(1, 3, image_size, image_size, device=device, dtype=torch.float32)
	for _ in range(num_seeds):
	z_init = torch.randn_like(content_lat)

	z_final = denoise(
	model, z_init, content_lat, depth_small, style_vec,
	phi, device, dtype, denoise_steps
	)
	decoded_sum += ae.decode(z_final).clamp(-1, 1).float()
	out = (decoded_sum / num_seeds).clamp(-1, 1)

	generated.append(add_label(out.float(), f"{h}h").to(device))

	content_labeled = add_label(content_tensor.float(), "Content").to(device)
	style_labeled = add_label(style_tensor.float(), "Style").to(device)
	depth_labeled = add_label(depth_vis.float(), "Depth").to(device)

	strip = torch.cat([content_labeled, style_labeled, depth_labeled] + generated, dim=3)
	return strip


	# ─────────────────────────────────────────────────────────────────────────────
	# CLI entry-point
	# ─────────────────────────────────────────────────────────────────────────────

	def main():
	parser = argparse.ArgumentParser(description="ChronFlux 24-hour inference")
	parser.add_argument("--checkpoint", type=str, default="checkpoints/full/best.pt")
	parser.add_argument("--content_dir", type=str, default="/workspace/RIFT/rafael_ds_ver2/img_test/cnt")
	parser.add_argument("--style_dir", type=str, default="/workspace/RIFT/rafael_ds_ver2/img_test/sty")
	parser.add_argument("--output_dir", type=str, default="/workspace/RIFT-9226/inference_outputs")
	parser.add_argument("--denoise_steps", type=int, default=50,
	help="More steps = better quality. 50 is a good default; try 75 for maximum quality.")
	parser.add_argument("--image_size", type=int, default=384)
	parser.add_argument("--depth_size", type=int, default=48)
	parser.add_argument("--noise_alpha", type=float, default=0.85,
	help="[DEPRECATED] No longer used. Model requires pure noise init.")
	parser.add_argument("--num_seeds", type=int, default=1,
	help="Average over N random seeds (slow but reduces noise artifacts).")
	parser.add_argument("--device", type=str, default="cuda")
	parser.add_argument("--dtype", type=str, default="bfloat16")
	args = parser.parse_args()

	device = args.device
	dtype = torch.bfloat16 if args.dtype == "bfloat16" else torch.float32

	os.makedirs(args.output_dir, exist_ok=True)

	print(f"[INFO] Loading config...")
	model_config = get_config("q1_paper")

	print(f"[INFO] Loading AutoEncoder...")
	ae = load_ae(device).to(device).to(dtype).eval().requires_grad_(False)

	print(f"[INFO] Loading Style Encoder (DINOv2)...")
	style_enc = EncoderFactory.create_encoder(
	"dinov2", device, 512, False, model_size="base"
	).to(device).to(dtype).eval().requires_grad_(False)

	print(f"[INFO] Loading Depth Estimator...")
	depth_est = DepthEstimator(device)

	print(f"[INFO] Loading ChronFlux from {args.checkpoint}...")
	model = load_model(args.checkpoint, model_config, device, dtype)
	model.requires_grad_(False)

	print(f"[INFO] denoise_steps={args.denoise_steps} \| num_seeds={args.num_seeds} \| Pure Noise Init: ENABLED")

	content_files = sorted([
	f for f in os.listdir(args.content_dir)
	if f.lower().endswith((".jpg", ".jpeg", ".png"))
	])
	style_files = sorted([
	f for f in os.listdir(args.style_dir)
	if f.lower().endswith((".jpg", ".jpeg", ".png"))
	])

	if not content_files:
	print("[ERROR] No content images found.")
	return
	if not style_files:
	print("[ERROR] No style images found.")
	return

	print(f"[INFO] Found {len(content_files)} content images and {len(style_files)} style images.")

	for content_file in tqdm(content_files, desc="Content images"):
	content_path = os.path.join(args.content_dir, content_file)
	content_stem = os.path.splitext(content_file)[0]

	for style_file in style_files:
	style_path = os.path.join(args.style_dir, style_file)
	style_stem = os.path.splitext(style_file)[0]

	out_name = f"{content_stem}__sty_{style_stem}.png"
	out_path = os.path.join(args.output_dir, out_name)

	if os.path.exists(out_path):
	print(f"[SKIP] {out_name} already exists.")
	continue

	try:
	strip = run_inference(
	model=model,
	ae=ae,
	style_enc=style_enc,
	depth_est=depth_est,
	content_path=content_path,
	style_path=style_path,
	device=device,
	dtype=dtype,
	denoise_steps=args.denoise_steps,
	image_size=args.image_size,
	depth_size=args.depth_size,
	noise_alpha=args.noise_alpha,
	num_seeds=args.num_seeds,
	)

	save_image(strip, out_path, normalize=True, value_range=(-1, 1))
	print(f"[SAVED] {out_path}")

	except Exception as e:
	print(f"[ERROR] Failed on {content_file} x {style_file}: {e}")
	import traceback
	traceback.print_exc()

	print(f"\n[DONE] All outputs saved to: {args.output_dir}")


	if __name__ == "__main__":
	main()