app.py

from __future__ import annotations

import ctypes
import importlib
import os
import random
import shutil
import subprocess
import sys
import tempfile
import uuid
from pathlib import Path

import gradio as gr
import numpy as np
import spaces
import torch
from diffusers import AutoencoderKL, DDPMScheduler, LCMScheduler, UNet2DConditionModel
from huggingface_hub import hf_hub_download
from PIL import Image
from torchvision import transforms
from transformers import AutoModelForImageSegmentation

# Keep the Space image build simple: fetch a pinned MV-Adapter source snapshot
# at runtime instead of asking pip to resolve the entire upstream package and
# its dependency tree during the Hugging Face image build.
MV_ADAPTER_COMMIT = "4277e0018232bac82bb2c103caf0893cedb711be"
MV_ADAPTER_ROOT = Path("/tmp/basilisc-mv-adapter")

def _ensure_mvadapter_source():
    pkg = MV_ADAPTER_ROOT / "mvadapter"
    if not pkg.is_dir():
        import urllib.request
        import zipfile
        archive = Path("/tmp/basilisc-mv-adapter.zip")
        url = f"https://codeload.github.com/huanngzh/MV-Adapter/zip/{MV_ADAPTER_COMMIT}"
        print(f"Fetching pinned MV-Adapter source: {MV_ADAPTER_COMMIT}", flush=True)
        urllib.request.urlretrieve(url, archive)
        extract_root = Path("/tmp/basilisc-mv-adapter-extract")
        shutil.rmtree(extract_root, ignore_errors=True)
        extract_root.mkdir(parents=True, exist_ok=True)
        with zipfile.ZipFile(archive, "r") as zf:
            zf.extractall(extract_root)
        children = [x for x in extract_root.iterdir() if x.is_dir()]
        if len(children) != 1 or not (children[0] / "mvadapter").is_dir():
            raise RuntimeError("Downloaded MV-Adapter archive had an unexpected layout")
        shutil.rmtree(MV_ADAPTER_ROOT, ignore_errors=True)
        shutil.move(str(children[0]), str(MV_ADAPTER_ROOT))
    if str(MV_ADAPTER_ROOT) not in sys.path:
        sys.path.insert(0, str(MV_ADAPTER_ROOT))

_ensure_mvadapter_source()

from mvadapter.models.attention_processor import DecoupledMVRowColSelfAttnProcessor2_0
from mvadapter.pipelines.pipeline_mvadapter_i2mv_sdxl import MVAdapterI2MVSDXLPipeline
from mvadapter.schedulers.scheduling_shift_snr import ShiftSNRScheduler

DEVICE = "cuda"
DTYPE = torch.float16
NUM_VIEWS = 6
HEIGHT = 768
WIDTH = 768
NEGATIVE = "watermark, ugly, deformed, noisy, blurry, low contrast"
ROOT = Path(__file__).resolve().parent
CHECKPOINTS = ROOT / "checkpoints"
OUTPUTS = ROOT / "outputs"
CHECKPOINTS.mkdir(exist_ok=True)
OUTPUTS.mkdir(exist_ok=True)


def _prepare_pipeline():
    vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix")
    pipe = MVAdapterI2MVSDXLPipeline.from_pretrained(
        "stabilityai/stable-diffusion-xl-base-1.0", vae=vae
    )
    pipe.scheduler = ShiftSNRScheduler.from_scheduler(
        pipe.scheduler, shift_mode="interpolated", shift_scale=8.0, scheduler_class=None
    )
    pipe.init_custom_adapter(
        num_views=NUM_VIEWS,
        self_attn_processor=DecoupledMVRowColSelfAttnProcessor2_0,
    )
    pipe.load_custom_adapter(
        "huanngzh/mv-adapter", weight_name="mvadapter_ig2mv_sdxl.safetensors"
    )
    pipe.to(device=DEVICE, dtype=DTYPE)
    pipe.cond_encoder.to(device=DEVICE, dtype=DTYPE)
    pipe.enable_vae_slicing()
    return pipe


# ZeroGPU emulation allows model placement on CUDA during startup. This is the
# recommended pattern because the real GPU is attached only while @spaces.GPU
# functions execute.
PIPE = _prepare_pipeline()
BIREFNET = AutoModelForImageSegmentation.from_pretrained(
    "ZhengPeng7/BiRefNet", trust_remote_code=True
).to(DEVICE)
BG_TRANSFORM = transforms.Compose(
    [
        transforms.Resize((1024, 1024)),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
    ]
)


def _ensure_checkpoints():
    real_esrgan = CHECKPOINTS / "RealESRGAN_x2plus.pth"
    if not real_esrgan.exists():
        hf_hub_download(
            "dtarnow/UPscaler",
            filename="RealESRGAN_x2plus.pth",
            local_dir=str(CHECKPOINTS),
        )
    lama = CHECKPOINTS / "big-lama.pt"
    if not lama.exists():
        subprocess.check_call(
            [
                "wget", "-q", "-O", str(lama),
                "https://github.com/Sanster/models/releases/download/add_big_lama/big-lama.pt",
            ]
        )
    return real_esrgan, lama


REAL_ESRGAN, BIG_LAMA = _ensure_checkpoints()
_NVD_READY = False
_TEXTURE_PIPE = None


def _cuda_home() -> str:
    candidates = [
        os.environ.get("CUDA_HOME"),
        "/cuda-image/usr/local/cuda-13.0",
        "/usr/local/cuda-13.0",
        "/usr/local/cuda",
    ]
    for item in candidates:
        if item and os.path.isdir(item):
            return item
    return "/cuda-image/usr/local/cuda-13.0"


def _ensure_nvdiffrast():
    """Compile/load nvdiffrast for current ZeroGPU Blackwell/PyTorch ABI."""
    global _NVD_READY
    if _NVD_READY:
        return
    try:
        import nvdiffrast.torch  # noqa: F401
        _NVD_READY = True
        return
    except Exception:
        pass

    cuda_home = _cuda_home()
    patch_dir = tempfile.mkdtemp(prefix="basilisc_torch_cuda_patch_")
    Path(patch_dir, "sitecustomize.py").write_text(
        "try:\n"
        " import torch.utils.cpp_extension as _c\n"
        " _c._check_cuda_version = lambda *a, **k: None\n"
        "except Exception:\n"
        " pass\n",
        encoding="utf-8",
    )
    env = os.environ.copy()
    env["CUDA_HOME"] = cuda_home
    env["CUDA_PATH"] = cuda_home
    env["PATH"] = os.path.join(cuda_home, "bin") + os.pathsep + env.get("PATH", "")
    env["PYTHONPATH"] = patch_dir + os.pathsep + env.get("PYTHONPATH", "")
    env["TORCH_CUDA_ARCH_LIST"] = "12.0"
    env["MAX_JOBS"] = "2"
    subprocess.check_call(
        [
            sys.executable,
            "-m",
            "pip",
            "install",
            "--no-build-isolation",
            "--no-deps",
            "git+https://github.com/NVlabs/nvdiffrast/",
        ],
        env=env,
    )
    importlib.invalidate_caches()

    libdir = os.path.join(cuda_home, "lib64")
    for libname in ("libcudart.so.13", "libcudart.so"):
        lib = os.path.join(libdir, libname)
        if os.path.exists(lib):
            try:
                ctypes.CDLL(lib, mode=ctypes.RTLD_GLOBAL)
                break
            except OSError:
                pass
    os.environ["LD_LIBRARY_PATH"] = libdir + os.pathsep + os.environ.get("LD_LIBRARY_PATH", "")
    import nvdiffrast.torch  # noqa: F401
    _NVD_READY = True


def _remove_bg(image: Image.Image) -> Image.Image:
    image = image.convert("RGB")
    original_size = image.size
    input_images = BG_TRANSFORM(image).unsqueeze(0).to(DEVICE)
    with torch.no_grad():
        preds = BIREFNET(input_images)[-1].sigmoid().cpu()
    mask = transforms.ToPILImage()(preds[0].squeeze()).resize(original_size)
    rgba = image.convert("RGBA")
    rgba.putalpha(mask)
    return rgba


def _preprocess_image(image: Image.Image, height: int, width: int) -> Image.Image:
    arr = np.array(image.convert("RGBA"))
    alpha = arr[..., 3] > 0
    ys, xs = np.where(alpha)
    if len(xs) == 0 or len(ys) == 0:
        return Image.new("RGB", (width, height), (127, 127, 127))
    y0, y1 = max(int(ys.min()) - 1, 0), min(int(ys.max()) + 2, arr.shape[0])
    x0, x1 = max(int(xs.min()) - 1, 0), min(int(xs.max()) + 2, arr.shape[1])
    crop = arr[y0:y1, x0:x1]
    ch, cw = crop.shape[:2]
    scale = min((height * 0.9) / max(ch, 1), (width * 0.9) / max(cw, 1))
    nh, nw = max(1, int(ch * scale)), max(1, int(cw * scale))
    crop = np.array(Image.fromarray(crop).resize((nw, nh), Image.Resampling.LANCZOS))
    canvas = np.zeros((height, width, 4), dtype=np.uint8)
    sy, sx = (height - nh) // 2, (width - nw) // 2
    canvas[sy:sy + nh, sx:sx + nw] = crop
    f = canvas.astype(np.float32) / 255.0
    rgb = f[..., :3] * f[..., 3:4] + (1.0 - f[..., 3:4]) * 0.5
    return Image.fromarray((rgb * 255).clip(0, 255).astype(np.uint8))


def _path(value) -> str:
    if isinstance(value, str):
        return value
    if isinstance(value, dict):
        return value.get("path") or value.get("name") or ""
    return getattr(value, "name", "") or ""


def _run_multiview(mesh_path: str, image_path: str, prompt: str, seed: int, steps: int, ref_strength: float):
    from mvadapter.utils import tensor_to_image
    from mvadapter.utils.mesh_utils import (
        NVDiffRastContextWrapper,
        get_orthogonal_camera,
        load_mesh,
        render,
    )

    cameras = get_orthogonal_camera(
        elevation_deg=[0, 0, 0, 0, 89.99, -89.99],
        distance=[1.8] * NUM_VIEWS,
        left=-0.55,
        right=0.55,
        bottom=-0.55,
        top=0.55,
        azimuth_deg=[x - 90 for x in [0, 90, 180, 270, 180, 180]],
        device=DEVICE,
    )
    ctx = NVDiffRastContextWrapper(device=DEVICE, context_type="cuda")
    mesh = load_mesh(mesh_path, rescale=True, device=DEVICE)
    render_out = render(
        ctx,
        mesh,
        cameras,
        height=HEIGHT,
        width=WIDTH,
        render_attr=False,
        normal_background=0.0,
    )
    control_images = (
        torch.cat(
            [
                (render_out.pos + 0.5).clamp(0, 1),
                (render_out.normal / 2 + 0.5).clamp(0, 1),
            ],
            dim=-1,
        )
        .permute(0, 3, 1, 2)
        .to(DEVICE)
    )
    reference = _preprocess_image(_remove_bg(Image.open(image_path)), HEIGHT, WIDTH)
    generator = torch.Generator(device=DEVICE).manual_seed(int(seed))
    images = PIPE(
        prompt or "high quality",
        height=HEIGHT,
        width=WIDTH,
        num_inference_steps=int(steps),
        guidance_scale=3.0,
        num_images_per_prompt=NUM_VIEWS,
        control_image=control_images,
        control_conditioning_scale=1.0,
        reference_image=reference,
        reference_conditioning_scale=float(ref_strength),
        negative_prompt=NEGATIVE,
        cross_attention_kwargs={"scale": 1.0},
        generator=generator,
    ).images
    del control_images, render_out, mesh, cameras, ctx
    torch.cuda.empty_cache()
    return images


def _texture_pipeline():
    global _TEXTURE_PIPE
    if _TEXTURE_PIPE is not None:
        return _TEXTURE_PIPE

    # MV-Adapter defaults to an OpenGL raster context in the texture pipeline.
    # ZeroGPU is headless, so patch those constructors to use CUDA rasterization.
    import mvadapter.pipelines.pipeline_texture as pt
    import mvadapter.utils.mesh_utils.projection as projection_mod
    import mvadapter.utils.mesh_utils.render as render_mod
    import mvadapter.utils.mesh_utils.smart_paint as smart_mod

    OrigCtx = render_mod.NVDiffRastContextWrapper
    OrigProjection = projection_mod.CameraProjection

    class CudaCtx(OrigCtx):
        def __init__(self, device: str, context_type: str = "cuda"):
            super().__init__(device=device, context_type="cuda")

    class CudaProjection(OrigProjection):
        def __init__(self, pb_backend: str, bg_remover, device: str, context_type: str = "cuda"):
            super().__init__(pb_backend=pb_backend, bg_remover=bg_remover, device=device, context_type="cuda")

    render_mod.NVDiffRastContextWrapper = CudaCtx
    projection_mod.NVDiffRastContextWrapper = CudaCtx
    projection_mod.CameraProjection = CudaProjection
    smart_mod.NVDiffRastContextWrapper = CudaCtx
    smart_mod.CameraProjection = CudaProjection
    pt.NVDiffRastContextWrapper = CudaCtx
    pt.CameraProjection = CudaProjection

    _TEXTURE_PIPE = pt.TexturePipeline(
        upscaler_ckpt_path=str(REAL_ESRGAN),
        inpaint_ckpt_path=str(BIG_LAMA),
        device=DEVICE,
    )
    return _TEXTURE_PIPE


def _estimate_gpu_duration(
    mesh_file,
    reference_file,
    prompt="high quality",
    seed=42,
    steps=25,
    reference_strength=1.0,
    uv_unwrap=False,
    uv_size=2048,
):
    """Reserve only the ZeroGPU time appropriate for the selected preset.

    Draft is intentionally kept below the current anonymous daily allowance;
    Standard and Final are intended for authenticated Hugging Face users.
    """
    try:
        step_count = int(steps)
    except Exception:
        step_count = 25
    try:
        texture_size = int(uv_size)
    except Exception:
        texture_size = 2048
    if step_count <= 15 and texture_size <= 1024:
        return 115
    if step_count <= 25 and texture_size <= 2048:
        return 210
    return 285


@spaces.GPU(duration=_estimate_gpu_duration)
@torch.no_grad()
def basilisc_texture(
    mesh_file,
    reference_file,
    prompt="high quality",
    seed=42,
    steps=25,
    reference_strength=1.0,
    uv_unwrap=False,
    uv_size=2048,
):
    """One-shot mesh + reference image -> textured GLB endpoint for Blender."""
    _ensure_nvdiffrast()
    mesh_path = _path(mesh_file)
    image_path = _path(reference_file)
    if not mesh_path or not os.path.isfile(mesh_path):
        raise gr.Error("Input mesh file was not received")
    if not image_path or not os.path.isfile(image_path):
        raise gr.Error("Reference image file was not received")

    steps = max(8, min(int(steps), 50))
    uv_size = max(1024, min(int(uv_size), 4096))
    # Texture projection operates best on 512-aligned sizes.
    uv_size = int(round(uv_size / 512.0) * 512)
    seed = int(seed) if int(seed) >= 0 else random.randint(0, 2_147_483_647)

    job_dir = OUTPUTS / uuid.uuid4().hex
    job_dir.mkdir(parents=True, exist_ok=True)
    images = _run_multiview(mesh_path, image_path, prompt, seed, steps, reference_strength)

    from mvadapter.utils import make_image_grid
    from mvadapter.pipelines.pipeline_texture import ModProcessConfig

    grid_path = job_dir / "multiview.png"
    make_image_grid(images, rows=1).save(grid_path)
    del images
    torch.cuda.empty_cache()

    tex = _texture_pipeline()
    result = tex(
        mesh_path=mesh_path,
        save_dir=str(job_dir),
        save_name="basilisc_texture",
        uv_unwarp=bool(uv_unwrap),
        preprocess_mesh=False,
        uv_size=uv_size,
        rgb_path=str(grid_path),
        rgb_process_config=ModProcessConfig(view_upscale=True, inpaint_mode="view"),
        camera_azimuth_deg=[x - 90 for x in [0, 90, 180, 270, 180, 180]],
    )
    torch.cuda.empty_cache()
    output = result.shaded_model_save_path
    if not output or not os.path.isfile(output):
        raise gr.Error("Texture pipeline completed without a GLB output")
    return output


with gr.Blocks(title="BASILISC Texture Worker") as demo:
    gr.Markdown(
        "# BASILISC Texture Worker\n"
        "Reference image + existing mesh → MV-Adapter multi-view generation → baked GLB texture."
    )
    with gr.Row():
        mesh = gr.File(label="Mesh (GLB)", file_types=[".glb"], type="filepath")
        reference = gr.Image(label="Reference Image", type="filepath")
    with gr.Accordion("Settings", open=False):
        prompt = gr.Textbox(label="Prompt", value="high quality")
        seed = gr.Number(label="Seed", value=42, precision=0)
        steps = gr.Slider(8, 50, value=25, step=1, label="Diffusion Steps")
        strength = gr.Slider(0.1, 2.0, value=1.0, step=0.05, label="Reference Strength")
        uv_unwrap = gr.Checkbox(label="Generate New UVs", value=False)
        uv_size = gr.Dropdown([1024, 2048, 4096], value=2048, label="Texture Size")
    run = gr.Button("Texture Model", variant="primary")
    output = gr.File(label="Textured GLB")
    run.click(
        fn=basilisc_texture,
        inputs=[mesh, reference, prompt, seed, steps, strength, uv_unwrap, uv_size],
        outputs=[output],
        api_name="basilisc_texture",
    )

if __name__ == "__main__":
    demo.queue(default_concurrency_limit=1).launch(show_error=True)