direct3d-v3 / app.py
multimodalart's picture
multimodalart HF Staff
Upload app.py with huggingface_hub
0a2b2a9 verified
Raw History Blame Contribute Delete
16 kB
"""Direct3D-V3: latent-free diffusion transformers in the native voxel domain.
ZeroGPU port of the repository's reference demo (app.py in
DreamTechAI/Direct3D-V3): a reference image plus an aligned coarse mesh are
refined into a high-resolution surface mesh by a three-stage voxel-domain
diffusion cascade (512 -> 1024 -> 1536).
"""
import os
# Fragmentation guard for the transient spikes of the voxel rasterizer.
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import spaces # noqa: E402 (must precede every CUDA-touching import)
import time
from concurrent.futures import CancelledError, ThreadPoolExecutor
from pathlib import Path
from queue import Queue
from tempfile import TemporaryDirectory
from threading import Event
import gradio as gr
import torch
from PIL import Image
from direct3d import Direct3DV3Pipeline
from direct3d.models.image_encoder import DINOv3FeatureExtractor
from direct3d.pipeline import DEFAULT_MAX_TOKENS, DEFAULT_MODEL_ID, DEFAULT_TARGET_FACES, RESOLUTIONS
from direct3d.utils.image import BackgroundRemover
ASSETS = Path(__file__).resolve().parent / "assets"
TEASER = ASSETS / "demo" / "paper-teaser.webp"
STYLES = ASSETS / "demo" / "app.css"
VIEWER = ASSETS / "demo" / "mesh-viewer.js"
# The image encoder configured in the model card (`facebook/dinov3-...`) is
# gated on the Hub; this byte-identical mirror (same safetensors sha256) keeps
# the demo runnable anonymously. Everything else is loaded from the official
# weights repo.
ENCODER_ID = "cocktailpeanut/dinov3-vith16plus-pretrain-lvd1689m"
pipeline = Direct3DV3Pipeline.from_pretrained(DEFAULT_MODEL_ID, image_encoder=ENCODER_ID)
pipeline.to("cuda")
# Pre-load the two auxiliary CPU models at startup so forked GPU workers
# inherit them instead of re-downloading from the Hub on a cold request.
pipeline._encoder = DINOv3FeatureExtractor(
**pipeline._encoder_options, dtype=pipeline.dtype, local_files_only=pipeline.local_files_only
)
pipeline._remover = BackgroundRemover(pipeline.local_files_only)
DEFAULTS = pipeline.sampling_defaults
# The authors' config uses 15 steps at every stage. A full 15-step 512->1536
# cascade measures ~281 s on zero-a10g, which leaves no headroom under
# ZeroGPU's 300 s per-call reservation limit, so the final refinement stage is
# budgeted at 8 steps (~200 s) and capped at 9.
STAGE_STEPS = {512: DEFAULTS[512][0], 1024: DEFAULTS[1024][0], 1536: 8}
# Measured on zero-a10g: per-stage wall-clock fit (fixed seconds, seconds per
# diffusion step) for one example, plus the shared base cost of background
# removal, voxelisation, marching cubes and GLB export.
_STAGE_COST = {512: (4.0, 0.60), 1024: (9.7, 3.76), 1536: (24.0, 11.6)}
_BASE_SECONDS = 4.0
_MIN_DURATION = 30
_MAX_DURATION = 300 # ZeroGPU refuses longer reservations for free accounts.
def _as_stage(value, default):
"""Coerce a resolution / step value to int (the API client sends radio choices as strings)."""
if value is None:
return default
try:
return int(value)
except (TypeError, ValueError):
return default
def _estimate(image, coarse, resolution, seed, simplify, target_faces, max_tokens, *args, **kwargs):
"""Reserve GPU seconds from the stage plan, sized to measured zero-a10g runs."""
resolution = _as_stage(resolution, 1024)
total = _BASE_SECONDS
for stage, steps in zip(RESOLUTIONS, args[0::2]):
if stage > resolution:
break
fixed, per_step = _STAGE_COST[stage]
total += fixed + per_step * _as_stage(steps, STAGE_STEPS[stage])
return min(_MAX_DURATION, max(_MIN_DURATION, round(total * 1.4)))
def build_demo():
examples = []
thumbnails = []
meshes = sorted((ASSETS / "coarse_meshes").glob("*.obj"), key=lambda path: int(path.stem))
image_dir = ASSETS / "input_imgs"
for normal_map in (False, True):
for mesh in meshes:
if normal_map:
reference = image_dir / f"{mesh.stem}_normal.png"
else:
reference, = image_dir.glob(f"{mesh.stem}.*")
with Image.open(reference) as source:
thumb = source.convert("RGB")
thumb.thumbnail((256, 192), Image.Resampling.LANCZOS)
thumbnails.append(thumb)
examples.append((reference, mesh))
def load_example(event: gr.SelectData):
reference, mesh = examples[event.index]
return str(reference), str(mesh), None, None, ""
def stage_bars(totals, completed):
rows = []
for stage, total in totals.items():
done = completed.get(stage, 0)
rows.append(
f'<label><b>{stage}³</b><progress max="{total}" value="{done}"></progress>'
f'<span>{done} / {total}</span></label>'
)
return '<div class="stage-bars" aria-label="Diffusion sampling progress">' + "".join(rows) + "</div>"
@spaces.GPU(duration=_estimate)
def generate(
image, coarse, resolution, seed, simplify, target_faces, max_tokens,
steps_512, cfg_512, steps_1024, cfg_1024, steps_1536, cfg_1536,
progress=gr.Progress(),
):
"""Refine an aligned coarse mesh into a detailed 3D mesh guided by a reference image.
Args:
image: reference image (photo, render, or surface normal map) guiding surface detail.
coarse: coarse mesh (.obj / .glb / .ply) aligned with the reference camera.
resolution: output voxel resolution; one of 512, 1024, or 1536.
seed: sampling seed; identical inputs and seed reproduce the same mesh.
simplify: decimate the exported mesh to target_faces with pyfqmr.
target_faces: triangle budget for the exported mesh when simplify is on.
max_tokens: per-stage sparse token cap; lower it when the GPU runs out of memory.
steps_512, steps_1024, steps_1536: diffusion steps per cascade stage.
The 1536 stage is capped at 9 steps to fit ZeroGPU's 300 s per-call budget.
cfg_512, cfg_1024, cfg_1536: classifier-free guidance per cascade stage.
Yields:
(foreground cutout, downloadable GLB, stage progress HTML). Intermediate
yields keep the previous mesh in the viewer; the final yield carries it.
"""
started = time.perf_counter()
if image is None or coarse is None:
raise gr.Error("Select an example or upload both a reference image and an aligned coarse mesh.")
stage_steps = {}
stage_cfg = {}
resolution = _as_stage(resolution, 1024)
for stage, count, cfg in (
(512, steps_512, cfg_512),
(1024, steps_1024, cfg_1024),
(1536, steps_1536, cfg_1536),
):
if stage > resolution:
break
stage_steps[stage] = _as_stage(count, STAGE_STEPS[stage])
if cfg is not None:
stage_cfg[stage] = cfg
completed = {}
yield None, None, stage_bars(stage_steps, completed)
progress(0, desc="Removing image background")
cutout = pipeline.remove_background(image)
yield cutout, gr.skip(), gr.skip()
events = Queue()
cancelled = Event()
def report(stage, done, total):
if cancelled.is_set():
raise CancelledError()
events.put((stage, done, total))
with ThreadPoolExecutor(max_workers=1) as executor:
task = executor.submit(
pipeline, cutout, coarse, background_removed=True, resolution=resolution,
num_inference_steps=stage_steps, guidance_scale=stage_cfg, seed=int(seed),
target_faces=int(target_faces) if simplify else None, max_tokens=max_tokens,
on_sampling_step=report,
)
task.add_done_callback(lambda _: events.put(None))
try:
while (event := events.get()) is not None:
stage, done, total = event
completed[stage] = done
if done == 0:
progress(None, desc=f"Preparing {stage}³ geometry")
elif done == total:
progress(None, desc=f"Extracting {stage}³ mesh")
else:
progress((done, total), desc=f"Sampling {stage}³")
yield gr.skip(), gr.skip(), stage_bars(stage_steps, completed)
mesh = task.result()
finally:
cancelled.set()
progress(None, desc="Exporting GLB")
with TemporaryDirectory(prefix="direct3d-") as directory:
output = Path(directory) / "generated.glb"
mesh.export(output)
cached = download.move_resource_to_block_cache(output)
print(
f"[direct3d] resolution={resolution} seed={int(seed)} "
f"elapsed={time.perf_counter() - started:.1f}s",
flush=True,
)
yield cutout, cached, stage_bars(stage_steps, completed)
# Gradio 6 moved custom styles from Blocks to launch and changed these kwargs.
gallery_tools = {"buttons": []}
ui_event = {"api_visibility": "private"}
with gr.Blocks(title="Direct3D-V3", delete_cache=(3600, 3600)) as demo:
gr.HTML(f"""
<header class="paper-banner">
<img class="paper-art" src="gradio_api/file={TEASER.as_posix()}"
alt="A scene assembled from meshes generated by Direct3D-V3">
<div class="paper-info">
<p class="paper-venue">ACM Transactions on Graphics · SIGGRAPH Asia 2026</p>
<h1>Direct3D-V3</h1>
<p class="paper-title">Latent-Free Diffusion Transformers in the Native Voxel Domain</p>
<p class="paper-authors">Shuang Wu, Jiachen Qian, Youtian Lin, Feihu Zhang,
Philip Torr, Yao Yao, and Xun Cao</p>
<nav class="paper-links" aria-label="Project links">
<a class="paper-link" href="https://dl.acm.org/doi/10.1145/3842524"
target="_blank" rel="noopener noreferrer">Paper ↗</a>
<a class="paper-link" href="https://github.com/DreamTechAI/Direct3D-V3"
target="_blank" rel="noopener noreferrer">GitHub ↗</a>
</nav>
</div>
</header>
""", elem_id="paper-header")
with gr.Column(elem_id="examples-strip"):
gr.HTML("""
<div class="examples-heading">
<h2>Examples</h2>
<p role="note">For a more detailed mesh, we recommend using GPT Image 2.5 or
<a href="https://github.com/EnVision-Research/Lotus-2" target="_blank" rel="noopener noreferrer">Lotus-2</a>
to convert your input image into a surface normal map or a white clay render
that preserves fine geometric details.</p>
</div>
""")
gallery = gr.Gallery(
thumbnails, label="Paired examples", show_label=False, columns=6, rows=2,
height="auto", object_fit="contain", allow_preview=False, interactive=False,
format="webp", elem_id="example-gallery", **gallery_tools,
)
with gr.Row(equal_height=False, elem_id="generation-workspace"):
with gr.Column(scale=5, min_width=300, variant="panel", elem_classes="work-panel"):
gr.Markdown("## Inputs")
with gr.Row():
image = gr.Image(
type="pil", label="Reference image", image_mode="RGBA", sources=["upload"],
height=192, min_width=125,
)
foreground = gr.Image(
type="pil", label="Foreground", image_mode="RGBA", format="png",
interactive=False, height=192, min_width=125,
)
coarse = gr.File(
file_types=[".obj", ".glb", ".ply"], type="filepath", label="Aligned coarse mesh",
height=116,
)
# Choices are strings so the public API schema matches what the component
# accepts; the pipeline call sites coerce with _as_stage().
resolution = gr.Radio(
[str(stage) for stage in RESOLUTIONS], value="1024", label="Output resolution",
)
with gr.Accordion("Sampling and mesh options", open=False):
stage_inputs = []
for stage in RESOLUTIONS:
steps, cfg = DEFAULTS[stage]
with gr.Row():
stage_inputs.append(gr.Number(
label=f"{stage} steps", precision=0, minimum=1,
maximum=9 if stage == 1536 else 30,
value=STAGE_STEPS[stage],
))
stage_inputs.append(gr.Number(label=f"{stage} CFG", minimum=0, value=cfg))
with gr.Row():
seed = gr.Number(value=2026, precision=0, label="Seed")
max_tokens = gr.Number(
value=DEFAULT_MAX_TOKENS, minimum=1, precision=0, label="Maximum tokens per stage",
info="If you run out of GPU memory, try a lower value. The mesh may lose fine surface detail.",
)
simplify = gr.Checkbox(value=True, label="Simplify mesh")
target_faces = gr.Slider(
100_000, 5_000_000, step=1, value=DEFAULT_TARGET_FACES, label="Target triangles",
)
simplify.change(
lambda enabled: gr.Slider(interactive=enabled), inputs=simplify, outputs=target_faces,
queue=False, show_progress="hidden", **ui_event,
)
submit = gr.Button("Generate", variant="primary", elem_id="generate-mesh")
stage_progress = gr.HTML("", elem_id="sampling-progress")
with gr.Column(scale=7, min_width=300, variant="panel", elem_classes="work-panel"):
gr.Markdown("## Mesh viewer")
gr.HTML("<direct3d-mesh-viewer></direct3d-mesh-viewer>", elem_id="mesh-viewer")
download = gr.File(label="Download GLB", interactive=False, height=80)
demo.load(fn=None, js=VIEWER.read_text(encoding="utf-8"), **ui_event)
download.change(
fn=None, inputs=download, js="""(file) => {
const viewer = document.querySelector("direct3d-mesh-viewer");
if (viewer) viewer.setAttribute("src", file?.url || "");
}""", **ui_event,
)
results = [foreground, download, stage_progress]
job = submit.click(
generate, [image, coarse, resolution, seed, simplify, target_faces, max_tokens, *stage_inputs],
results, concurrency_limit=1, api_name="generate", show_progress_on=[download],
)
gallery.select(
load_example, outputs=[image, coarse, *results], cancels=[job],
queue=False, show_progress="hidden", **ui_event,
)
gr.on(
triggers=[image.upload, image.clear, coarse.upload, coarse.clear],
fn=lambda: (None, None, ""), outputs=results, cancels=[job],
queue=False, show_progress="hidden", **ui_event,
)
return demo
if __name__ == "__main__":
build_demo().queue().launch(
server_name="0.0.0.0", server_port=7860,
allowed_paths=[str(TEASER)],
theme=gr.themes.Citrus(),
css=".dark .gradio-container { color: var(--body-text-color); }",
css_paths=[STYLES],
mcp_server=True,
)