Spaces:
Running on Zero
Running on Zero
Download app.py from hugging-apps/direct3d-v3: direct link, hf CLI and curl.
- Browser
- Download file 16 kB
-
https://huggingface.co/spaces/hugging-apps/direct3d-v3/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-apps/direct3d-v3/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-apps/direct3d-v3/resolve/main/app.py
16 kB
| """Direct3D-V3: latent-free diffusion transformers in the native voxel domain. | |
| ZeroGPU port of the repository's reference demo (app.py in | |
| DreamTechAI/Direct3D-V3): a reference image plus an aligned coarse mesh are | |
| refined into a high-resolution surface mesh by a three-stage voxel-domain | |
| diffusion cascade (512 -> 1024 -> 1536). | |
| """ | |
| import os | |
| # Fragmentation guard for the transient spikes of the voxel rasterizer. | |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") | |
| import spaces # noqa: E402 (must precede every CUDA-touching import) | |
| import time | |
| from concurrent.futures import CancelledError, ThreadPoolExecutor | |
| from pathlib import Path | |
| from queue import Queue | |
| from tempfile import TemporaryDirectory | |
| from threading import Event | |
| import gradio as gr | |
| import torch | |
| from PIL import Image | |
| from direct3d import Direct3DV3Pipeline | |
| from direct3d.models.image_encoder import DINOv3FeatureExtractor | |
| from direct3d.pipeline import DEFAULT_MAX_TOKENS, DEFAULT_MODEL_ID, DEFAULT_TARGET_FACES, RESOLUTIONS | |
| from direct3d.utils.image import BackgroundRemover | |
| ASSETS = Path(__file__).resolve().parent / "assets" | |
| TEASER = ASSETS / "demo" / "paper-teaser.webp" | |
| STYLES = ASSETS / "demo" / "app.css" | |
| VIEWER = ASSETS / "demo" / "mesh-viewer.js" | |
| # The image encoder configured in the model card (`facebook/dinov3-...`) is | |
| # gated on the Hub; this byte-identical mirror (same safetensors sha256) keeps | |
| # the demo runnable anonymously. Everything else is loaded from the official | |
| # weights repo. | |
| ENCODER_ID = "cocktailpeanut/dinov3-vith16plus-pretrain-lvd1689m" | |
| pipeline = Direct3DV3Pipeline.from_pretrained(DEFAULT_MODEL_ID, image_encoder=ENCODER_ID) | |
| pipeline.to("cuda") | |
| # Pre-load the two auxiliary CPU models at startup so forked GPU workers | |
| # inherit them instead of re-downloading from the Hub on a cold request. | |
| pipeline._encoder = DINOv3FeatureExtractor( | |
| **pipeline._encoder_options, dtype=pipeline.dtype, local_files_only=pipeline.local_files_only | |
| ) | |
| pipeline._remover = BackgroundRemover(pipeline.local_files_only) | |
| DEFAULTS = pipeline.sampling_defaults | |
| # The authors' config uses 15 steps at every stage. A full 15-step 512->1536 | |
| # cascade measures ~281 s on zero-a10g, which leaves no headroom under | |
| # ZeroGPU's 300 s per-call reservation limit, so the final refinement stage is | |
| # budgeted at 8 steps (~200 s) and capped at 9. | |
| STAGE_STEPS = {512: DEFAULTS[512][0], 1024: DEFAULTS[1024][0], 1536: 8} | |
| # Measured on zero-a10g: per-stage wall-clock fit (fixed seconds, seconds per | |
| # diffusion step) for one example, plus the shared base cost of background | |
| # removal, voxelisation, marching cubes and GLB export. | |
| _STAGE_COST = {512: (4.0, 0.60), 1024: (9.7, 3.76), 1536: (24.0, 11.6)} | |
| _BASE_SECONDS = 4.0 | |
| _MIN_DURATION = 30 | |
| _MAX_DURATION = 300 # ZeroGPU refuses longer reservations for free accounts. | |
| def _as_stage(value, default): | |
| """Coerce a resolution / step value to int (the API client sends radio choices as strings).""" | |
| if value is None: | |
| return default | |
| try: | |
| return int(value) | |
| except (TypeError, ValueError): | |
| return default | |
| def _estimate(image, coarse, resolution, seed, simplify, target_faces, max_tokens, *args, **kwargs): | |
| """Reserve GPU seconds from the stage plan, sized to measured zero-a10g runs.""" | |
| resolution = _as_stage(resolution, 1024) | |
| total = _BASE_SECONDS | |
| for stage, steps in zip(RESOLUTIONS, args[0::2]): | |
| if stage > resolution: | |
| break | |
| fixed, per_step = _STAGE_COST[stage] | |
| total += fixed + per_step * _as_stage(steps, STAGE_STEPS[stage]) | |
| return min(_MAX_DURATION, max(_MIN_DURATION, round(total * 1.4))) | |
| def build_demo(): | |
| examples = [] | |
| thumbnails = [] | |
| meshes = sorted((ASSETS / "coarse_meshes").glob("*.obj"), key=lambda path: int(path.stem)) | |
| image_dir = ASSETS / "input_imgs" | |
| for normal_map in (False, True): | |
| for mesh in meshes: | |
| if normal_map: | |
| reference = image_dir / f"{mesh.stem}_normal.png" | |
| else: | |
| reference, = image_dir.glob(f"{mesh.stem}.*") | |
| with Image.open(reference) as source: | |
| thumb = source.convert("RGB") | |
| thumb.thumbnail((256, 192), Image.Resampling.LANCZOS) | |
| thumbnails.append(thumb) | |
| examples.append((reference, mesh)) | |
| def load_example(event: gr.SelectData): | |
| reference, mesh = examples[event.index] | |
| return str(reference), str(mesh), None, None, "" | |
| def stage_bars(totals, completed): | |
| rows = [] | |
| for stage, total in totals.items(): | |
| done = completed.get(stage, 0) | |
| rows.append( | |
| f'<label><b>{stage}³</b><progress max="{total}" value="{done}"></progress>' | |
| f'<span>{done} / {total}</span></label>' | |
| ) | |
| return '<div class="stage-bars" aria-label="Diffusion sampling progress">' + "".join(rows) + "</div>" | |
| def generate( | |
| image, coarse, resolution, seed, simplify, target_faces, max_tokens, | |
| steps_512, cfg_512, steps_1024, cfg_1024, steps_1536, cfg_1536, | |
| progress=gr.Progress(), | |
| ): | |
| """Refine an aligned coarse mesh into a detailed 3D mesh guided by a reference image. | |
| Args: | |
| image: reference image (photo, render, or surface normal map) guiding surface detail. | |
| coarse: coarse mesh (.obj / .glb / .ply) aligned with the reference camera. | |
| resolution: output voxel resolution; one of 512, 1024, or 1536. | |
| seed: sampling seed; identical inputs and seed reproduce the same mesh. | |
| simplify: decimate the exported mesh to target_faces with pyfqmr. | |
| target_faces: triangle budget for the exported mesh when simplify is on. | |
| max_tokens: per-stage sparse token cap; lower it when the GPU runs out of memory. | |
| steps_512, steps_1024, steps_1536: diffusion steps per cascade stage. | |
| The 1536 stage is capped at 9 steps to fit ZeroGPU's 300 s per-call budget. | |
| cfg_512, cfg_1024, cfg_1536: classifier-free guidance per cascade stage. | |
| Yields: | |
| (foreground cutout, downloadable GLB, stage progress HTML). Intermediate | |
| yields keep the previous mesh in the viewer; the final yield carries it. | |
| """ | |
| started = time.perf_counter() | |
| if image is None or coarse is None: | |
| raise gr.Error("Select an example or upload both a reference image and an aligned coarse mesh.") | |
| stage_steps = {} | |
| stage_cfg = {} | |
| resolution = _as_stage(resolution, 1024) | |
| for stage, count, cfg in ( | |
| (512, steps_512, cfg_512), | |
| (1024, steps_1024, cfg_1024), | |
| (1536, steps_1536, cfg_1536), | |
| ): | |
| if stage > resolution: | |
| break | |
| stage_steps[stage] = _as_stage(count, STAGE_STEPS[stage]) | |
| if cfg is not None: | |
| stage_cfg[stage] = cfg | |
| completed = {} | |
| yield None, None, stage_bars(stage_steps, completed) | |
| progress(0, desc="Removing image background") | |
| cutout = pipeline.remove_background(image) | |
| yield cutout, gr.skip(), gr.skip() | |
| events = Queue() | |
| cancelled = Event() | |
| def report(stage, done, total): | |
| if cancelled.is_set(): | |
| raise CancelledError() | |
| events.put((stage, done, total)) | |
| with ThreadPoolExecutor(max_workers=1) as executor: | |
| task = executor.submit( | |
| pipeline, cutout, coarse, background_removed=True, resolution=resolution, | |
| num_inference_steps=stage_steps, guidance_scale=stage_cfg, seed=int(seed), | |
| target_faces=int(target_faces) if simplify else None, max_tokens=max_tokens, | |
| on_sampling_step=report, | |
| ) | |
| task.add_done_callback(lambda _: events.put(None)) | |
| try: | |
| while (event := events.get()) is not None: | |
| stage, done, total = event | |
| completed[stage] = done | |
| if done == 0: | |
| progress(None, desc=f"Preparing {stage}³ geometry") | |
| elif done == total: | |
| progress(None, desc=f"Extracting {stage}³ mesh") | |
| else: | |
| progress((done, total), desc=f"Sampling {stage}³") | |
| yield gr.skip(), gr.skip(), stage_bars(stage_steps, completed) | |
| mesh = task.result() | |
| finally: | |
| cancelled.set() | |
| progress(None, desc="Exporting GLB") | |
| with TemporaryDirectory(prefix="direct3d-") as directory: | |
| output = Path(directory) / "generated.glb" | |
| mesh.export(output) | |
| cached = download.move_resource_to_block_cache(output) | |
| print( | |
| f"[direct3d] resolution={resolution} seed={int(seed)} " | |
| f"elapsed={time.perf_counter() - started:.1f}s", | |
| flush=True, | |
| ) | |
| yield cutout, cached, stage_bars(stage_steps, completed) | |
| # Gradio 6 moved custom styles from Blocks to launch and changed these kwargs. | |
| gallery_tools = {"buttons": []} | |
| ui_event = {"api_visibility": "private"} | |
| with gr.Blocks(title="Direct3D-V3", delete_cache=(3600, 3600)) as demo: | |
| gr.HTML(f""" | |
| <header class="paper-banner"> | |
| <img class="paper-art" src="gradio_api/file={TEASER.as_posix()}" | |
| alt="A scene assembled from meshes generated by Direct3D-V3"> | |
| <div class="paper-info"> | |
| <p class="paper-venue">ACM Transactions on Graphics · SIGGRAPH Asia 2026</p> | |
| <h1>Direct3D-V3</h1> | |
| <p class="paper-title">Latent-Free Diffusion Transformers in the Native Voxel Domain</p> | |
| <p class="paper-authors">Shuang Wu, Jiachen Qian, Youtian Lin, Feihu Zhang, | |
| Philip Torr, Yao Yao, and Xun Cao</p> | |
| <nav class="paper-links" aria-label="Project links"> | |
| <a class="paper-link" href="https://dl.acm.org/doi/10.1145/3842524" | |
| target="_blank" rel="noopener noreferrer">Paper ↗</a> | |
| <a class="paper-link" href="https://github.com/DreamTechAI/Direct3D-V3" | |
| target="_blank" rel="noopener noreferrer">GitHub ↗</a> | |
| </nav> | |
| </div> | |
| </header> | |
| """, elem_id="paper-header") | |
| with gr.Column(elem_id="examples-strip"): | |
| gr.HTML(""" | |
| <div class="examples-heading"> | |
| <h2>Examples</h2> | |
| <p role="note">For a more detailed mesh, we recommend using GPT Image 2.5 or | |
| <a href="https://github.com/EnVision-Research/Lotus-2" target="_blank" rel="noopener noreferrer">Lotus-2</a> | |
| to convert your input image into a surface normal map or a white clay render | |
| that preserves fine geometric details.</p> | |
| </div> | |
| """) | |
| gallery = gr.Gallery( | |
| thumbnails, label="Paired examples", show_label=False, columns=6, rows=2, | |
| height="auto", object_fit="contain", allow_preview=False, interactive=False, | |
| format="webp", elem_id="example-gallery", **gallery_tools, | |
| ) | |
| with gr.Row(equal_height=False, elem_id="generation-workspace"): | |
| with gr.Column(scale=5, min_width=300, variant="panel", elem_classes="work-panel"): | |
| gr.Markdown("## Inputs") | |
| with gr.Row(): | |
| image = gr.Image( | |
| type="pil", label="Reference image", image_mode="RGBA", sources=["upload"], | |
| height=192, min_width=125, | |
| ) | |
| foreground = gr.Image( | |
| type="pil", label="Foreground", image_mode="RGBA", format="png", | |
| interactive=False, height=192, min_width=125, | |
| ) | |
| coarse = gr.File( | |
| file_types=[".obj", ".glb", ".ply"], type="filepath", label="Aligned coarse mesh", | |
| height=116, | |
| ) | |
| # Choices are strings so the public API schema matches what the component | |
| # accepts; the pipeline call sites coerce with _as_stage(). | |
| resolution = gr.Radio( | |
| [str(stage) for stage in RESOLUTIONS], value="1024", label="Output resolution", | |
| ) | |
| with gr.Accordion("Sampling and mesh options", open=False): | |
| stage_inputs = [] | |
| for stage in RESOLUTIONS: | |
| steps, cfg = DEFAULTS[stage] | |
| with gr.Row(): | |
| stage_inputs.append(gr.Number( | |
| label=f"{stage} steps", precision=0, minimum=1, | |
| maximum=9 if stage == 1536 else 30, | |
| value=STAGE_STEPS[stage], | |
| )) | |
| stage_inputs.append(gr.Number(label=f"{stage} CFG", minimum=0, value=cfg)) | |
| with gr.Row(): | |
| seed = gr.Number(value=2026, precision=0, label="Seed") | |
| max_tokens = gr.Number( | |
| value=DEFAULT_MAX_TOKENS, minimum=1, precision=0, label="Maximum tokens per stage", | |
| info="If you run out of GPU memory, try a lower value. The mesh may lose fine surface detail.", | |
| ) | |
| simplify = gr.Checkbox(value=True, label="Simplify mesh") | |
| target_faces = gr.Slider( | |
| 100_000, 5_000_000, step=1, value=DEFAULT_TARGET_FACES, label="Target triangles", | |
| ) | |
| simplify.change( | |
| lambda enabled: gr.Slider(interactive=enabled), inputs=simplify, outputs=target_faces, | |
| queue=False, show_progress="hidden", **ui_event, | |
| ) | |
| submit = gr.Button("Generate", variant="primary", elem_id="generate-mesh") | |
| stage_progress = gr.HTML("", elem_id="sampling-progress") | |
| with gr.Column(scale=7, min_width=300, variant="panel", elem_classes="work-panel"): | |
| gr.Markdown("## Mesh viewer") | |
| gr.HTML("<direct3d-mesh-viewer></direct3d-mesh-viewer>", elem_id="mesh-viewer") | |
| download = gr.File(label="Download GLB", interactive=False, height=80) | |
| demo.load(fn=None, js=VIEWER.read_text(encoding="utf-8"), **ui_event) | |
| download.change( | |
| fn=None, inputs=download, js="""(file) => { | |
| const viewer = document.querySelector("direct3d-mesh-viewer"); | |
| if (viewer) viewer.setAttribute("src", file?.url || ""); | |
| }""", **ui_event, | |
| ) | |
| results = [foreground, download, stage_progress] | |
| job = submit.click( | |
| generate, [image, coarse, resolution, seed, simplify, target_faces, max_tokens, *stage_inputs], | |
| results, concurrency_limit=1, api_name="generate", show_progress_on=[download], | |
| ) | |
| gallery.select( | |
| load_example, outputs=[image, coarse, *results], cancels=[job], | |
| queue=False, show_progress="hidden", **ui_event, | |
| ) | |
| gr.on( | |
| triggers=[image.upload, image.clear, coarse.upload, coarse.clear], | |
| fn=lambda: (None, None, ""), outputs=results, cancels=[job], | |
| queue=False, show_progress="hidden", **ui_event, | |
| ) | |
| return demo | |
| if __name__ == "__main__": | |
| build_demo().queue().launch( | |
| server_name="0.0.0.0", server_port=7860, | |
| allowed_paths=[str(TEASER)], | |
| theme=gr.themes.Citrus(), | |
| css=".dark .gradio-container { color: var(--body-text-color); }", | |
| css_paths=[STYLES], | |
| mcp_server=True, | |
| ) | |