"""Direct3D-V3: latent-free diffusion transformers in the native voxel domain. ZeroGPU port of the repository's reference demo (app.py in DreamTechAI/Direct3D-V3): a reference image plus an aligned coarse mesh are refined into a high-resolution surface mesh by a three-stage voxel-domain diffusion cascade (512 -> 1024 -> 1536). """ import os # Fragmentation guard for the transient spikes of the voxel rasterizer. os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import spaces # noqa: E402 (must precede every CUDA-touching import) import time from concurrent.futures import CancelledError, ThreadPoolExecutor from pathlib import Path from queue import Queue from tempfile import TemporaryDirectory from threading import Event import gradio as gr import torch from PIL import Image from direct3d import Direct3DV3Pipeline from direct3d.models.image_encoder import DINOv3FeatureExtractor from direct3d.pipeline import DEFAULT_MAX_TOKENS, DEFAULT_MODEL_ID, DEFAULT_TARGET_FACES, RESOLUTIONS from direct3d.utils.image import BackgroundRemover ASSETS = Path(__file__).resolve().parent / "assets" TEASER = ASSETS / "demo" / "paper-teaser.webp" STYLES = ASSETS / "demo" / "app.css" VIEWER = ASSETS / "demo" / "mesh-viewer.js" # The image encoder configured in the model card (`facebook/dinov3-...`) is # gated on the Hub; this byte-identical mirror (same safetensors sha256) keeps # the demo runnable anonymously. Everything else is loaded from the official # weights repo. ENCODER_ID = "cocktailpeanut/dinov3-vith16plus-pretrain-lvd1689m" pipeline = Direct3DV3Pipeline.from_pretrained(DEFAULT_MODEL_ID, image_encoder=ENCODER_ID) pipeline.to("cuda") # Pre-load the two auxiliary CPU models at startup so forked GPU workers # inherit them instead of re-downloading from the Hub on a cold request. pipeline._encoder = DINOv3FeatureExtractor( **pipeline._encoder_options, dtype=pipeline.dtype, local_files_only=pipeline.local_files_only ) pipeline._remover = BackgroundRemover(pipeline.local_files_only) DEFAULTS = pipeline.sampling_defaults # The authors' config uses 15 steps at every stage. A full 15-step 512->1536 # cascade measures ~281 s on zero-a10g, which leaves no headroom under # ZeroGPU's 300 s per-call reservation limit, so the final refinement stage is # budgeted at 8 steps (~200 s) and capped at 9. STAGE_STEPS = {512: DEFAULTS[512][0], 1024: DEFAULTS[1024][0], 1536: 8} # Measured on zero-a10g: per-stage wall-clock fit (fixed seconds, seconds per # diffusion step) for one example, plus the shared base cost of background # removal, voxelisation, marching cubes and GLB export. _STAGE_COST = {512: (4.0, 0.60), 1024: (9.7, 3.76), 1536: (24.0, 11.6)} _BASE_SECONDS = 4.0 _MIN_DURATION = 30 _MAX_DURATION = 300 # ZeroGPU refuses longer reservations for free accounts. def _as_stage(value, default): """Coerce a resolution / step value to int (the API client sends radio choices as strings).""" if value is None: return default try: return int(value) except (TypeError, ValueError): return default def _estimate(image, coarse, resolution, seed, simplify, target_faces, max_tokens, *args, **kwargs): """Reserve GPU seconds from the stage plan, sized to measured zero-a10g runs.""" resolution = _as_stage(resolution, 1024) total = _BASE_SECONDS for stage, steps in zip(RESOLUTIONS, args[0::2]): if stage > resolution: break fixed, per_step = _STAGE_COST[stage] total += fixed + per_step * _as_stage(steps, STAGE_STEPS[stage]) return min(_MAX_DURATION, max(_MIN_DURATION, round(total * 1.4))) def build_demo(): examples = [] thumbnails = [] meshes = sorted((ASSETS / "coarse_meshes").glob("*.obj"), key=lambda path: int(path.stem)) image_dir = ASSETS / "input_imgs" for normal_map in (False, True): for mesh in meshes: if normal_map: reference = image_dir / f"{mesh.stem}_normal.png" else: reference, = image_dir.glob(f"{mesh.stem}.*") with Image.open(reference) as source: thumb = source.convert("RGB") thumb.thumbnail((256, 192), Image.Resampling.LANCZOS) thumbnails.append(thumb) examples.append((reference, mesh)) def load_example(event: gr.SelectData): reference, mesh = examples[event.index] return str(reference), str(mesh), None, None, "" def stage_bars(totals, completed): rows = [] for stage, total in totals.items(): done = completed.get(stage, 0) rows.append( f'' ) return '
" @spaces.GPU(duration=_estimate) def generate( image, coarse, resolution, seed, simplify, target_faces, max_tokens, steps_512, cfg_512, steps_1024, cfg_1024, steps_1536, cfg_1536, progress=gr.Progress(), ): """Refine an aligned coarse mesh into a detailed 3D mesh guided by a reference image. Args: image: reference image (photo, render, or surface normal map) guiding surface detail. coarse: coarse mesh (.obj / .glb / .ply) aligned with the reference camera. resolution: output voxel resolution; one of 512, 1024, or 1536. seed: sampling seed; identical inputs and seed reproduce the same mesh. simplify: decimate the exported mesh to target_faces with pyfqmr. target_faces: triangle budget for the exported mesh when simplify is on. max_tokens: per-stage sparse token cap; lower it when the GPU runs out of memory. steps_512, steps_1024, steps_1536: diffusion steps per cascade stage. The 1536 stage is capped at 9 steps to fit ZeroGPU's 300 s per-call budget. cfg_512, cfg_1024, cfg_1536: classifier-free guidance per cascade stage. Yields: (foreground cutout, downloadable GLB, stage progress HTML). Intermediate yields keep the previous mesh in the viewer; the final yield carries it. """ started = time.perf_counter() if image is None or coarse is None: raise gr.Error("Select an example or upload both a reference image and an aligned coarse mesh.") stage_steps = {} stage_cfg = {} resolution = _as_stage(resolution, 1024) for stage, count, cfg in ( (512, steps_512, cfg_512), (1024, steps_1024, cfg_1024), (1536, steps_1536, cfg_1536), ): if stage > resolution: break stage_steps[stage] = _as_stage(count, STAGE_STEPS[stage]) if cfg is not None: stage_cfg[stage] = cfg completed = {} yield None, None, stage_bars(stage_steps, completed) progress(0, desc="Removing image background") cutout = pipeline.remove_background(image) yield cutout, gr.skip(), gr.skip() events = Queue() cancelled = Event() def report(stage, done, total): if cancelled.is_set(): raise CancelledError() events.put((stage, done, total)) with ThreadPoolExecutor(max_workers=1) as executor: task = executor.submit( pipeline, cutout, coarse, background_removed=True, resolution=resolution, num_inference_steps=stage_steps, guidance_scale=stage_cfg, seed=int(seed), target_faces=int(target_faces) if simplify else None, max_tokens=max_tokens, on_sampling_step=report, ) task.add_done_callback(lambda _: events.put(None)) try: while (event := events.get()) is not None: stage, done, total = event completed[stage] = done if done == 0: progress(None, desc=f"Preparing {stage}³ geometry") elif done == total: progress(None, desc=f"Extracting {stage}³ mesh") else: progress((done, total), desc=f"Sampling {stage}³") yield gr.skip(), gr.skip(), stage_bars(stage_steps, completed) mesh = task.result() finally: cancelled.set() progress(None, desc="Exporting GLB") with TemporaryDirectory(prefix="direct3d-") as directory: output = Path(directory) / "generated.glb" mesh.export(output) cached = download.move_resource_to_block_cache(output) print( f"[direct3d] resolution={resolution} seed={int(seed)} " f"elapsed={time.perf_counter() - started:.1f}s", flush=True, ) yield cutout, cached, stage_bars(stage_steps, completed) # Gradio 6 moved custom styles from Blocks to launch and changed these kwargs. gallery_tools = {"buttons": []} ui_event = {"api_visibility": "private"} with gr.Blocks(title="Direct3D-V3", delete_cache=(3600, 3600)) as demo: gr.HTML(f"""ACM Transactions on Graphics · SIGGRAPH Asia 2026
Latent-Free Diffusion Transformers in the Native Voxel Domain
For a more detailed mesh, we recommend using GPT Image 2.5 or Lotus-2 to convert your input image into a surface normal map or a white clay render that preserves fine geometric details.