Spaces:
Running on Zero
Running on Zero
Download app.py from hugging-apps/gaussianrenderformer: direct link, hf CLI and curl.
- Browser
- Download file 12.7 kB
-
https://huggingface.co/spaces/hugging-apps/gaussianrenderformer/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-apps/gaussianrenderformer/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-apps/gaussianrenderformer/resolve/main/app.py
12.7 kB
| """GaussianRenderFormer — transformer-based rendering of 3D Gaussian splats. | |
| Paper: arXiv:2611.05593 (Valencia Levy & Benaim, 2026). | |
| Checkpoint: shahafvl/gaussianformer (MIT) · Code: github.com/SVLwoof/gaussianformer. | |
| Point a camera at a 3D Gaussian Splatting scene and a transformer renders the | |
| view — each Gaussian is one scene token, no rasterizer, no per-scene | |
| optimization. | |
| """ | |
| import os | |
| os.environ.setdefault("ATTN_IMPL", "sdpa") # pure-PyTorch attention (flash_attn is optional upstream) | |
| import spaces # must precede torch | |
| import math | |
| import time | |
| import traceback | |
| from pathlib import Path | |
| import gradio as gr | |
| import numpy as np | |
| import torch | |
| from gaussianformer import GaussianFormerRenderingPipeline | |
| from gaussianformer.splat import normalize, read_ply, to_tensor | |
| from gaussianformer.utils.cameras import look_at | |
| MODEL_ID = "shahafvl/gaussianformer" | |
| FOV = 45.0 | |
| RESOLUTION = 512 | |
| MAX_GAUSSIANS = 20_000 # the model's training size; larger inputs are pruned by importance | |
| HERE = Path(__file__).parent | |
| SCENE_FILES = {"Gem": "examples/gem.ply", "Vase": "examples/vase.ply", "Planet": "examples/planet.ply"} | |
| print("loading model…") | |
| PIPE = GaussianFormerRenderingPipeline.from_pretrained(MODEL_ID).to("cuda") | |
| PIPE.model.eval() | |
| print("model ready") | |
| # Bundled example scenes, pre-parsed on the CPU: name -> [N, 14] float32 tensor. | |
| SCENES: dict[str, torch.Tensor] = { | |
| name: to_tensor(normalize(read_ply(HERE / rel), "y")) for name, rel in SCENE_FILES.items() | |
| } | |
| for name, t in SCENES.items(): | |
| print(f"scene {name!r}: {t.shape[0]:,} Gaussians") | |
| # --------------------------------------------------------------------------- # | |
| # camera + splat helpers (CPU, main process) | |
| # --------------------------------------------------------------------------- # | |
| def camera_c2w(azimuth: float, elevation: float, distance: float) -> torch.Tensor: | |
| """Orbit camera around the origin -> c2w [1, 4, 4], Blender convention (-Z fwd, +Y up).""" | |
| az, el = math.radians(azimuth), math.radians(elevation) | |
| eye = np.array( | |
| [distance * math.cos(el) * math.cos(az), distance * math.sin(el), | |
| distance * math.cos(el) * math.sin(az)], np.float32) | |
| return torch.from_numpy(look_at(eye, np.zeros(3, np.float32), np.array([0, 1, 0], np.float32))[None]) | |
| def _prune_score(g: dict) -> np.ndarray: | |
| """Importance ~ opacity x projected area, adapted from LightGaussian as in the reference | |
| pipeline, but without gsplat: area per view ~ (max scale x focal / depth)^2.""" | |
| focal = 0.5 * RESOLUTION / math.tan(math.radians(FOV) / 2) | |
| max_scale = g["scales"].max(-1) | |
| score = np.zeros(len(g["means"]), np.float64) | |
| for az in range(0, 360, 45): # 8-view orbit, as the reference scores a 64-view one | |
| c2w = camera_c2w(az, 20.0, 1.7)[0].numpy() | |
| depth = np.clip((c2w[:3, 3] - g["means"]) @ c2w[:3, 2], 1e-3, None) | |
| score += g["opacities"] * (max_scale * focal / depth) ** 2 | |
| return score * max_scale ** 0.1 | |
| def load_gaussians(path: str, up: str) -> tuple[torch.Tensor, dict]: | |
| """PLY -> ([N, 14] CPU tensor, stats dict). Prunes to MAX_GAUSSIANS by importance when needed.""" | |
| try: | |
| g = normalize(read_ply(path), up) | |
| except (ValueError, KeyError, IndexError): | |
| raise gr.Error( | |
| "This does not look like a 3D Gaussian Splatting .ply — the renderer needs the " | |
| "raw optimizer fields scale_0..2, rot_0..3, f_dc_0..2, opacity (as written by " | |
| "every 3DGS implementation)." | |
| ) | |
| if g is None: | |
| raise gr.Error("Could not normalize this splat: all Gaussians sit at (or near) one point.") | |
| n = len(g["means"]) | |
| if n > MAX_GAUSSIANS: | |
| idx = np.argsort(_prune_score(g))[::-1][:MAX_GAUSSIANS] | |
| g = {k: v[idx] for k, v in g.items()} | |
| g14 = to_tensor(g) | |
| return g14, {"n": n, "kept": g14.shape[0]} | |
| def _info_line(stats: dict, seconds: float) -> str: | |
| line = f"**{stats['kept']:,} Gaussians** · {RESOLUTION}×{RESOLUTION} · **{seconds:.2f} s** on the GPU" | |
| if stats["n"] > stats["kept"]: | |
| line += ( | |
| f" — pruned from {stats['n']:,} by importance. Large pruned splats can turn dark: the" | |
| " reference pipeline also fine-tunes the kept Gaussians, which does not fit a demo." | |
| ) | |
| return line | |
| # --------------------------------------------------------------------------- # | |
| # GPU worker (ZeroGPU) | |
| # --------------------------------------------------------------------------- # | |
| def _gpu_duration(*_args, **_kwargs) -> int: | |
| """One 512 px view of <=20k Gaussians takes ~0.3-1.3 s measured on this hardware. The | |
| reservation is padded to cover a cold worker after a reboot (780 MB of weights must be | |
| streamed in before the first render, which also pays an autotune + CUDA init).""" | |
| return 15 | |
| def _gpu_render(gaussians: torch.Tensor, c2w: torch.Tensor) -> tuple[np.ndarray, float]: | |
| """[N, 14] + [1, 4, 4] (CPU tensors) -> (uint8 RGB image [512, 512, 3], GPU seconds).""" | |
| started = time.perf_counter() | |
| with torch.no_grad(): | |
| images = PIPE(gaussians.to("cuda"), c2w.to("cuda"), fov=FOV, resolution=RESOLUTION) | |
| torch.cuda.synchronize() | |
| seconds = time.perf_counter() - started | |
| img = np.clip(images[0].float().cpu().numpy(), 0.0, 1.0) | |
| return (img * 255.0 + 0.5).astype(np.uint8), seconds | |
| # --------------------------------------------------------------------------- # | |
| # handlers | |
| # --------------------------------------------------------------------------- # | |
| def render_example( | |
| scene: str = "Gem", | |
| azimuth: float = 35.0, | |
| elevation: float = 20.0, | |
| distance: float = 1.7, | |
| ) -> tuple[np.ndarray, str]: | |
| """Render one of the bundled example splats from an orbit camera. | |
| Args: | |
| scene: Which bundled scene to render. | |
| azimuth: Orbit the camera around the object, in degrees. | |
| elevation: Camera height above the object's equator, in degrees. | |
| distance: Camera distance from the object's center (trained on 1.15–2.45). | |
| """ | |
| if scene not in SCENES: | |
| raise gr.Error(f"Unknown scene {scene!r}.") | |
| try: | |
| img, seconds = _gpu_render(SCENES[scene], camera_c2w(azimuth, elevation, distance)) | |
| return img, _info_line({"n": SCENES[scene].shape[0], "kept": SCENES[scene].shape[0]}, seconds) | |
| except gr.Error: | |
| raise | |
| except Exception: | |
| traceback.print_exc() | |
| raise gr.Error("Rendering failed — see this Space's logs for the traceback.") | |
| def render_upload( | |
| ply_file, | |
| up: str = "y", | |
| azimuth: float = 35.0, | |
| elevation: float = 20.0, | |
| distance: float = 1.7, | |
| ) -> tuple[np.ndarray, str]: | |
| """Render your own 3D Gaussian Splatting .ply from an orbit camera. | |
| Args: | |
| ply_file: Path to a 3DGS .ply (log scales, wxyz quaternions, SH colors, opacity logits). | |
| up: The file's up axis: 'y', 'z' (e.g. Objaverse-style splats) or '-y' (COLMAP captures). | |
| azimuth: Orbit the camera around the object, in degrees. | |
| elevation: Camera height above the object's equator, in degrees. | |
| distance: Camera distance from the object's center (trained on 1.15–2.45). | |
| """ | |
| if not ply_file: | |
| raise gr.Error("Upload a 3D Gaussian Splatting .ply file first.") | |
| path = ply_file if isinstance(ply_file, str) else getattr(ply_file, "name", None) | |
| try: | |
| g14, stats = load_gaussians(path, up) | |
| img, seconds = _gpu_render(g14, camera_c2w(azimuth, elevation, distance)) | |
| return img, _info_line(stats, seconds) | |
| except gr.Error: | |
| raise | |
| except Exception: | |
| traceback.print_exc() | |
| raise gr.Error("Could not render this file — see this Space's logs for the traceback.") | |
| # --------------------------------------------------------------------------- # | |
| # UI | |
| # --------------------------------------------------------------------------- # | |
| CSS = "#col-container { max-width: 1180px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); }" | |
| with gr.Blocks() as demo: | |
| with gr.Column(elem_id="col-container"): | |
| gr.Markdown( | |
| "# GaussianRenderFormer\n" | |
| "**Transformer-based rendering of 3D Gaussian splats** — " | |
| "[arXiv:2611.05593](https://arxiv.org/abs/2611.05593) · " | |
| "[code](https://github.com/SVLwoof/gaussianformer) · " | |
| "[project page](https://svlwoof.github.io/gaussianrenderformer/)\n\n" | |
| "Each Gaussian becomes one scene token; a view decoder cross-attends over 4-px image " | |
| "patches and renders the view — **no rasterizer, no per-scene optimization**. " | |
| "Point the camera and render." | |
| ) | |
| with gr.Tabs(): | |
| with gr.Tab("Example scenes"): | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| scene = gr.Dropdown(choices=list(SCENES), value="Gem", | |
| label="Scene (synthetic splats authored for this demo)") | |
| azimuth = gr.Slider(0, 360, value=35, step=1, label="Azimuth (°)") | |
| elevation = gr.Slider(-60, 60, value=20, step=1, label="Elevation (°)") | |
| distance = gr.Slider(1.15, 2.45, value=1.7, step=0.01, | |
| label="Distance (trained on 1.15–2.45)") | |
| ex_run = gr.Button("Render", variant="primary") | |
| with gr.Column(scale=1): | |
| ex_img = gr.Image(label="GaussianFormer render", type="numpy", height=512) | |
| ex_info = gr.Markdown() | |
| gr.Examples( | |
| examples=[ | |
| ["Gem", 35, 20, 1.7], | |
| ["Vase", 210, -5, 1.45], | |
| ["Planet", 120, 35, 2.45], | |
| ], | |
| inputs=[scene, azimuth, elevation, distance], | |
| outputs=[ex_img, ex_info], | |
| fn=render_example, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| label="Try a scene", | |
| ) | |
| with gr.Tab("Your own splat"): | |
| gr.Markdown( | |
| "Upload a 3D Gaussian Splatting `.ply` (raw optimizer parameters: log scales, " | |
| "wxyz quaternions, SH colors, opacity logits — the format every 3DGS " | |
| "implementation writes). Splats larger than 20k Gaussians are pruned by " | |
| "importance, as the model was trained on ~20k-Gaussian objects." | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| ply_file = gr.File(label="3DGS .ply", file_types=[".ply"], type="filepath") | |
| up_axis = gr.Radio(choices=["y", "z", "-y"], value="y", label="File's up axis") | |
| azimuth_u = gr.Slider(0, 360, value=35, step=1, label="Azimuth (°)") | |
| elevation_u = gr.Slider(-60, 60, value=20, step=1, label="Elevation (°)") | |
| distance_u = gr.Slider(1.15, 2.45, value=1.7, step=0.01, | |
| label="Distance (trained on 1.15–2.45)") | |
| up_run = gr.Button("Render", variant="primary") | |
| with gr.Column(scale=1): | |
| up_img = gr.Image(label="GaussianFormer render", type="numpy", height=512) | |
| up_info = gr.Markdown() | |
| gr.Examples( | |
| examples=[[str(HERE / "examples" / "gem.ply")]], | |
| inputs=[ply_file], | |
| outputs=[up_img, up_info], | |
| fn=render_upload, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| label="Try the upload path", | |
| ) | |
| gr.Markdown( | |
| "**Limitations** (from the model card): trained on isolated single objects centered, " | |
| "Y-up and scaled into [-0.45, 0.45]³ at 512 px, 45° FOV and 20k Gaussians — denser " | |
| "inputs lose quality, colors are view-independent, and fine detail renders softer " | |
| "than a rasterizer. MIT license, for the weights as for the code." | |
| ) | |
| ex_run.click(render_example, [scene, azimuth, elevation, distance], [ex_img, ex_info], | |
| api_name="render_scene") | |
| up_run.click(render_upload, [ply_file, up_axis, azimuth_u, elevation_u, distance_u], | |
| [up_img, up_info], api_name="render_splat") | |
| if __name__ == "__main__": | |
| demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True, show_error=True) | |