"""GaussianRenderFormer — transformer-based rendering of 3D Gaussian splats. Paper: arXiv:2611.05593 (Valencia Levy & Benaim, 2026). Checkpoint: shahafvl/gaussianformer (MIT) · Code: github.com/SVLwoof/gaussianformer. Point a camera at a 3D Gaussian Splatting scene and a transformer renders the view — each Gaussian is one scene token, no rasterizer, no per-scene optimization. """ import os os.environ.setdefault("ATTN_IMPL", "sdpa") # pure-PyTorch attention (flash_attn is optional upstream) import spaces # must precede torch import math import time import traceback from pathlib import Path import gradio as gr import numpy as np import torch from gaussianformer import GaussianFormerRenderingPipeline from gaussianformer.splat import normalize, read_ply, to_tensor from gaussianformer.utils.cameras import look_at MODEL_ID = "shahafvl/gaussianformer" FOV = 45.0 RESOLUTION = 512 MAX_GAUSSIANS = 20_000 # the model's training size; larger inputs are pruned by importance HERE = Path(__file__).parent SCENE_FILES = {"Gem": "examples/gem.ply", "Vase": "examples/vase.ply", "Planet": "examples/planet.ply"} print("loading model…") PIPE = GaussianFormerRenderingPipeline.from_pretrained(MODEL_ID).to("cuda") PIPE.model.eval() print("model ready") # Bundled example scenes, pre-parsed on the CPU: name -> [N, 14] float32 tensor. SCENES: dict[str, torch.Tensor] = { name: to_tensor(normalize(read_ply(HERE / rel), "y")) for name, rel in SCENE_FILES.items() } for name, t in SCENES.items(): print(f"scene {name!r}: {t.shape[0]:,} Gaussians") # --------------------------------------------------------------------------- # # camera + splat helpers (CPU, main process) # --------------------------------------------------------------------------- # def camera_c2w(azimuth: float, elevation: float, distance: float) -> torch.Tensor: """Orbit camera around the origin -> c2w [1, 4, 4], Blender convention (-Z fwd, +Y up).""" az, el = math.radians(azimuth), math.radians(elevation) eye = np.array( [distance * math.cos(el) * math.cos(az), distance * math.sin(el), distance * math.cos(el) * math.sin(az)], np.float32) return torch.from_numpy(look_at(eye, np.zeros(3, np.float32), np.array([0, 1, 0], np.float32))[None]) def _prune_score(g: dict) -> np.ndarray: """Importance ~ opacity x projected area, adapted from LightGaussian as in the reference pipeline, but without gsplat: area per view ~ (max scale x focal / depth)^2.""" focal = 0.5 * RESOLUTION / math.tan(math.radians(FOV) / 2) max_scale = g["scales"].max(-1) score = np.zeros(len(g["means"]), np.float64) for az in range(0, 360, 45): # 8-view orbit, as the reference scores a 64-view one c2w = camera_c2w(az, 20.0, 1.7)[0].numpy() depth = np.clip((c2w[:3, 3] - g["means"]) @ c2w[:3, 2], 1e-3, None) score += g["opacities"] * (max_scale * focal / depth) ** 2 return score * max_scale ** 0.1 def load_gaussians(path: str, up: str) -> tuple[torch.Tensor, dict]: """PLY -> ([N, 14] CPU tensor, stats dict). Prunes to MAX_GAUSSIANS by importance when needed.""" try: g = normalize(read_ply(path), up) except (ValueError, KeyError, IndexError): raise gr.Error( "This does not look like a 3D Gaussian Splatting .ply — the renderer needs the " "raw optimizer fields scale_0..2, rot_0..3, f_dc_0..2, opacity (as written by " "every 3DGS implementation)." ) if g is None: raise gr.Error("Could not normalize this splat: all Gaussians sit at (or near) one point.") n = len(g["means"]) if n > MAX_GAUSSIANS: idx = np.argsort(_prune_score(g))[::-1][:MAX_GAUSSIANS] g = {k: v[idx] for k, v in g.items()} g14 = to_tensor(g) return g14, {"n": n, "kept": g14.shape[0]} def _info_line(stats: dict, seconds: float) -> str: line = f"**{stats['kept']:,} Gaussians** · {RESOLUTION}×{RESOLUTION} · **{seconds:.2f} s** on the GPU" if stats["n"] > stats["kept"]: line += ( f" — pruned from {stats['n']:,} by importance. Large pruned splats can turn dark: the" " reference pipeline also fine-tunes the kept Gaussians, which does not fit a demo." ) return line # --------------------------------------------------------------------------- # # GPU worker (ZeroGPU) # --------------------------------------------------------------------------- # def _gpu_duration(*_args, **_kwargs) -> int: """One 512 px view of <=20k Gaussians takes ~0.3-1.3 s measured on this hardware. The reservation is padded to cover a cold worker after a reboot (780 MB of weights must be streamed in before the first render, which also pays an autotune + CUDA init).""" return 15 @spaces.GPU(duration=_gpu_duration) def _gpu_render(gaussians: torch.Tensor, c2w: torch.Tensor) -> tuple[np.ndarray, float]: """[N, 14] + [1, 4, 4] (CPU tensors) -> (uint8 RGB image [512, 512, 3], GPU seconds).""" started = time.perf_counter() with torch.no_grad(): images = PIPE(gaussians.to("cuda"), c2w.to("cuda"), fov=FOV, resolution=RESOLUTION) torch.cuda.synchronize() seconds = time.perf_counter() - started img = np.clip(images[0].float().cpu().numpy(), 0.0, 1.0) return (img * 255.0 + 0.5).astype(np.uint8), seconds # --------------------------------------------------------------------------- # # handlers # --------------------------------------------------------------------------- # def render_example( scene: str = "Gem", azimuth: float = 35.0, elevation: float = 20.0, distance: float = 1.7, ) -> tuple[np.ndarray, str]: """Render one of the bundled example splats from an orbit camera. Args: scene: Which bundled scene to render. azimuth: Orbit the camera around the object, in degrees. elevation: Camera height above the object's equator, in degrees. distance: Camera distance from the object's center (trained on 1.15–2.45). """ if scene not in SCENES: raise gr.Error(f"Unknown scene {scene!r}.") try: img, seconds = _gpu_render(SCENES[scene], camera_c2w(azimuth, elevation, distance)) return img, _info_line({"n": SCENES[scene].shape[0], "kept": SCENES[scene].shape[0]}, seconds) except gr.Error: raise except Exception: traceback.print_exc() raise gr.Error("Rendering failed — see this Space's logs for the traceback.") def render_upload( ply_file, up: str = "y", azimuth: float = 35.0, elevation: float = 20.0, distance: float = 1.7, ) -> tuple[np.ndarray, str]: """Render your own 3D Gaussian Splatting .ply from an orbit camera. Args: ply_file: Path to a 3DGS .ply (log scales, wxyz quaternions, SH colors, opacity logits). up: The file's up axis: 'y', 'z' (e.g. Objaverse-style splats) or '-y' (COLMAP captures). azimuth: Orbit the camera around the object, in degrees. elevation: Camera height above the object's equator, in degrees. distance: Camera distance from the object's center (trained on 1.15–2.45). """ if not ply_file: raise gr.Error("Upload a 3D Gaussian Splatting .ply file first.") path = ply_file if isinstance(ply_file, str) else getattr(ply_file, "name", None) try: g14, stats = load_gaussians(path, up) img, seconds = _gpu_render(g14, camera_c2w(azimuth, elevation, distance)) return img, _info_line(stats, seconds) except gr.Error: raise except Exception: traceback.print_exc() raise gr.Error("Could not render this file — see this Space's logs for the traceback.") # --------------------------------------------------------------------------- # # UI # --------------------------------------------------------------------------- # CSS = "#col-container { max-width: 1180px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); }" with gr.Blocks() as demo: with gr.Column(elem_id="col-container"): gr.Markdown( "# GaussianRenderFormer\n" "**Transformer-based rendering of 3D Gaussian splats** — " "[arXiv:2611.05593](https://arxiv.org/abs/2611.05593) · " "[code](https://github.com/SVLwoof/gaussianformer) · " "[project page](https://svlwoof.github.io/gaussianrenderformer/)\n\n" "Each Gaussian becomes one scene token; a view decoder cross-attends over 4-px image " "patches and renders the view — **no rasterizer, no per-scene optimization**. " "Point the camera and render." ) with gr.Tabs(): with gr.Tab("Example scenes"): with gr.Row(): with gr.Column(scale=1): scene = gr.Dropdown(choices=list(SCENES), value="Gem", label="Scene (synthetic splats authored for this demo)") azimuth = gr.Slider(0, 360, value=35, step=1, label="Azimuth (°)") elevation = gr.Slider(-60, 60, value=20, step=1, label="Elevation (°)") distance = gr.Slider(1.15, 2.45, value=1.7, step=0.01, label="Distance (trained on 1.15–2.45)") ex_run = gr.Button("Render", variant="primary") with gr.Column(scale=1): ex_img = gr.Image(label="GaussianFormer render", type="numpy", height=512) ex_info = gr.Markdown() gr.Examples( examples=[ ["Gem", 35, 20, 1.7], ["Vase", 210, -5, 1.45], ["Planet", 120, 35, 2.45], ], inputs=[scene, azimuth, elevation, distance], outputs=[ex_img, ex_info], fn=render_example, cache_examples=True, cache_mode="lazy", label="Try a scene", ) with gr.Tab("Your own splat"): gr.Markdown( "Upload a 3D Gaussian Splatting `.ply` (raw optimizer parameters: log scales, " "wxyz quaternions, SH colors, opacity logits — the format every 3DGS " "implementation writes). Splats larger than 20k Gaussians are pruned by " "importance, as the model was trained on ~20k-Gaussian objects." ) with gr.Row(): with gr.Column(scale=1): ply_file = gr.File(label="3DGS .ply", file_types=[".ply"], type="filepath") up_axis = gr.Radio(choices=["y", "z", "-y"], value="y", label="File's up axis") azimuth_u = gr.Slider(0, 360, value=35, step=1, label="Azimuth (°)") elevation_u = gr.Slider(-60, 60, value=20, step=1, label="Elevation (°)") distance_u = gr.Slider(1.15, 2.45, value=1.7, step=0.01, label="Distance (trained on 1.15–2.45)") up_run = gr.Button("Render", variant="primary") with gr.Column(scale=1): up_img = gr.Image(label="GaussianFormer render", type="numpy", height=512) up_info = gr.Markdown() gr.Examples( examples=[[str(HERE / "examples" / "gem.ply")]], inputs=[ply_file], outputs=[up_img, up_info], fn=render_upload, cache_examples=True, cache_mode="lazy", label="Try the upload path", ) gr.Markdown( "**Limitations** (from the model card): trained on isolated single objects centered, " "Y-up and scaled into [-0.45, 0.45]³ at 512 px, 45° FOV and 20k Gaussians — denser " "inputs lose quality, colors are view-independent, and fine detail renders softer " "than a rasterizer. MIT license, for the weights as for the code." ) ex_run.click(render_example, [scene, azimuth, elevation, distance], [ex_img, ex_info], api_name="render_scene") up_run.click(render_upload, [ply_file, up_axis, azimuth_u, elevation_u, distance_u], [up_img, up_info], api_name="render_splat") if __name__ == "__main__": demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True, show_error=True)