File size: 12,713 Bytes
87446a8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
eceb8db
 
 
 
87446a8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
eceb8db
87446a8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
eceb8db
87446a8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
eceb8db
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
"""GaussianRenderFormer — transformer-based rendering of 3D Gaussian splats.

Paper: arXiv:2611.05593 (Valencia Levy & Benaim, 2026).
Checkpoint: shahafvl/gaussianformer (MIT) · Code: github.com/SVLwoof/gaussianformer.

Point a camera at a 3D Gaussian Splatting scene and a transformer renders the
view — each Gaussian is one scene token, no rasterizer, no per-scene
optimization.
"""
import os

os.environ.setdefault("ATTN_IMPL", "sdpa")  # pure-PyTorch attention (flash_attn is optional upstream)

import spaces  # must precede torch

import math
import time
import traceback
from pathlib import Path

import gradio as gr
import numpy as np
import torch

from gaussianformer import GaussianFormerRenderingPipeline
from gaussianformer.splat import normalize, read_ply, to_tensor
from gaussianformer.utils.cameras import look_at

MODEL_ID = "shahafvl/gaussianformer"
FOV = 45.0
RESOLUTION = 512
MAX_GAUSSIANS = 20_000  # the model's training size; larger inputs are pruned by importance
HERE = Path(__file__).parent
SCENE_FILES = {"Gem": "examples/gem.ply", "Vase": "examples/vase.ply", "Planet": "examples/planet.ply"}

print("loading model…")
PIPE = GaussianFormerRenderingPipeline.from_pretrained(MODEL_ID).to("cuda")
PIPE.model.eval()
print("model ready")

# Bundled example scenes, pre-parsed on the CPU: name -> [N, 14] float32 tensor.
SCENES: dict[str, torch.Tensor] = {
    name: to_tensor(normalize(read_ply(HERE / rel), "y")) for name, rel in SCENE_FILES.items()
}
for name, t in SCENES.items():
    print(f"scene {name!r}: {t.shape[0]:,} Gaussians")


# --------------------------------------------------------------------------- #
# camera + splat helpers (CPU, main process)
# --------------------------------------------------------------------------- #
def camera_c2w(azimuth: float, elevation: float, distance: float) -> torch.Tensor:
    """Orbit camera around the origin -> c2w [1, 4, 4], Blender convention (-Z fwd, +Y up)."""
    az, el = math.radians(azimuth), math.radians(elevation)
    eye = np.array(
        [distance * math.cos(el) * math.cos(az), distance * math.sin(el),
         distance * math.cos(el) * math.sin(az)], np.float32)
    return torch.from_numpy(look_at(eye, np.zeros(3, np.float32), np.array([0, 1, 0], np.float32))[None])


def _prune_score(g: dict) -> np.ndarray:
    """Importance ~ opacity x projected area, adapted from LightGaussian as in the reference
    pipeline, but without gsplat: area per view ~ (max scale x focal / depth)^2."""
    focal = 0.5 * RESOLUTION / math.tan(math.radians(FOV) / 2)
    max_scale = g["scales"].max(-1)
    score = np.zeros(len(g["means"]), np.float64)
    for az in range(0, 360, 45):  # 8-view orbit, as the reference scores a 64-view one
        c2w = camera_c2w(az, 20.0, 1.7)[0].numpy()
        depth = np.clip((c2w[:3, 3] - g["means"]) @ c2w[:3, 2], 1e-3, None)
        score += g["opacities"] * (max_scale * focal / depth) ** 2
    return score * max_scale ** 0.1


def load_gaussians(path: str, up: str) -> tuple[torch.Tensor, dict]:
    """PLY -> ([N, 14] CPU tensor, stats dict). Prunes to MAX_GAUSSIANS by importance when needed."""
    try:
        g = normalize(read_ply(path), up)
    except (ValueError, KeyError, IndexError):
        raise gr.Error(
            "This does not look like a 3D Gaussian Splatting .ply — the renderer needs the "
            "raw optimizer fields scale_0..2, rot_0..3, f_dc_0..2, opacity (as written by "
            "every 3DGS implementation)."
        )
    if g is None:
        raise gr.Error("Could not normalize this splat: all Gaussians sit at (or near) one point.")
    n = len(g["means"])
    if n > MAX_GAUSSIANS:
        idx = np.argsort(_prune_score(g))[::-1][:MAX_GAUSSIANS]
        g = {k: v[idx] for k, v in g.items()}
    g14 = to_tensor(g)
    return g14, {"n": n, "kept": g14.shape[0]}


def _info_line(stats: dict, seconds: float) -> str:
    line = f"**{stats['kept']:,} Gaussians** · {RESOLUTION}×{RESOLUTION} · **{seconds:.2f} s** on the GPU"
    if stats["n"] > stats["kept"]:
        line += (
            f" — pruned from {stats['n']:,} by importance. Large pruned splats can turn dark: the"
            " reference pipeline also fine-tunes the kept Gaussians, which does not fit a demo."
        )
    return line


# --------------------------------------------------------------------------- #
# GPU worker (ZeroGPU)
# --------------------------------------------------------------------------- #
def _gpu_duration(*_args, **_kwargs) -> int:
    """One 512 px view of <=20k Gaussians takes ~0.3-1.3 s measured on this hardware. The
    reservation is padded to cover a cold worker after a reboot (780 MB of weights must be
    streamed in before the first render, which also pays an autotune + CUDA init)."""
    return 15


@spaces.GPU(duration=_gpu_duration)
def _gpu_render(gaussians: torch.Tensor, c2w: torch.Tensor) -> tuple[np.ndarray, float]:
    """[N, 14] + [1, 4, 4] (CPU tensors) -> (uint8 RGB image [512, 512, 3], GPU seconds)."""
    started = time.perf_counter()
    with torch.no_grad():
        images = PIPE(gaussians.to("cuda"), c2w.to("cuda"), fov=FOV, resolution=RESOLUTION)
        torch.cuda.synchronize()
    seconds = time.perf_counter() - started
    img = np.clip(images[0].float().cpu().numpy(), 0.0, 1.0)
    return (img * 255.0 + 0.5).astype(np.uint8), seconds


# --------------------------------------------------------------------------- #
# handlers
# --------------------------------------------------------------------------- #
def render_example(
    scene: str = "Gem",
    azimuth: float = 35.0,
    elevation: float = 20.0,
    distance: float = 1.7,
) -> tuple[np.ndarray, str]:
    """Render one of the bundled example splats from an orbit camera.

    Args:
        scene: Which bundled scene to render.
        azimuth: Orbit the camera around the object, in degrees.
        elevation: Camera height above the object's equator, in degrees.
        distance: Camera distance from the object's center (trained on 1.15–2.45).
    """
    if scene not in SCENES:
        raise gr.Error(f"Unknown scene {scene!r}.")
    try:
        img, seconds = _gpu_render(SCENES[scene], camera_c2w(azimuth, elevation, distance))
        return img, _info_line({"n": SCENES[scene].shape[0], "kept": SCENES[scene].shape[0]}, seconds)
    except gr.Error:
        raise
    except Exception:
        traceback.print_exc()
        raise gr.Error("Rendering failed — see this Space's logs for the traceback.")


def render_upload(
    ply_file,
    up: str = "y",
    azimuth: float = 35.0,
    elevation: float = 20.0,
    distance: float = 1.7,
) -> tuple[np.ndarray, str]:
    """Render your own 3D Gaussian Splatting .ply from an orbit camera.

    Args:
        ply_file: Path to a 3DGS .ply (log scales, wxyz quaternions, SH colors, opacity logits).
        up: The file's up axis: 'y', 'z' (e.g. Objaverse-style splats) or '-y' (COLMAP captures).
        azimuth: Orbit the camera around the object, in degrees.
        elevation: Camera height above the object's equator, in degrees.
        distance: Camera distance from the object's center (trained on 1.15–2.45).
    """
    if not ply_file:
        raise gr.Error("Upload a 3D Gaussian Splatting .ply file first.")
    path = ply_file if isinstance(ply_file, str) else getattr(ply_file, "name", None)
    try:
        g14, stats = load_gaussians(path, up)
        img, seconds = _gpu_render(g14, camera_c2w(azimuth, elevation, distance))
        return img, _info_line(stats, seconds)
    except gr.Error:
        raise
    except Exception:
        traceback.print_exc()
        raise gr.Error("Could not render this file — see this Space's logs for the traceback.")


# --------------------------------------------------------------------------- #
# UI
# --------------------------------------------------------------------------- #
CSS = "#col-container { max-width: 1180px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); }"

with gr.Blocks() as demo:
    with gr.Column(elem_id="col-container"):
        gr.Markdown(
            "# GaussianRenderFormer\n"
            "**Transformer-based rendering of 3D Gaussian splats** — "
            "[arXiv:2611.05593](https://arxiv.org/abs/2611.05593) · "
            "[code](https://github.com/SVLwoof/gaussianformer) · "
            "[project page](https://svlwoof.github.io/gaussianrenderformer/)\n\n"
            "Each Gaussian becomes one scene token; a view decoder cross-attends over 4-px image "
            "patches and renders the view — **no rasterizer, no per-scene optimization**. "
            "Point the camera and render."
        )
        with gr.Tabs():
            with gr.Tab("Example scenes"):
                with gr.Row():
                    with gr.Column(scale=1):
                        scene = gr.Dropdown(choices=list(SCENES), value="Gem",
                                           label="Scene (synthetic splats authored for this demo)")
                        azimuth = gr.Slider(0, 360, value=35, step=1, label="Azimuth (°)")
                        elevation = gr.Slider(-60, 60, value=20, step=1, label="Elevation (°)")
                        distance = gr.Slider(1.15, 2.45, value=1.7, step=0.01,
                                             label="Distance (trained on 1.15–2.45)")
                        ex_run = gr.Button("Render", variant="primary")
                    with gr.Column(scale=1):
                        ex_img = gr.Image(label="GaussianFormer render", type="numpy", height=512)
                        ex_info = gr.Markdown()
                gr.Examples(
                    examples=[
                        ["Gem", 35, 20, 1.7],
                        ["Vase", 210, -5, 1.45],
                        ["Planet", 120, 35, 2.45],
                    ],
                    inputs=[scene, azimuth, elevation, distance],
                    outputs=[ex_img, ex_info],
                    fn=render_example,
                    cache_examples=True,
                    cache_mode="lazy",
                    label="Try a scene",
                )
            with gr.Tab("Your own splat"):
                gr.Markdown(
                    "Upload a 3D Gaussian Splatting `.ply` (raw optimizer parameters: log scales, "
                    "wxyz quaternions, SH colors, opacity logits — the format every 3DGS "
                    "implementation writes). Splats larger than 20k Gaussians are pruned by "
                    "importance, as the model was trained on ~20k-Gaussian objects."
                )
                with gr.Row():
                    with gr.Column(scale=1):
                        ply_file = gr.File(label="3DGS .ply", file_types=[".ply"], type="filepath")
                        up_axis = gr.Radio(choices=["y", "z", "-y"], value="y", label="File's up axis")
                        azimuth_u = gr.Slider(0, 360, value=35, step=1, label="Azimuth (°)")
                        elevation_u = gr.Slider(-60, 60, value=20, step=1, label="Elevation (°)")
                        distance_u = gr.Slider(1.15, 2.45, value=1.7, step=0.01,
                                               label="Distance (trained on 1.15–2.45)")
                        up_run = gr.Button("Render", variant="primary")
                    with gr.Column(scale=1):
                        up_img = gr.Image(label="GaussianFormer render", type="numpy", height=512)
                        up_info = gr.Markdown()
                gr.Examples(
                    examples=[[str(HERE / "examples" / "gem.ply")]],
                    inputs=[ply_file],
                    outputs=[up_img, up_info],
                    fn=render_upload,
                    cache_examples=True,
                    cache_mode="lazy",
                    label="Try the upload path",
                )
        gr.Markdown(
            "**Limitations** (from the model card): trained on isolated single objects centered, "
            "Y-up and scaled into [-0.45, 0.45]³ at 512 px, 45° FOV and 20k Gaussians — denser "
            "inputs lose quality, colors are view-independent, and fine detail renders softer "
            "than a rasterizer. MIT license, for the weights as for the code."
        )
        ex_run.click(render_example, [scene, azimuth, elevation, distance], [ex_img, ex_info],
                     api_name="render_scene")
        up_run.click(render_upload, [ply_file, up_axis, azimuth_u, elevation_u, distance_u],
                     [up_img, up_info], api_name="render_splat")

if __name__ == "__main__":
    demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True, show_error=True)