Spaces:
Sleeping
Sleeping
Improve camera capture and scene controls
Browse files- INSTALL.md +7 -0
- README.md +1 -1
- app.py +9 -2
- requirements-dev.txt +8 -0
- src/demo/hf_runtime.py +5 -1
- src/demo/infer_single_image.py +54 -1
- src/demo/server_frontend/app.js +213 -10
- src/demo/server_frontend/index.html +24 -2
- src/demo/server_frontend/styles.css +70 -1
- src/model/encoder/depth/infinidepth/implicit_pda.py +5 -1
- src/model/encoder/depth/infinidepth/implicit_promptda_simple/prompt_models/selfattn.py +5 -1
- tests/test_hf_runtime.py +53 -3
- tests/test_webcam_frontend.py +25 -1
INSTALL.md
CHANGED
|
@@ -27,6 +27,13 @@ uv pip install torch==2.9.0 torchvision==0.24.0 xformers==0.0.33.post1 --index-u
|
|
| 27 |
uv pip install -r requirements.txt
|
| 28 |
```
|
| 29 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
### 3) Optional output dependencies
|
| 31 |
|
| 32 |
PLY export works without the dependencies in this section.
|
|
|
|
| 27 |
uv pip install -r requirements.txt
|
| 28 |
```
|
| 29 |
|
| 30 |
+
For development and tests, install the test dependencies instead:
|
| 31 |
+
|
| 32 |
+
```bash
|
| 33 |
+
uv pip install -r requirements-dev.txt
|
| 34 |
+
python -m pytest
|
| 35 |
+
```
|
| 36 |
+
|
| 37 |
### 3) Optional output dependencies
|
| 38 |
|
| 39 |
PLY export works without the dependencies in this section.
|
README.md
CHANGED
|
@@ -116,7 +116,7 @@ See [docs/inference.md](docs/inference.md) for camera parameters, output control
|
|
| 116 |
|
| 117 |
## 🖥️ Web Demo
|
| 118 |
|
| 119 |
-
The local Gradio demo accepts uploaded images or landscape/portrait photos captured from
|
| 120 |
|
| 121 |
```bash
|
| 122 |
python demo.py
|
|
|
|
| 116 |
|
| 117 |
## 🖥️ Web Demo
|
| 118 |
|
| 119 |
+
The local Gradio demo accepts uploaded images or high-resolution landscape/portrait photos captured from a selectable webcam, with an option to process exactly the visible camera frame. It also provides RGB reconstruction, staged PLY and standalone HTML downloads, an interactive Gaussian viewer with camera-distance controls, and an optional world-Z translation for exported scenes. After preparing the inference environment, launch it with:
|
| 120 |
|
| 121 |
```bash
|
| 122 |
python demo.py
|
app.py
CHANGED
|
@@ -3,6 +3,7 @@ from __future__ import annotations
|
|
| 3 |
import asyncio
|
| 4 |
import contextlib
|
| 5 |
import json
|
|
|
|
| 6 |
import os
|
| 7 |
import shutil
|
| 8 |
import time
|
|
@@ -237,14 +238,20 @@ def reconstruct(image: FileData) -> dict[str, str | int]:
|
|
| 237 |
concurrency_id="infinisplat-export",
|
| 238 |
time_limit=180,
|
| 239 |
)
|
| 240 |
-
def export_ply(request_id: str) -> dict[str, str | int]:
|
| 241 |
"""Export the reconstructed scene as a filtered Gaussian PLY."""
|
|
|
|
|
|
|
| 242 |
request_dir = _request_dir(request_id)
|
| 243 |
artifact = request_dir / "gaussians.pt"
|
| 244 |
if not artifact.is_file():
|
| 245 |
raise FileNotFoundError("The reconstruction artifact is unavailable.")
|
| 246 |
started_at = time.perf_counter()
|
| 247 |
-
scene_ply = export_filtered_gaussian_ply(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 248 |
artifact.unlink()
|
| 249 |
_log_timing("ply_export", request_dir, started_at, ply_bytes=scene_ply.stat().st_size)
|
| 250 |
return {"url": _file_url(scene_ply), "size": scene_ply.stat().st_size}
|
|
|
|
| 3 |
import asyncio
|
| 4 |
import contextlib
|
| 5 |
import json
|
| 6 |
+
import math
|
| 7 |
import os
|
| 8 |
import shutil
|
| 9 |
import time
|
|
|
|
| 238 |
concurrency_id="infinisplat-export",
|
| 239 |
time_limit=180,
|
| 240 |
)
|
| 241 |
+
def export_ply(request_id: str, z_offset: float = 0.0) -> dict[str, str | int]:
|
| 242 |
"""Export the reconstructed scene as a filtered Gaussian PLY."""
|
| 243 |
+
if not math.isfinite(z_offset) or abs(z_offset) > 100:
|
| 244 |
+
raise ValueError("The export Z offset must be between -100 and 100.")
|
| 245 |
request_dir = _request_dir(request_id)
|
| 246 |
artifact = request_dir / "gaussians.pt"
|
| 247 |
if not artifact.is_file():
|
| 248 |
raise FileNotFoundError("The reconstruction artifact is unavailable.")
|
| 249 |
started_at = time.perf_counter()
|
| 250 |
+
scene_ply = export_filtered_gaussian_ply(
|
| 251 |
+
artifact_path=artifact,
|
| 252 |
+
output_dir=request_dir,
|
| 253 |
+
z_offset=z_offset,
|
| 254 |
+
)
|
| 255 |
artifact.unlink()
|
| 256 |
_log_timing("ply_export", request_dir, started_at, ply_bytes=scene_ply.stat().st_size)
|
| 257 |
return {"url": _file_url(scene_ply), "size": scene_ply.stat().st_size}
|
requirements-dev.txt
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
-r requirements.txt
|
| 2 |
+
|
| 3 |
+
# Hugging Face injects these into the hosted Space; local tests need them explicitly.
|
| 4 |
+
gradio==6.20.0
|
| 5 |
+
spaces
|
| 6 |
+
|
| 7 |
+
# Test runner
|
| 8 |
+
pytest>=8,<10
|
src/demo/hf_runtime.py
CHANGED
|
@@ -326,10 +326,14 @@ def create_request_viewer(
|
|
| 326 |
def export_filtered_gaussian_ply(
|
| 327 |
artifact_path: Path,
|
| 328 |
output_dir: Path,
|
|
|
|
| 329 |
) -> Path:
|
| 330 |
-
"""Filter spatial outliers and export
|
| 331 |
artifact = load_gaussian_artifact(artifact_path)
|
| 332 |
gaussians = filter_final_gaussian_floaters(artifact.gaussians)
|
|
|
|
|
|
|
|
|
|
| 333 |
|
| 334 |
output_dir.mkdir(parents=True, exist_ok=True)
|
| 335 |
scene_ply = output_dir / "scene.ply"
|
|
|
|
| 326 |
def export_filtered_gaussian_ply(
|
| 327 |
artifact_path: Path,
|
| 328 |
output_dir: Path,
|
| 329 |
+
z_offset: float = 0.0,
|
| 330 |
) -> Path:
|
| 331 |
+
"""Filter spatial outliers, optionally translate Z, and export a Gaussian PLY."""
|
| 332 |
artifact = load_gaussian_artifact(artifact_path)
|
| 333 |
gaussians = filter_final_gaussian_floaters(artifact.gaussians)
|
| 334 |
+
if z_offset:
|
| 335 |
+
offset = gaussians.mean_vectors.new_tensor([0.0, 0.0, z_offset])
|
| 336 |
+
gaussians = gaussians._replace(mean_vectors=gaussians.mean_vectors + offset)
|
| 337 |
|
| 338 |
output_dir.mkdir(parents=True, exist_ok=True)
|
| 339 |
scene_ply = output_dir / "scene.ply"
|
src/demo/infer_single_image.py
CHANGED
|
@@ -862,7 +862,7 @@ def patch_supersplat_html_viewer_bridge(
|
|
| 862 |
html_path: Path,
|
| 863 |
viewer_script_path: Path | None = None,
|
| 864 |
) -> None:
|
| 865 |
-
"""Report
|
| 866 |
source = html_path.read_text(encoding="utf-8")
|
| 867 |
marker = "data-infinisplat-viewer-bridge"
|
| 868 |
first_frame_hook = "window.firstFrame?.();"
|
|
@@ -876,6 +876,49 @@ def patch_supersplat_html_viewer_bridge(
|
|
| 876 |
"Could not install the SuperSplat viewer bridge; the viewer bundle changed."
|
| 877 |
)
|
| 878 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 879 |
bridge = f"""
|
| 880 |
<script {marker}>
|
| 881 |
(() => {{
|
|
@@ -893,6 +936,16 @@ def patch_supersplat_html_viewer_bridge(
|
|
| 893 |
}};
|
| 894 |
|
| 895 |
window.firstFrame = () => setViewerState("ready");
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 896 |
window.addEventListener("error", () => setViewerState("error"));
|
| 897 |
window.addEventListener(
|
| 898 |
"unhandledrejection",
|
|
|
|
| 862 |
html_path: Path,
|
| 863 |
viewer_script_path: Path | None = None,
|
| 864 |
) -> None:
|
| 865 |
+
"""Report viewer readiness and expose safe host camera controls."""
|
| 866 |
source = html_path.read_text(encoding="utf-8")
|
| 867 |
marker = "data-infinisplat-viewer-bridge"
|
| 868 |
first_frame_hook = "window.firstFrame?.();"
|
|
|
|
| 876 |
"Could not install the SuperSplat viewer bridge; the viewer bundle changed."
|
| 877 |
)
|
| 878 |
|
| 879 |
+
if viewer_script_path is not None:
|
| 880 |
+
script_source = viewer_script_path.read_text(encoding="utf-8")
|
| 881 |
+
old_viewer_creation = """ // Create the viewer
|
| 882 |
+
return new Viewer(global, gsplatLoad, skyboxLoad, collisionLoad);"""
|
| 883 |
+
new_viewer_creation = """ // Create the viewer
|
| 884 |
+
const viewer = new Viewer(global, gsplatLoad, skyboxLoad, collisionLoad);
|
| 885 |
+
let infinisplatDistanceMultiplier = 1;
|
| 886 |
+
window.infinisplatViewerControls = {
|
| 887 |
+
setDistance(multiplier) {
|
| 888 |
+
const manager = viewer.cameraManager;
|
| 889 |
+
const nextMultiplier = Math.min(3, Math.max(0.25, Number(multiplier)));
|
| 890 |
+
if (!manager || !Number.isFinite(nextMultiplier)) return false;
|
| 891 |
+
const camera = manager.camera;
|
| 892 |
+
const focus = new Vec3();
|
| 893 |
+
camera.calcFocusPoint(focus);
|
| 894 |
+
const nextPosition = camera.position.clone()
|
| 895 |
+
.sub(focus)
|
| 896 |
+
.normalize()
|
| 897 |
+
.mulScalar(camera.distance * nextMultiplier / infinisplatDistanceMultiplier)
|
| 898 |
+
.add(focus);
|
| 899 |
+
global.state.cameraMode = 'orbit';
|
| 900 |
+
camera.look(nextPosition, focus);
|
| 901 |
+
manager.snap();
|
| 902 |
+
infinisplatDistanceMultiplier = nextMultiplier;
|
| 903 |
+
return true;
|
| 904 |
+
},
|
| 905 |
+
reset() {
|
| 906 |
+
global.events.fire('inputEvent', 'reset');
|
| 907 |
+
infinisplatDistanceMultiplier = 1;
|
| 908 |
+
return true;
|
| 909 |
+
}
|
| 910 |
+
};
|
| 911 |
+
return viewer;"""
|
| 912 |
+
if old_viewer_creation in script_source:
|
| 913 |
+
viewer_script_path.write_text(
|
| 914 |
+
script_source.replace(old_viewer_creation, new_viewer_creation, 1),
|
| 915 |
+
encoding="utf-8",
|
| 916 |
+
)
|
| 917 |
+
elif "window.infinisplatViewerControls" not in script_source:
|
| 918 |
+
raise RuntimeError(
|
| 919 |
+
"Could not install the SuperSplat camera controls; the viewer bundle changed."
|
| 920 |
+
)
|
| 921 |
+
|
| 922 |
bridge = f"""
|
| 923 |
<script {marker}>
|
| 924 |
(() => {{
|
|
|
|
| 936 |
}};
|
| 937 |
|
| 938 |
window.firstFrame = () => setViewerState("ready");
|
| 939 |
+
window.addEventListener("message", (event) => {{
|
| 940 |
+
if (event.source !== window.parent ||
|
| 941 |
+
event.data?.type !== "infinisplat:viewer-control") return;
|
| 942 |
+
const controls = window.infinisplatViewerControls;
|
| 943 |
+
if (event.data.action === "distance") {{
|
| 944 |
+
controls?.setDistance(event.data.multiplier);
|
| 945 |
+
}} else if (event.data.action === "reset") {{
|
| 946 |
+
controls?.reset();
|
| 947 |
+
}}
|
| 948 |
+
}});
|
| 949 |
window.addEventListener("error", () => setViewerState("error"));
|
| 950 |
window.addEventListener(
|
| 951 |
"unhandledrejection",
|
src/demo/server_frontend/app.js
CHANGED
|
@@ -8,6 +8,7 @@ const cameraButton = document.getElementById("cameraButton");
|
|
| 8 |
const cameraStage = document.getElementById("cameraStage");
|
| 9 |
const cameraPreview = document.getElementById("cameraPreview");
|
| 10 |
const cameraOptions = document.getElementById("cameraOptions");
|
|
|
|
| 11 |
const cameraFormatInputs = document.querySelectorAll('input[name="cameraFormat"]');
|
| 12 |
const cropToFrame = document.getElementById("cropToFrame");
|
| 13 |
const cameraOutputSize = document.getElementById("cameraOutputSize");
|
|
@@ -20,6 +21,12 @@ const viewerFrame = document.getElementById("viewerFrame");
|
|
| 20 |
const viewerPreloader = document.getElementById("viewerPreloader");
|
| 21 |
const viewerTitle = document.getElementById("viewerTitle");
|
| 22 |
const viewerDetail = document.getElementById("viewerDetail");
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
const plyDownload = document.getElementById("plyDownload");
|
| 24 |
const htmlDownload = document.getElementById("htmlDownload");
|
| 25 |
const examplesGrid = document.getElementById("examplesGrid");
|
|
@@ -33,6 +40,10 @@ let previewUrl = null;
|
|
| 33 |
let running = false;
|
| 34 |
let toastTimer = null;
|
| 35 |
let cameraStream = null;
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
const supportedImageTypes = new Set(["image/jpeg", "image/png", "image/webp"]);
|
| 37 |
const cameraFormats = {
|
| 38 |
landscape: { width: 1536, height: 1152 },
|
|
@@ -103,6 +114,21 @@ function setRunning(value) {
|
|
| 103 |
reconstructButton.disabled = value || !selectedFile || !client;
|
| 104 |
reconstructButton.textContent = value ? "Reconstructing..." : "Reconstruct scene";
|
| 105 |
examplesGrid.toggleAttribute("inert", value);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
}
|
| 107 |
|
| 108 |
function selectFile(file, fromCamera = false) {
|
|
@@ -122,8 +148,12 @@ function selectFile(file, fromCamera = false) {
|
|
| 122 |
}
|
| 123 |
|
| 124 |
function stopCamera() {
|
| 125 |
-
|
| 126 |
cameraStream = null;
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
cameraPreview.srcObject = null;
|
| 128 |
cameraStage.hidden = true;
|
| 129 |
cameraOptions.hidden = true;
|
|
@@ -143,9 +173,111 @@ function updateCameraFrame() {
|
|
| 143 |
const dimensions = cameraFormats[format];
|
| 144 |
cameraStage.dataset.format = format;
|
| 145 |
cameraStage.dataset.crop = cropToFrame.checked ? "frame" : "full";
|
|
|
|
|
|
|
|
|
|
| 146 |
cameraOutputSize.textContent = cropToFrame.checked
|
| 147 |
-
? `
|
| 148 |
-
:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 149 |
}
|
| 150 |
|
| 151 |
async function startCamera() {
|
|
@@ -156,12 +288,37 @@ async function startCamera() {
|
|
| 156 |
}
|
| 157 |
|
| 158 |
cameraButton.disabled = true;
|
|
|
|
| 159 |
try {
|
|
|
|
|
|
|
|
|
|
| 160 |
cameraStream = await navigator.mediaDevices.getUserMedia({
|
| 161 |
audio: false,
|
| 162 |
-
video: {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
});
|
| 164 |
cameraPreview.srcObject = cameraStream;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 165 |
imagePreview.hidden = true;
|
| 166 |
chooseImageButton.hidden = true;
|
| 167 |
updateCameraFrame();
|
|
@@ -172,6 +329,9 @@ async function startCamera() {
|
|
| 172 |
await cameraPreview.play();
|
| 173 |
} catch (error) {
|
| 174 |
console.error(error);
|
|
|
|
|
|
|
|
|
|
| 175 |
stopCamera();
|
| 176 |
const permissionDenied = error?.name === "NotAllowedError" || error?.name === "SecurityError";
|
| 177 |
showToast(
|
|
@@ -182,19 +342,23 @@ async function startCamera() {
|
|
| 182 |
);
|
| 183 |
} finally {
|
| 184 |
cameraButton.disabled = running || Boolean(cameraStream);
|
|
|
|
| 185 |
}
|
| 186 |
}
|
| 187 |
|
| 188 |
async function capturePhoto() {
|
| 189 |
-
|
| 190 |
-
const sourceHeight = cameraPreview.videoHeight;
|
| 191 |
-
if (!cameraStream || !sourceWidth || !sourceHeight) {
|
| 192 |
showToast("The camera is still starting. Try again in a moment.", true);
|
| 193 |
return;
|
| 194 |
}
|
| 195 |
|
| 196 |
captureButton.disabled = true;
|
|
|
|
| 197 |
try {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 198 |
const canvas = document.createElement("canvas");
|
| 199 |
let sourceX = 0;
|
| 200 |
let sourceY = 0;
|
|
@@ -221,8 +385,10 @@ async function capturePhoto() {
|
|
| 221 |
|
| 222 |
const context = canvas.getContext("2d");
|
| 223 |
if (!context) throw new Error("The browser could not create an image canvas.");
|
|
|
|
|
|
|
| 224 |
context.drawImage(
|
| 225 |
-
|
| 226 |
sourceX,
|
| 227 |
sourceY,
|
| 228 |
cropWidth,
|
|
@@ -232,7 +398,16 @@ async function capturePhoto() {
|
|
| 232 |
canvas.width,
|
| 233 |
canvas.height,
|
| 234 |
);
|
| 235 |
-
const
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 236 |
if (!blob) throw new Error("The browser could not encode the captured photo.");
|
| 237 |
stopCamera();
|
| 238 |
selectFile(
|
|
@@ -243,6 +418,7 @@ async function capturePhoto() {
|
|
| 243 |
console.error(error);
|
| 244 |
showToast("Could not capture the photo. Please try again.", true);
|
| 245 |
} finally {
|
|
|
|
| 246 |
captureButton.disabled = false;
|
| 247 |
}
|
| 248 |
}
|
|
@@ -259,10 +435,22 @@ imagePreview.addEventListener("keydown", (event) => {
|
|
| 259 |
}
|
| 260 |
});
|
| 261 |
cameraButton.addEventListener("click", startCamera);
|
|
|
|
| 262 |
cancelCameraButton.addEventListener("click", stopCamera);
|
| 263 |
captureButton.addEventListener("click", capturePhoto);
|
| 264 |
for (const input of cameraFormatInputs) input.addEventListener("change", updateCameraFrame);
|
| 265 |
cropToFrame.addEventListener("change", updateCameraFrame);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 266 |
window.addEventListener("pagehide", stopCamera);
|
| 267 |
|
| 268 |
if (!navigator.mediaDevices?.getUserMedia) {
|
|
@@ -270,6 +458,13 @@ if (!navigator.mediaDevices?.getUserMedia) {
|
|
| 270 |
cameraButton.title = "Camera access is not supported by this browser";
|
| 271 |
}
|
| 272 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 273 |
for (const eventName of ["dragenter", "dragover"]) {
|
| 274 |
uploadSurface.addEventListener(eventName, (event) => {
|
| 275 |
event.preventDefault();
|
|
@@ -295,6 +490,8 @@ window.addEventListener("message", (event) => {
|
|
| 295 |
if (event.source !== viewerFrame.contentWindow || event.data?.type !== "infinisplat:viewer-state") return;
|
| 296 |
if (event.data.state === "ready") {
|
| 297 |
setViewerState("ready", "Viewer ready", "Interactive scene loaded");
|
|
|
|
|
|
|
| 298 |
} else if (event.data.state === "error") {
|
| 299 |
setViewerState("error", "Viewer initialization failed", "Check the browser console for details");
|
| 300 |
} else if (event.data.state === "timeout") {
|
|
@@ -339,6 +536,9 @@ async function loadExamples() {
|
|
| 339 |
async function runPipeline() {
|
| 340 |
if (!selectedFile || !client || running) return;
|
| 341 |
setRunning(true);
|
|
|
|
|
|
|
|
|
|
| 342 |
viewerFrame.removeAttribute("src");
|
| 343 |
setDownload(plyDownload, null, "Preparing PLY...");
|
| 344 |
setDownload(htmlDownload, null, "Preparing HTML viewer...");
|
|
@@ -352,7 +552,10 @@ async function runPipeline() {
|
|
| 352 |
|
| 353 |
setViewerState("loading", "Preparing scene", "Creating the downloadable Gaussian PLY");
|
| 354 |
const ply = resultValue(
|
| 355 |
-
await client.predict("/export_ply", {
|
|
|
|
|
|
|
|
|
|
| 356 |
);
|
| 357 |
if (!ply?.url) throw new Error("The PLY export returned no file.");
|
| 358 |
setDownload(plyDownload, ply.url, "Download PLY - Ready");
|
|
|
|
| 8 |
const cameraStage = document.getElementById("cameraStage");
|
| 9 |
const cameraPreview = document.getElementById("cameraPreview");
|
| 10 |
const cameraOptions = document.getElementById("cameraOptions");
|
| 11 |
+
const cameraSelect = document.getElementById("cameraSelect");
|
| 12 |
const cameraFormatInputs = document.querySelectorAll('input[name="cameraFormat"]');
|
| 13 |
const cropToFrame = document.getElementById("cropToFrame");
|
| 14 |
const cameraOutputSize = document.getElementById("cameraOutputSize");
|
|
|
|
| 21 |
const viewerPreloader = document.getElementById("viewerPreloader");
|
| 22 |
const viewerTitle = document.getElementById("viewerTitle");
|
| 23 |
const viewerDetail = document.getElementById("viewerDetail");
|
| 24 |
+
const cameraDistance = document.getElementById("cameraDistance");
|
| 25 |
+
const cameraDistanceValue = document.getElementById("cameraDistanceValue");
|
| 26 |
+
const resetViewButton = document.getElementById("resetViewButton");
|
| 27 |
+
const bakeZOffset = document.getElementById("bakeZOffset");
|
| 28 |
+
const zOffset = document.getElementById("zOffset");
|
| 29 |
+
const zOffsetValue = document.getElementById("zOffsetValue");
|
| 30 |
const plyDownload = document.getElementById("plyDownload");
|
| 31 |
const htmlDownload = document.getElementById("htmlDownload");
|
| 32 |
const examplesGrid = document.getElementById("examplesGrid");
|
|
|
|
| 40 |
let running = false;
|
| 41 |
let toastTimer = null;
|
| 42 |
let cameraStream = null;
|
| 43 |
+
let imageCapture = null;
|
| 44 |
+
let photoSettings = null;
|
| 45 |
+
let cameraResolution = null;
|
| 46 |
+
let selectedCameraDeviceId = "";
|
| 47 |
const supportedImageTypes = new Set(["image/jpeg", "image/png", "image/webp"]);
|
| 48 |
const cameraFormats = {
|
| 49 |
landscape: { width: 1536, height: 1152 },
|
|
|
|
| 114 |
reconstructButton.disabled = value || !selectedFile || !client;
|
| 115 |
reconstructButton.textContent = value ? "Reconstructing..." : "Reconstruct scene";
|
| 116 |
examplesGrid.toggleAttribute("inert", value);
|
| 117 |
+
bakeZOffset.disabled = value;
|
| 118 |
+
zOffset.disabled = value || !bakeZOffset.checked;
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
function sendViewerControl(action, detail = {}) {
|
| 122 |
+
viewerFrame.contentWindow?.postMessage(
|
| 123 |
+
{ type: "infinisplat:viewer-control", action, ...detail },
|
| 124 |
+
"*",
|
| 125 |
+
);
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
function resetViewerControls(sendReset = false) {
|
| 129 |
+
cameraDistance.value = "100";
|
| 130 |
+
cameraDistanceValue.value = "100%";
|
| 131 |
+
if (sendReset) sendViewerControl("reset");
|
| 132 |
}
|
| 133 |
|
| 134 |
function selectFile(file, fromCamera = false) {
|
|
|
|
| 148 |
}
|
| 149 |
|
| 150 |
function stopCamera() {
|
| 151 |
+
const stream = cameraStream;
|
| 152 |
cameraStream = null;
|
| 153 |
+
for (const track of stream?.getTracks() || []) track.stop();
|
| 154 |
+
imageCapture = null;
|
| 155 |
+
photoSettings = null;
|
| 156 |
+
cameraResolution = null;
|
| 157 |
cameraPreview.srcObject = null;
|
| 158 |
cameraStage.hidden = true;
|
| 159 |
cameraOptions.hidden = true;
|
|
|
|
| 173 |
const dimensions = cameraFormats[format];
|
| 174 |
cameraStage.dataset.format = format;
|
| 175 |
cameraStage.dataset.crop = cropToFrame.checked ? "frame" : "full";
|
| 176 |
+
const resolution = cameraResolution
|
| 177 |
+
? `Camera: ${cameraResolution.width} × ${cameraResolution.height} · `
|
| 178 |
+
: "";
|
| 179 |
cameraOutputSize.textContent = cropToFrame.checked
|
| 180 |
+
? `${resolution}output crop: ${dimensions.width} × ${dimensions.height}`
|
| 181 |
+
: `${resolution}output: full camera image`;
|
| 182 |
+
}
|
| 183 |
+
|
| 184 |
+
async function prepareHighResolutionCapture(track) {
|
| 185 |
+
const settings = track.getSettings?.() || {};
|
| 186 |
+
if (settings.width && settings.height) {
|
| 187 |
+
cameraResolution = { width: settings.width, height: settings.height };
|
| 188 |
+
}
|
| 189 |
+
|
| 190 |
+
if (typeof ImageCapture === "undefined" || typeof createImageBitmap === "undefined") return;
|
| 191 |
+
try {
|
| 192 |
+
imageCapture = new ImageCapture(track);
|
| 193 |
+
const capabilities = await imageCapture.getPhotoCapabilities?.();
|
| 194 |
+
const maxWidth = capabilities?.imageWidth?.max;
|
| 195 |
+
const maxHeight = capabilities?.imageHeight?.max;
|
| 196 |
+
if (maxWidth && maxHeight) {
|
| 197 |
+
photoSettings = { imageWidth: maxWidth, imageHeight: maxHeight };
|
| 198 |
+
cameraResolution = { width: maxWidth, height: maxHeight };
|
| 199 |
+
}
|
| 200 |
+
} catch (error) {
|
| 201 |
+
console.warn("High-resolution still capture is unavailable; using the video stream.", error);
|
| 202 |
+
imageCapture = null;
|
| 203 |
+
photoSettings = null;
|
| 204 |
+
}
|
| 205 |
+
}
|
| 206 |
+
|
| 207 |
+
async function optimizeCameraTrack(track) {
|
| 208 |
+
const capabilities = track.getCapabilities?.() || {};
|
| 209 |
+
const advanced = {};
|
| 210 |
+
for (const mode of ["focusMode", "exposureMode", "whiteBalanceMode"]) {
|
| 211 |
+
if (capabilities[mode]?.includes?.("continuous")) advanced[mode] = "continuous";
|
| 212 |
+
}
|
| 213 |
+
if (!Object.keys(advanced).length) return;
|
| 214 |
+
try {
|
| 215 |
+
await track.applyConstraints({ advanced: [advanced] });
|
| 216 |
+
} catch (error) {
|
| 217 |
+
console.warn("The camera rejected continuous image-quality controls.", error);
|
| 218 |
+
}
|
| 219 |
+
}
|
| 220 |
+
|
| 221 |
+
async function acquireCameraImage() {
|
| 222 |
+
if (imageCapture) {
|
| 223 |
+
try {
|
| 224 |
+
let blob;
|
| 225 |
+
try {
|
| 226 |
+
blob = await imageCapture.takePhoto(photoSettings || undefined);
|
| 227 |
+
} catch (settingsError) {
|
| 228 |
+
console.warn("The maximum photo settings were rejected; retrying camera defaults.", settingsError);
|
| 229 |
+
blob = await imageCapture.takePhoto();
|
| 230 |
+
}
|
| 231 |
+
const bitmap = await createImageBitmap(blob);
|
| 232 |
+
return {
|
| 233 |
+
source: bitmap,
|
| 234 |
+
width: bitmap.width,
|
| 235 |
+
height: bitmap.height,
|
| 236 |
+
release: () => bitmap.close(),
|
| 237 |
+
};
|
| 238 |
+
} catch (error) {
|
| 239 |
+
console.warn("Still-photo capture failed; using the current video frame.", error);
|
| 240 |
+
}
|
| 241 |
+
}
|
| 242 |
+
return {
|
| 243 |
+
source: cameraPreview,
|
| 244 |
+
width: cameraPreview.videoWidth,
|
| 245 |
+
height: cameraPreview.videoHeight,
|
| 246 |
+
release: () => {},
|
| 247 |
+
};
|
| 248 |
+
}
|
| 249 |
+
|
| 250 |
+
async function refreshCameraDevices(activeDeviceId = "") {
|
| 251 |
+
const devices = (await navigator.mediaDevices.enumerateDevices())
|
| 252 |
+
.filter((device) => device.kind === "videoinput");
|
| 253 |
+
const availableIds = new Set(devices.map((device) => device.deviceId));
|
| 254 |
+
if (selectedCameraDeviceId && !availableIds.has(selectedCameraDeviceId)) {
|
| 255 |
+
selectedCameraDeviceId = "";
|
| 256 |
+
}
|
| 257 |
+
|
| 258 |
+
cameraSelect.replaceChildren();
|
| 259 |
+
const automaticOption = document.createElement("option");
|
| 260 |
+
automaticOption.value = "";
|
| 261 |
+
automaticOption.textContent = "Automatic";
|
| 262 |
+
cameraSelect.append(automaticOption);
|
| 263 |
+
devices.forEach((device, index) => {
|
| 264 |
+
const option = document.createElement("option");
|
| 265 |
+
option.value = device.deviceId;
|
| 266 |
+
option.textContent = device.label || `Camera ${index + 1}`;
|
| 267 |
+
cameraSelect.append(option);
|
| 268 |
+
});
|
| 269 |
+
|
| 270 |
+
const preferredId = selectedCameraDeviceId || activeDeviceId;
|
| 271 |
+
cameraSelect.value = availableIds.has(preferredId) ? preferredId : "";
|
| 272 |
+
cameraSelect.disabled = running || devices.length < 2;
|
| 273 |
+
}
|
| 274 |
+
|
| 275 |
+
async function switchCamera() {
|
| 276 |
+
if (running) return;
|
| 277 |
+
selectedCameraDeviceId = cameraSelect.value;
|
| 278 |
+
cameraSelect.disabled = true;
|
| 279 |
+
stopCamera();
|
| 280 |
+
await startCamera();
|
| 281 |
}
|
| 282 |
|
| 283 |
async function startCamera() {
|
|
|
|
| 288 |
}
|
| 289 |
|
| 290 |
cameraButton.disabled = true;
|
| 291 |
+
cameraSelect.disabled = true;
|
| 292 |
try {
|
| 293 |
+
const deviceConstraint = selectedCameraDeviceId
|
| 294 |
+
? { deviceId: { exact: selectedCameraDeviceId } }
|
| 295 |
+
: { facingMode: { ideal: "environment" } };
|
| 296 |
cameraStream = await navigator.mediaDevices.getUserMedia({
|
| 297 |
audio: false,
|
| 298 |
+
video: {
|
| 299 |
+
...deviceConstraint,
|
| 300 |
+
width: { ideal: 2560 },
|
| 301 |
+
height: { ideal: 1920 },
|
| 302 |
+
aspectRatio: { ideal: 4 / 3 },
|
| 303 |
+
},
|
| 304 |
});
|
| 305 |
cameraPreview.srcObject = cameraStream;
|
| 306 |
+
const videoTrack = cameraStream.getVideoTracks()[0];
|
| 307 |
+
videoTrack.addEventListener("ended", () => {
|
| 308 |
+
if (cameraStream && !running) {
|
| 309 |
+
selectedCameraDeviceId = "";
|
| 310 |
+
stopCamera();
|
| 311 |
+
showToast("The selected camera disconnected. Choose another camera to continue.", true);
|
| 312 |
+
}
|
| 313 |
+
});
|
| 314 |
+
await optimizeCameraTrack(videoTrack);
|
| 315 |
+
await prepareHighResolutionCapture(videoTrack);
|
| 316 |
+
try {
|
| 317 |
+
await refreshCameraDevices(videoTrack.getSettings?.().deviceId || "");
|
| 318 |
+
} catch (error) {
|
| 319 |
+
console.warn("Could not enumerate camera devices.", error);
|
| 320 |
+
cameraSelect.disabled = true;
|
| 321 |
+
}
|
| 322 |
imagePreview.hidden = true;
|
| 323 |
chooseImageButton.hidden = true;
|
| 324 |
updateCameraFrame();
|
|
|
|
| 329 |
await cameraPreview.play();
|
| 330 |
} catch (error) {
|
| 331 |
console.error(error);
|
| 332 |
+
if (error?.name === "NotFoundError" || error?.name === "OverconstrainedError") {
|
| 333 |
+
selectedCameraDeviceId = "";
|
| 334 |
+
}
|
| 335 |
stopCamera();
|
| 336 |
const permissionDenied = error?.name === "NotAllowedError" || error?.name === "SecurityError";
|
| 337 |
showToast(
|
|
|
|
| 342 |
);
|
| 343 |
} finally {
|
| 344 |
cameraButton.disabled = running || Boolean(cameraStream);
|
| 345 |
+
cameraSelect.disabled = running || cameraSelect.options.length < 3;
|
| 346 |
}
|
| 347 |
}
|
| 348 |
|
| 349 |
async function capturePhoto() {
|
| 350 |
+
if (!cameraStream || !cameraPreview.videoWidth || !cameraPreview.videoHeight) {
|
|
|
|
|
|
|
| 351 |
showToast("The camera is still starting. Try again in a moment.", true);
|
| 352 |
return;
|
| 353 |
}
|
| 354 |
|
| 355 |
captureButton.disabled = true;
|
| 356 |
+
let cameraImage = null;
|
| 357 |
try {
|
| 358 |
+
cameraImage = await acquireCameraImage();
|
| 359 |
+
const sourceWidth = cameraImage.width;
|
| 360 |
+
const sourceHeight = cameraImage.height;
|
| 361 |
+
if (!sourceWidth || !sourceHeight) throw new Error("The camera returned an empty image.");
|
| 362 |
const canvas = document.createElement("canvas");
|
| 363 |
let sourceX = 0;
|
| 364 |
let sourceY = 0;
|
|
|
|
| 385 |
|
| 386 |
const context = canvas.getContext("2d");
|
| 387 |
if (!context) throw new Error("The browser could not create an image canvas.");
|
| 388 |
+
context.imageSmoothingEnabled = true;
|
| 389 |
+
context.imageSmoothingQuality = "high";
|
| 390 |
context.drawImage(
|
| 391 |
+
cameraImage.source,
|
| 392 |
sourceX,
|
| 393 |
sourceY,
|
| 394 |
cropWidth,
|
|
|
|
| 398 |
canvas.width,
|
| 399 |
canvas.height,
|
| 400 |
);
|
| 401 |
+
const inferenceDimensions = cropToFrame.checked
|
| 402 |
+
? cameraFormats[selectedCameraFormat()]
|
| 403 |
+
: cameraFormats[sourceHeight > sourceWidth ? "portrait" : "landscape"];
|
| 404 |
+
if (cropWidth < inferenceDimensions.width || cropHeight < inferenceDimensions.height) {
|
| 405 |
+
showToast(
|
| 406 |
+
`Camera crop is ${Math.round(cropWidth)} × ${Math.round(cropHeight)} and will be enlarged; it may look soft.`,
|
| 407 |
+
true,
|
| 408 |
+
);
|
| 409 |
+
}
|
| 410 |
+
const blob = await new Promise((resolve) => canvas.toBlob(resolve, "image/jpeg", 0.95));
|
| 411 |
if (!blob) throw new Error("The browser could not encode the captured photo.");
|
| 412 |
stopCamera();
|
| 413 |
selectFile(
|
|
|
|
| 418 |
console.error(error);
|
| 419 |
showToast("Could not capture the photo. Please try again.", true);
|
| 420 |
} finally {
|
| 421 |
+
cameraImage?.release();
|
| 422 |
captureButton.disabled = false;
|
| 423 |
}
|
| 424 |
}
|
|
|
|
| 435 |
}
|
| 436 |
});
|
| 437 |
cameraButton.addEventListener("click", startCamera);
|
| 438 |
+
cameraSelect.addEventListener("change", switchCamera);
|
| 439 |
cancelCameraButton.addEventListener("click", stopCamera);
|
| 440 |
captureButton.addEventListener("click", capturePhoto);
|
| 441 |
for (const input of cameraFormatInputs) input.addEventListener("change", updateCameraFrame);
|
| 442 |
cropToFrame.addEventListener("change", updateCameraFrame);
|
| 443 |
+
cameraDistance.addEventListener("input", () => {
|
| 444 |
+
cameraDistanceValue.value = `${cameraDistance.value}%`;
|
| 445 |
+
sendViewerControl("distance", { multiplier: Number(cameraDistance.value) / 100 });
|
| 446 |
+
});
|
| 447 |
+
resetViewButton.addEventListener("click", () => resetViewerControls(true));
|
| 448 |
+
bakeZOffset.addEventListener("change", () => {
|
| 449 |
+
zOffset.disabled = running || !bakeZOffset.checked;
|
| 450 |
+
});
|
| 451 |
+
zOffset.addEventListener("input", () => {
|
| 452 |
+
zOffsetValue.value = Number(zOffset.value).toFixed(2);
|
| 453 |
+
});
|
| 454 |
window.addEventListener("pagehide", stopCamera);
|
| 455 |
|
| 456 |
if (!navigator.mediaDevices?.getUserMedia) {
|
|
|
|
| 458 |
cameraButton.title = "Camera access is not supported by this browser";
|
| 459 |
}
|
| 460 |
|
| 461 |
+
navigator.mediaDevices?.addEventListener?.("devicechange", () => {
|
| 462 |
+
if (cameraStream) {
|
| 463 |
+
refreshCameraDevices(cameraStream.getVideoTracks()[0]?.getSettings?.().deviceId || "")
|
| 464 |
+
.catch((error) => console.warn("Could not refresh the camera list.", error));
|
| 465 |
+
}
|
| 466 |
+
});
|
| 467 |
+
|
| 468 |
for (const eventName of ["dragenter", "dragover"]) {
|
| 469 |
uploadSurface.addEventListener(eventName, (event) => {
|
| 470 |
event.preventDefault();
|
|
|
|
| 490 |
if (event.source !== viewerFrame.contentWindow || event.data?.type !== "infinisplat:viewer-state") return;
|
| 491 |
if (event.data.state === "ready") {
|
| 492 |
setViewerState("ready", "Viewer ready", "Interactive scene loaded");
|
| 493 |
+
cameraDistance.disabled = false;
|
| 494 |
+
resetViewButton.disabled = false;
|
| 495 |
} else if (event.data.state === "error") {
|
| 496 |
setViewerState("error", "Viewer initialization failed", "Check the browser console for details");
|
| 497 |
} else if (event.data.state === "timeout") {
|
|
|
|
| 536 |
async function runPipeline() {
|
| 537 |
if (!selectedFile || !client || running) return;
|
| 538 |
setRunning(true);
|
| 539 |
+
cameraDistance.disabled = true;
|
| 540 |
+
resetViewButton.disabled = true;
|
| 541 |
+
resetViewerControls();
|
| 542 |
viewerFrame.removeAttribute("src");
|
| 543 |
setDownload(plyDownload, null, "Preparing PLY...");
|
| 544 |
setDownload(htmlDownload, null, "Preparing HTML viewer...");
|
|
|
|
| 552 |
|
| 553 |
setViewerState("loading", "Preparing scene", "Creating the downloadable Gaussian PLY");
|
| 554 |
const ply = resultValue(
|
| 555 |
+
await client.predict("/export_ply", {
|
| 556 |
+
request_id: reconstruction.request_id,
|
| 557 |
+
z_offset: bakeZOffset.checked ? Number(zOffset.value) : 0,
|
| 558 |
+
}),
|
| 559 |
);
|
| 560 |
if (!ply?.url) throw new Error("The PLY export returned no file.");
|
| 561 |
setDownload(plyDownload, ply.url, "Download PLY - Ready");
|
src/demo/server_frontend/index.html
CHANGED
|
@@ -13,7 +13,7 @@
|
|
| 13 |
? requestedTheme
|
| 14 |
: (prefersDark ? "dark" : "light");
|
| 15 |
</script>
|
| 16 |
-
<link rel="stylesheet" href="/infinisplat/assets/styles.css?v=
|
| 17 |
</head>
|
| 18 |
<body>
|
| 19 |
<main class="app-shell">
|
|
@@ -49,6 +49,12 @@
|
|
| 49 |
<small>JPG, PNG, or WebP</small>
|
| 50 |
</button>
|
| 51 |
<div class="camera-options" id="cameraOptions" hidden>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
<fieldset class="format-control">
|
| 53 |
<legend>Camera format</legend>
|
| 54 |
<label>
|
|
@@ -94,6 +100,22 @@
|
|
| 94 |
</div>
|
| 95 |
</div>
|
| 96 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
<div class="download-bar">
|
| 98 |
<a id="plyDownload" class="artifact-button disabled" aria-disabled="true">Download PLY</a>
|
| 99 |
<a id="htmlDownload" class="artifact-button disabled" aria-disabled="true">Download HTML viewer</a>
|
|
@@ -111,6 +133,6 @@
|
|
| 111 |
</main>
|
| 112 |
|
| 113 |
<div class="toast" id="toast" role="status" aria-live="polite"></div>
|
| 114 |
-
<script type="module" src="/infinisplat/assets/app.js?v=
|
| 115 |
</body>
|
| 116 |
</html>
|
|
|
|
| 13 |
? requestedTheme
|
| 14 |
: (prefersDark ? "dark" : "light");
|
| 15 |
</script>
|
| 16 |
+
<link rel="stylesheet" href="/infinisplat/assets/styles.css?v=8">
|
| 17 |
</head>
|
| 18 |
<body>
|
| 19 |
<main class="app-shell">
|
|
|
|
| 49 |
<small>JPG, PNG, or WebP</small>
|
| 50 |
</button>
|
| 51 |
<div class="camera-options" id="cameraOptions" hidden>
|
| 52 |
+
<label class="camera-select-control">
|
| 53 |
+
<span>Camera</span>
|
| 54 |
+
<select id="cameraSelect" aria-label="Camera device">
|
| 55 |
+
<option value="">Automatic</option>
|
| 56 |
+
</select>
|
| 57 |
+
</label>
|
| 58 |
<fieldset class="format-control">
|
| 59 |
<legend>Camera format</legend>
|
| 60 |
<label>
|
|
|
|
| 100 |
</div>
|
| 101 |
</div>
|
| 102 |
</div>
|
| 103 |
+
<div class="viewer-tools" aria-label="Viewer and export positioning">
|
| 104 |
+
<div class="viewer-tool-row">
|
| 105 |
+
<label for="cameraDistance">Camera distance</label>
|
| 106 |
+
<input id="cameraDistance" type="range" min="25" max="300" value="100" step="5" disabled>
|
| 107 |
+
<output id="cameraDistanceValue" for="cameraDistance">100%</output>
|
| 108 |
+
<button id="resetViewButton" type="button" disabled>Reset view</button>
|
| 109 |
+
</div>
|
| 110 |
+
<div class="viewer-tool-row export-z-row">
|
| 111 |
+
<label class="bake-z-control" title="Move every splat along the world Z axis before creating the PLY and HTML viewer">
|
| 112 |
+
<input id="bakeZOffset" type="checkbox">
|
| 113 |
+
<span>Bake Z offset into export</span>
|
| 114 |
+
</label>
|
| 115 |
+
<input id="zOffset" type="range" min="-2" max="2" value="0" step="0.05" disabled>
|
| 116 |
+
<output id="zOffsetValue" for="zOffset">0.00</output>
|
| 117 |
+
</div>
|
| 118 |
+
</div>
|
| 119 |
<div class="download-bar">
|
| 120 |
<a id="plyDownload" class="artifact-button disabled" aria-disabled="true">Download PLY</a>
|
| 121 |
<a id="htmlDownload" class="artifact-button disabled" aria-disabled="true">Download HTML viewer</a>
|
|
|
|
| 133 |
</main>
|
| 134 |
|
| 135 |
<div class="toast" id="toast" role="status" aria-live="polite"></div>
|
| 136 |
+
<script type="module" src="/infinisplat/assets/app.js?v=10"></script>
|
| 137 |
</body>
|
| 138 |
</html>
|
src/demo/server_frontend/styles.css
CHANGED
|
@@ -336,6 +336,34 @@ button, input { font: inherit; }
|
|
| 336 |
backdrop-filter: blur(8px);
|
| 337 |
}
|
| 338 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 339 |
.format-control {
|
| 340 |
display: grid;
|
| 341 |
grid-template-columns: 1fr 1fr;
|
|
@@ -538,6 +566,44 @@ button, input { font: inherit; }
|
|
| 538 |
|
| 539 |
@keyframes viewer-spin { to { transform: rotate(360deg); } }
|
| 540 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 541 |
.artifact-button {
|
| 542 |
display: inline-flex;
|
| 543 |
align-items: center;
|
|
@@ -660,7 +726,10 @@ button, input { font: inherit; }
|
|
| 660 |
.panel-heading { min-height: 52px; padding: 14px; }
|
| 661 |
.source-panel .panel-heading p { margin-left: 8px; }
|
| 662 |
.panel-content { padding: 9px; }
|
| 663 |
-
.source-actions, .download-bar { padding: 0 9px 9px; }
|
|
|
|
|
|
|
|
|
|
| 664 |
.upload-surface { min-height: 360px; }
|
| 665 |
.upload-surface img, .viewer-shell { height: 360px; }
|
| 666 |
.camera-stage[data-format="landscape"] { width: min(100%, 480px); }
|
|
|
|
| 336 |
backdrop-filter: blur(8px);
|
| 337 |
}
|
| 338 |
|
| 339 |
+
.camera-select-control {
|
| 340 |
+
display: grid;
|
| 341 |
+
grid-template-columns: auto minmax(0, 1fr);
|
| 342 |
+
align-items: center;
|
| 343 |
+
gap: 9px;
|
| 344 |
+
width: 100%;
|
| 345 |
+
color: var(--muted);
|
| 346 |
+
font-size: 0.72rem;
|
| 347 |
+
font-weight: 650;
|
| 348 |
+
}
|
| 349 |
+
|
| 350 |
+
.camera-select-control select {
|
| 351 |
+
min-width: 0;
|
| 352 |
+
min-height: 34px;
|
| 353 |
+
padding: 0 28px 0 9px;
|
| 354 |
+
overflow: hidden;
|
| 355 |
+
border: 1px solid var(--border-strong);
|
| 356 |
+
border-radius: 5px;
|
| 357 |
+
background: var(--surface);
|
| 358 |
+
color: var(--text);
|
| 359 |
+
font-size: 0.74rem;
|
| 360 |
+
text-overflow: ellipsis;
|
| 361 |
+
cursor: pointer;
|
| 362 |
+
}
|
| 363 |
+
|
| 364 |
+
.camera-select-control select:focus-visible { outline: 2px solid var(--primary); outline-offset: 2px; }
|
| 365 |
+
.camera-select-control select:disabled { cursor: not-allowed; opacity: 0.65; }
|
| 366 |
+
|
| 367 |
.format-control {
|
| 368 |
display: grid;
|
| 369 |
grid-template-columns: 1fr 1fr;
|
|
|
|
| 566 |
|
| 567 |
@keyframes viewer-spin { to { transform: rotate(360deg); } }
|
| 568 |
|
| 569 |
+
.viewer-tools {
|
| 570 |
+
display: grid;
|
| 571 |
+
gap: 8px;
|
| 572 |
+
padding: 0 12px 12px;
|
| 573 |
+
}
|
| 574 |
+
|
| 575 |
+
.viewer-tool-row {
|
| 576 |
+
display: grid;
|
| 577 |
+
grid-template-columns: auto minmax(80px, 1fr) 44px auto;
|
| 578 |
+
align-items: center;
|
| 579 |
+
gap: 9px;
|
| 580 |
+
min-height: 34px;
|
| 581 |
+
color: var(--muted);
|
| 582 |
+
font-size: 0.72rem;
|
| 583 |
+
font-weight: 650;
|
| 584 |
+
}
|
| 585 |
+
|
| 586 |
+
.viewer-tool-row input[type="range"] { width: 100%; accent-color: var(--primary); }
|
| 587 |
+
.viewer-tool-row output { color: var(--text); font-variant-numeric: tabular-nums; text-align: right; }
|
| 588 |
+
|
| 589 |
+
.viewer-tool-row button {
|
| 590 |
+
min-height: 32px;
|
| 591 |
+
padding: 0 10px;
|
| 592 |
+
border: 1px solid var(--border-strong);
|
| 593 |
+
border-radius: 5px;
|
| 594 |
+
background: var(--surface);
|
| 595 |
+
color: var(--text);
|
| 596 |
+
font-size: 0.72rem;
|
| 597 |
+
font-weight: 650;
|
| 598 |
+
cursor: pointer;
|
| 599 |
+
}
|
| 600 |
+
|
| 601 |
+
.viewer-tool-row button:hover:not(:disabled) { border-color: var(--primary); background: var(--surface-hover); }
|
| 602 |
+
.viewer-tool-row button:disabled, .viewer-tool-row input:disabled { cursor: not-allowed; opacity: 0.5; }
|
| 603 |
+
.export-z-row { grid-template-columns: minmax(155px, auto) minmax(80px, 1fr) 44px; }
|
| 604 |
+
.bake-z-control { display: flex; align-items: center; gap: 6px; cursor: pointer; }
|
| 605 |
+
.bake-z-control input { accent-color: var(--primary); }
|
| 606 |
+
|
| 607 |
.artifact-button {
|
| 608 |
display: inline-flex;
|
| 609 |
align-items: center;
|
|
|
|
| 726 |
.panel-heading { min-height: 52px; padding: 14px; }
|
| 727 |
.source-panel .panel-heading p { margin-left: 8px; }
|
| 728 |
.panel-content { padding: 9px; }
|
| 729 |
+
.source-actions, .download-bar, .viewer-tools { padding: 0 9px 9px; }
|
| 730 |
+
.viewer-tool-row { grid-template-columns: 1fr 44px auto; }
|
| 731 |
+
.viewer-tool-row > label:first-child { grid-column: 1 / -1; }
|
| 732 |
+
.export-z-row { grid-template-columns: minmax(0, 1fr) 44px; }
|
| 733 |
.upload-surface { min-height: 360px; }
|
| 734 |
.upload-surface img, .viewer-shell { height: 360px; }
|
| 735 |
.camera-stage[data-format="landscape"] { width: min(100%, 480px); }
|
src/model/encoder/depth/infinidepth/implicit_pda.py
CHANGED
|
@@ -26,7 +26,11 @@ from src.model.encoder.depth.infinidepth.warp_utils import WarpMedian
|
|
| 26 |
from typing import Optional
|
| 27 |
|
| 28 |
EPS = 1e-6
|
| 29 |
-
acc_dtype =
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
|
| 32 |
def _clear_torchhub_package(package_name: str) -> None:
|
|
|
|
| 26 |
from typing import Optional
|
| 27 |
|
| 28 |
EPS = 1e-6
|
| 29 |
+
acc_dtype = (
|
| 30 |
+
torch.bfloat16
|
| 31 |
+
if torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 8
|
| 32 |
+
else torch.float16
|
| 33 |
+
)
|
| 34 |
|
| 35 |
|
| 36 |
def _clear_torchhub_package(package_name: str) -> None:
|
src/model/encoder/depth/infinidepth/implicit_promptda_simple/prompt_models/selfattn.py
CHANGED
|
@@ -5,7 +5,11 @@ from src.model.encoder.depth.infinidepth.implicit_promptda_simple.perceive_io im
|
|
| 5 |
from src.model.encoder.depth.infinidepth.implicit_promptda_simple.prompt_models.utils.pe_utils import PositionEmbeddingRandom
|
| 6 |
from torch import Tensor
|
| 7 |
|
| 8 |
-
acc_dtype =
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
|
| 10 |
|
| 11 |
class SelfAttnPromptModel(nn.Module):
|
|
|
|
| 5 |
from src.model.encoder.depth.infinidepth.implicit_promptda_simple.prompt_models.utils.pe_utils import PositionEmbeddingRandom
|
| 6 |
from torch import Tensor
|
| 7 |
|
| 8 |
+
acc_dtype = (
|
| 9 |
+
torch.bfloat16
|
| 10 |
+
if torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 8
|
| 11 |
+
else torch.float16
|
| 12 |
+
)
|
| 13 |
|
| 14 |
|
| 15 |
class SelfAttnPromptModel(nn.Module):
|
tests/test_hf_runtime.py
CHANGED
|
@@ -161,6 +161,36 @@ def test_filtered_ply_export_removes_spatial_outlier(tmp_path, monkeypatch) -> N
|
|
| 161 |
assert float(captured["gaussians"].mean_vectors.abs().max()) < 1.0
|
| 162 |
|
| 163 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
def test_build_splat_transform_command_is_fixed_and_non_interactive() -> None:
|
| 165 |
command = build_splat_transform_command(
|
| 166 |
scene_ply=Path("scene.ply"),
|
|
@@ -251,13 +281,33 @@ def test_supersplat_viewer_bridge_accepts_external_script(tmp_path) -> None:
|
|
| 251 |
viewer_path = tmp_path / "viewer.html"
|
| 252 |
script_path = tmp_path / "index.js"
|
| 253 |
viewer_path.write_text("<html><head></head><body></body></html>")
|
| 254 |
-
script_path.write_text(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 255 |
|
| 256 |
patch_supersplat_html_viewer_bridge(viewer_path, script_path)
|
| 257 |
|
| 258 |
assert "data-infinisplat-viewer-bridge" in viewer_path.read_text()
|
| 259 |
|
| 260 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 261 |
def test_supersplat_viewer_bridge_rejects_changed_bundle(tmp_path) -> None:
|
| 262 |
viewer_path = tmp_path / "viewer.html"
|
| 263 |
viewer_path.write_text("<html><head></head><body></body></html>")
|
|
@@ -329,8 +379,8 @@ def test_hf_space_header_content() -> None:
|
|
| 329 |
assert hf_ui.FULL_TITLE == (
|
| 330 |
"Implicit Gaussian Decoding for Large-Baseline Monocular View Synthesis"
|
| 331 |
)
|
| 332 |
-
assert hf_ui.GITHUB_URL == "https://github.com/
|
| 333 |
-
assert hf_ui.PROJECT_PAGE_URL == "https://
|
| 334 |
assert hf_ui.INPUT_IMAGE_HINT == (
|
| 335 |
"Better for indoor scenes due to HyperSim-only training"
|
| 336 |
)
|
|
|
|
| 161 |
assert float(captured["gaussians"].mean_vectors.abs().max()) < 1.0
|
| 162 |
|
| 163 |
|
| 164 |
+
def test_filtered_ply_export_can_translate_z(tmp_path, monkeypatch) -> None:
|
| 165 |
+
artifact_path = tmp_path / "gaussians.pt"
|
| 166 |
+
gaussians = _gaussians()
|
| 167 |
+
save_gaussian_artifact(
|
| 168 |
+
GaussianArtifact(
|
| 169 |
+
gaussians=gaussians,
|
| 170 |
+
focal_length_px=1200.0,
|
| 171 |
+
image_shape=(1152, 1536),
|
| 172 |
+
),
|
| 173 |
+
artifact_path,
|
| 174 |
+
)
|
| 175 |
+
captured = {}
|
| 176 |
+
monkeypatch.setattr(hf_runtime, "filter_final_gaussian_floaters", lambda value: value)
|
| 177 |
+
|
| 178 |
+
def fake_save_ply(*, gaussians, path, **_) -> None:
|
| 179 |
+
captured["gaussians"] = gaussians
|
| 180 |
+
path.write_bytes(b"ply")
|
| 181 |
+
|
| 182 |
+
monkeypatch.setattr(
|
| 183 |
+
hf_runtime,
|
| 184 |
+
"save_ply",
|
| 185 |
+
fake_save_ply,
|
| 186 |
+
)
|
| 187 |
+
|
| 188 |
+
export_filtered_gaussian_ply(artifact_path, tmp_path / "output", z_offset=1.25)
|
| 189 |
+
|
| 190 |
+
expected = gaussians.mean_vectors + torch.tensor([0.0, 0.0, 1.25])
|
| 191 |
+
assert torch.equal(captured["gaussians"].mean_vectors, expected)
|
| 192 |
+
|
| 193 |
+
|
| 194 |
def test_build_splat_transform_command_is_fixed_and_non_interactive() -> None:
|
| 195 |
command = build_splat_transform_command(
|
| 196 |
scene_ply=Path("scene.ply"),
|
|
|
|
| 281 |
viewer_path = tmp_path / "viewer.html"
|
| 282 |
script_path = tmp_path / "index.js"
|
| 283 |
viewer_path.write_text("<html><head></head><body></body></html>")
|
| 284 |
+
script_path.write_text(
|
| 285 |
+
"window.firstFrame?.();\n"
|
| 286 |
+
" // Create the viewer\n"
|
| 287 |
+
" return new Viewer(global, gsplatLoad, skyboxLoad, collisionLoad);"
|
| 288 |
+
)
|
| 289 |
|
| 290 |
patch_supersplat_html_viewer_bridge(viewer_path, script_path)
|
| 291 |
|
| 292 |
assert "data-infinisplat-viewer-bridge" in viewer_path.read_text()
|
| 293 |
|
| 294 |
|
| 295 |
+
def test_supersplat_viewer_bridge_patches_camera_controls(tmp_path) -> None:
|
| 296 |
+
viewer_path = tmp_path / "viewer.html"
|
| 297 |
+
script_path = tmp_path / "index.js"
|
| 298 |
+
viewer_path.write_text("<html><head></head><body></body></html>")
|
| 299 |
+
script_path.write_text(
|
| 300 |
+
"window.firstFrame?.();\n"
|
| 301 |
+
" // Create the viewer\n"
|
| 302 |
+
" return new Viewer(global, gsplatLoad, skyboxLoad, collisionLoad);"
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
patch_supersplat_html_viewer_bridge(viewer_path, script_path)
|
| 306 |
+
|
| 307 |
+
assert "window.infinisplatViewerControls" in script_path.read_text()
|
| 308 |
+
assert 'event.data.action === "distance"' in viewer_path.read_text()
|
| 309 |
+
|
| 310 |
+
|
| 311 |
def test_supersplat_viewer_bridge_rejects_changed_bundle(tmp_path) -> None:
|
| 312 |
viewer_path = tmp_path / "viewer.html"
|
| 313 |
viewer_path.write_text("<html><head></head><body></body></html>")
|
|
|
|
| 379 |
assert hf_ui.FULL_TITLE == (
|
| 380 |
"Implicit Gaussian Decoding for Large-Baseline Monocular View Synthesis"
|
| 381 |
)
|
| 382 |
+
assert hf_ui.GITHUB_URL == "https://github.com/zju3dv/InfiniSplat"
|
| 383 |
+
assert hf_ui.PROJECT_PAGE_URL == "https://zju3dv.github.io/InfiniSplat"
|
| 384 |
assert hf_ui.INPUT_IMAGE_HINT == (
|
| 385 |
"Better for indoor scenes due to HyperSim-only training"
|
| 386 |
)
|
tests/test_webcam_frontend.py
CHANGED
|
@@ -10,6 +10,7 @@ def test_custom_frontend_exposes_camera_controls() -> None:
|
|
| 10 |
|
| 11 |
assert 'id="cameraButton"' in markup
|
| 12 |
assert 'id="cameraPreview"' in markup
|
|
|
|
| 13 |
assert 'id="captureButton"' in markup
|
| 14 |
assert 'id="cancelCameraButton"' in markup
|
| 15 |
assert 'name="cameraFormat" value="landscape"' in markup
|
|
@@ -22,14 +23,37 @@ def test_custom_frontend_captures_camera_as_jpeg_file() -> None:
|
|
| 22 |
script = (FRONTEND_ROOT / "app.js").read_text()
|
| 23 |
|
| 24 |
assert "navigator.mediaDevices.getUserMedia" in script
|
| 25 |
-
assert
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
assert "targetRatio = dimensions.width / dimensions.height" in script
|
| 27 |
assert "cropWidth = sourceHeight * targetRatio" in script
|
| 28 |
assert "cropHeight = sourceWidth / targetRatio" in script
|
|
|
|
| 29 |
assert "new File([blob]" in script
|
| 30 |
assert "stopCamera();" in script
|
| 31 |
|
| 32 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
def test_gradio_fallback_accepts_webcam_input() -> None:
|
| 34 |
source = (REPO_ROOT / "src/demo/hf_ui.py").read_text()
|
| 35 |
|
|
|
|
| 10 |
|
| 11 |
assert 'id="cameraButton"' in markup
|
| 12 |
assert 'id="cameraPreview"' in markup
|
| 13 |
+
assert 'id="cameraSelect"' in markup
|
| 14 |
assert 'id="captureButton"' in markup
|
| 15 |
assert 'id="cancelCameraButton"' in markup
|
| 16 |
assert 'name="cameraFormat" value="landscape"' in markup
|
|
|
|
| 23 |
script = (FRONTEND_ROOT / "app.js").read_text()
|
| 24 |
|
| 25 |
assert "navigator.mediaDevices.getUserMedia" in script
|
| 26 |
+
assert "navigator.mediaDevices.enumerateDevices" in script
|
| 27 |
+
assert 'deviceId: { exact: selectedCameraDeviceId }' in script
|
| 28 |
+
assert 'addEventListener?.("devicechange"' in script
|
| 29 |
+
assert 'width: { ideal: 2560 }' in script
|
| 30 |
+
assert 'height: { ideal: 1920 }' in script
|
| 31 |
+
assert 'aspectRatio: { ideal: 4 / 3 }' in script
|
| 32 |
+
assert 'new ImageCapture(track)' in script
|
| 33 |
+
assert 'advanced[mode] = "continuous"' in script
|
| 34 |
+
assert 'context.imageSmoothingQuality = "high"' in script
|
| 35 |
+
assert 'imageCapture.takePhoto(photoSettings || undefined)' in script
|
| 36 |
+
assert 'canvas.toBlob(resolve, "image/jpeg", 0.95)' in script
|
| 37 |
assert "targetRatio = dimensions.width / dimensions.height" in script
|
| 38 |
assert "cropWidth = sourceHeight * targetRatio" in script
|
| 39 |
assert "cropHeight = sourceWidth / targetRatio" in script
|
| 40 |
+
assert "will be enlarged; it may look soft" in script
|
| 41 |
assert "new File([blob]" in script
|
| 42 |
assert "stopCamera();" in script
|
| 43 |
|
| 44 |
|
| 45 |
+
def test_custom_frontend_exposes_view_and_export_depth_controls() -> None:
|
| 46 |
+
markup = (FRONTEND_ROOT / "index.html").read_text()
|
| 47 |
+
script = (FRONTEND_ROOT / "app.js").read_text()
|
| 48 |
+
|
| 49 |
+
assert 'id="cameraDistance" type="range"' in markup
|
| 50 |
+
assert 'id="resetViewButton"' in markup
|
| 51 |
+
assert 'id="bakeZOffset" type="checkbox"' in markup
|
| 52 |
+
assert 'id="zOffset" type="range"' in markup
|
| 53 |
+
assert 'type: "infinisplat:viewer-control"' in script
|
| 54 |
+
assert 'z_offset: bakeZOffset.checked ? Number(zOffset.value) : 0' in script
|
| 55 |
+
|
| 56 |
+
|
| 57 |
def test_gradio_fallback_accepts_webcam_input() -> None:
|
| 58 |
source = (REPO_ROOT / "src/demo/hf_ui.py").read_text()
|
| 59 |
|