Add portrait-aware webcam cropping

README.md

@@ -116,7 +116,7 @@ See [docs/inference.md](docs/inference.md) for camera parameters, output control
 
 ## 🖥️ Web Demo
 
-The local Gradio demo accepts uploaded images or photos captured from your webcam, and provides RGB reconstruction, staged PLY and standalone HTML downloads, and an interactive Gaussian viewer. After preparing the inference environment, launch it with:
+The local Gradio demo accepts uploaded images or landscape/portrait photos captured from your webcam, with an option to process exactly the visible camera frame. It also provides RGB reconstruction, staged PLY and standalone HTML downloads, and an interactive Gaussian viewer. After preparing the inference environment, launch it with:
 
 ```bash
 python demo.py

src/demo/infer_single_image.py

@@ -36,6 +36,13 @@ PROMPT_DEPTH_EPS = 1e-6
 PROMPT_DISPARITY_MIN_DEPTH = 1e-2
 
 
+def resolve_inference_shape(image_width: int, image_height: int) -> tuple[int, int]:
+    """Return (height, width), preserving landscape or portrait orientation."""
+    if image_height > image_width:
+        return INFERENCE_WIDTH, INFERENCE_HEIGHT
+    return INFERENCE_HEIGHT, INFERENCE_WIDTH
+
+
 @dataclass(frozen=True)
 class ResolvedIntrinsics:
     """Resolved intrinsics for one input image.
@@ -268,8 +275,10 @@ def load_demo_image_bundle(
             default_focal_35mm_mm=default_focal_35mm_mm,
         )
 
-        target_height = INFERENCE_HEIGHT
-        target_width = INFERENCE_WIDTH
+        target_height, target_width = resolve_inference_shape(
+            image_width=original_width,
+            image_height=original_height,
+        )
 
         inference_image_rgb = image_rgb
         inference_intrinsics = original_intrinsics

src/demo/server_frontend/app.js

@@ -5,7 +5,12 @@ const imagePreview = document.getElementById("imagePreview");
 const uploadSurface = document.getElementById("uploadSurface");
 const chooseImageButton = document.getElementById("chooseImageButton");
 const cameraButton = document.getElementById("cameraButton");
+const cameraStage = document.getElementById("cameraStage");
 const cameraPreview = document.getElementById("cameraPreview");
+const cameraOptions = document.getElementById("cameraOptions");
+const cameraFormatInputs = document.querySelectorAll('input[name="cameraFormat"]');
+const cropToFrame = document.getElementById("cropToFrame");
+const cameraOutputSize = document.getElementById("cameraOutputSize");
 const cameraActions = document.getElementById("cameraActions");
 const cancelCameraButton = document.getElementById("cancelCameraButton");
 const captureButton = document.getElementById("captureButton");
@@ -29,6 +34,10 @@ let running = false;
 let toastTimer = null;
 let cameraStream = null;
 const supportedImageTypes = new Set(["image/jpeg", "image/png", "image/webp"]);
+const cameraFormats = {
+  landscape: { width: 1536, height: 1152 },
+  portrait: { width: 1152, height: 1536 },
+};
 
 const colorScheme = window.matchMedia("(prefers-color-scheme: dark)");
 const requestedTheme = new URLSearchParams(window.location.search).get("__theme");
@@ -96,7 +105,7 @@ function setRunning(value) {
   examplesGrid.toggleAttribute("inert", value);
 }
 
-function selectFile(file) {
+function selectFile(file, fromCamera = false) {
   if (!file || !supportedImageTypes.has(file.type)) {
     showToast("Please choose a JPG, PNG, or WebP image.", true);
     return;
@@ -108,6 +117,7 @@ function selectFile(file) {
   imagePreview.src = previewUrl;
   imagePreview.hidden = false;
   chooseImageButton.hidden = true;
+  cameraButton.textContent = fromCamera ? "Retake photo" : "Use camera";
   reconstructButton.disabled = !client;
 }
 
@@ -115,7 +125,8 @@ function stopCamera() {
   for (const track of cameraStream?.getTracks() || []) track.stop();
   cameraStream = null;
   cameraPreview.srcObject = null;
-  cameraPreview.hidden = true;
+  cameraStage.hidden = true;
+  cameraOptions.hidden = true;
   cameraActions.hidden = true;
   imagePreview.hidden = !selectedFile;
   chooseImageButton.hidden = Boolean(selectedFile);
@@ -123,6 +134,20 @@ function stopCamera() {
   reconstructButton.disabled = running || !selectedFile || !client;
 }
 
+function selectedCameraFormat() {
+  return document.querySelector('input[name="cameraFormat"]:checked')?.value || "landscape";
+}
+
+function updateCameraFrame() {
+  const format = selectedCameraFormat();
+  const dimensions = cameraFormats[format];
+  cameraStage.dataset.format = format;
+  cameraStage.dataset.crop = cropToFrame.checked ? "frame" : "full";
+  cameraOutputSize.textContent = cropToFrame.checked
+    ? `Output crop: ${dimensions.width} × ${dimensions.height}`
+    : "Output: full camera image";
+}
+
 async function startCamera() {
   if (running || cameraStream) return;
   if (!navigator.mediaDevices?.getUserMedia) {
@@ -139,7 +164,9 @@ async function startCamera() {
     cameraPreview.srcObject = cameraStream;
     imagePreview.hidden = true;
     chooseImageButton.hidden = true;
-    cameraPreview.hidden = false;
+    updateCameraFrame();
+    cameraStage.hidden = false;
+    cameraOptions.hidden = false;
     cameraActions.hidden = false;
     reconstructButton.disabled = true;
     await cameraPreview.play();
@@ -159,9 +186,9 @@ async function startCamera() {
 }
 
 async function capturePhoto() {
-  const width = cameraPreview.videoWidth;
-  const height = cameraPreview.videoHeight;
-  if (!cameraStream || !width || !height) {
+  const sourceWidth = cameraPreview.videoWidth;
+  const sourceHeight = cameraPreview.videoHeight;
+  if (!cameraStream || !sourceWidth || !sourceHeight) {
     showToast("The camera is still starting. Try again in a moment.", true);
     return;
   }
@@ -169,13 +196,49 @@ async function capturePhoto() {
   captureButton.disabled = true;
   try {
     const canvas = document.createElement("canvas");
-    canvas.width = width;
-    canvas.height = height;
-    canvas.getContext("2d").drawImage(cameraPreview, 0, 0, width, height);
+    let sourceX = 0;
+    let sourceY = 0;
+    let cropWidth = sourceWidth;
+    let cropHeight = sourceHeight;
+
+    if (cropToFrame.checked) {
+      const dimensions = cameraFormats[selectedCameraFormat()];
+      const targetRatio = dimensions.width / dimensions.height;
+      const sourceRatio = sourceWidth / sourceHeight;
+      canvas.width = dimensions.width;
+      canvas.height = dimensions.height;
+      if (sourceRatio > targetRatio) {
+        cropWidth = sourceHeight * targetRatio;
+        sourceX = (sourceWidth - cropWidth) / 2;
+      } else {
+        cropHeight = sourceWidth / targetRatio;
+        sourceY = (sourceHeight - cropHeight) / 2;
+      }
+    } else {
+      canvas.width = sourceWidth;
+      canvas.height = sourceHeight;
+    }
+
+    const context = canvas.getContext("2d");
+    if (!context) throw new Error("The browser could not create an image canvas.");
+    context.drawImage(
+      cameraPreview,
+      sourceX,
+      sourceY,
+      cropWidth,
+      cropHeight,
+      0,
+      0,
+      canvas.width,
+      canvas.height,
+    );
     const blob = await new Promise((resolve) => canvas.toBlob(resolve, "image/jpeg", 0.92));
     if (!blob) throw new Error("The browser could not encode the captured photo.");
     stopCamera();
-    selectFile(new File([blob], `webcam-${Date.now()}.jpg`, { type: "image/jpeg" }));
+    selectFile(
+      new File([blob], `webcam-${Date.now()}.jpg`, { type: "image/jpeg" }),
+      true,
+    );
   } catch (error) {
     console.error(error);
     showToast("Could not capture the photo. Please try again.", true);
@@ -198,6 +261,8 @@ imagePreview.addEventListener("keydown", (event) => {
 cameraButton.addEventListener("click", startCamera);
 cancelCameraButton.addEventListener("click", stopCamera);
 captureButton.addEventListener("click", capturePhoto);
+for (const input of cameraFormatInputs) input.addEventListener("change", updateCameraFrame);
+cropToFrame.addEventListener("change", updateCameraFrame);
 window.addEventListener("pagehide", stopCamera);
 
 if (!navigator.mediaDevices?.getUserMedia) {

src/demo/server_frontend/index.html

@@ -13,7 +13,7 @@
         ? requestedTheme
         : (prefersDark ? "dark" : "light");
   </script>
-  <link rel="stylesheet" href="/infinisplat/assets/styles.css?v=5">
+  <link rel="stylesheet" href="/infinisplat/assets/styles.css?v=6">
 </head>
 <body>
   <main class="app-shell">
@@ -41,11 +41,31 @@
           <div class="upload-surface" id="uploadSurface">
             <input id="imageInput" type="file" accept="image/jpeg,image/png,image/webp">
             <img id="imagePreview" alt="Selected input; activate to choose another image" role="button" tabindex="0" hidden>
-            <video id="cameraPreview" aria-label="Live camera preview" autoplay playsinline muted hidden></video>
+            <div class="camera-stage" id="cameraStage" data-format="landscape" hidden>
+              <video id="cameraPreview" aria-label="Live camera preview" autoplay playsinline muted></video>
+            </div>
             <button class="upload-empty" id="chooseImageButton" type="button">
               <strong>Choose an image</strong>
               <small>JPG, PNG, or WebP</small>
             </button>
+            <div class="camera-options" id="cameraOptions" hidden>
+              <fieldset class="format-control">
+                <legend>Camera format</legend>
+                <label>
+                  <input type="radio" name="cameraFormat" value="landscape" checked>
+                  <span>Landscape 4:3</span>
+                </label>
+                <label>
+                  <input type="radio" name="cameraFormat" value="portrait">
+                  <span>Portrait 3:4</span>
+                </label>
+              </fieldset>
+              <label class="crop-control">
+                <input id="cropToFrame" type="checkbox" checked>
+                <span>Process visible frame only</span>
+              </label>
+              <small id="cameraOutputSize" aria-live="polite">Output crop: 1536 × 1152</small>
+            </div>
             <div class="camera-actions" id="cameraActions" hidden>
               <button class="camera-cancel" id="cancelCameraButton" type="button">Cancel</button>
               <button class="camera-capture" id="captureButton" type="button">Take photo</button>
@@ -91,6 +111,6 @@
   </main>
 
   <div class="toast" id="toast" role="status" aria-live="polite"></div>
-  <script type="module" src="/infinisplat/assets/app.js?v=6"></script>
+  <script type="module" src="/infinisplat/assets/app.js?v=7"></script>
 </body>
 </html>

src/demo/server_frontend/styles.css

@@ -267,17 +267,41 @@ button, input { font: inherit; }
   opacity: 0;
 }
 
-.upload-surface img,
-.upload-surface video {
+.upload-surface img {
   width: 100%;
   height: 510px;
   object-fit: contain;
   background: var(--surface-muted);
 }
 
-.upload-surface video { object-fit: cover; }
 .upload-surface img { cursor: pointer; }
 
+.camera-stage {
+  position: relative;
+  width: min(100%, 680px);
+  max-width: 100%;
+  overflow: hidden;
+  aspect-ratio: 4 / 3;
+  background: #050706;
+  box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.16);
+}
+
+.camera-stage[data-format="portrait"] {
+  width: auto;
+  height: 510px;
+  aspect-ratio: 3 / 4;
+}
+
+.camera-stage video {
+  display: block;
+  width: 100%;
+  height: 100%;
+  object-fit: cover;
+  object-position: center;
+}
+
+.camera-stage[data-crop="full"] video { object-fit: contain; }
+
 .upload-empty {
   display: flex;
   flex-direction: column;
@@ -293,10 +317,85 @@ button, input { font: inherit; }
 .upload-empty strong { color: var(--text); font-size: 0.9rem; }
 .upload-empty small { font-size: 0.75rem; }
 
+.camera-options {
+  position: absolute;
+  top: 12px;
+  left: 50%;
+  z-index: 2;
+  display: grid;
+  justify-items: center;
+  gap: 7px;
+  width: min(94%, 390px);
+  padding: 8px;
+  border: 1px solid var(--border-strong);
+  border-radius: 8px;
+  background: color-mix(in srgb, var(--surface) 90%, transparent);
+  box-shadow: var(--panel-shadow);
+  color: var(--text);
+  transform: translateX(-50%);
+  backdrop-filter: blur(8px);
+}
+
+.format-control {
+  display: grid;
+  grid-template-columns: 1fr 1fr;
+  width: 100%;
+  margin: 0;
+  padding: 0;
+  border: 0;
+}
+
+.format-control legend {
+  position: absolute;
+  width: 1px;
+  height: 1px;
+  overflow: hidden;
+  clip: rect(0 0 0 0);
+  white-space: nowrap;
+}
+
+.format-control label { cursor: pointer; }
+.format-control input { position: absolute; opacity: 0; }
+
+.format-control span {
+  display: grid;
+  place-items: center;
+  min-height: 34px;
+  padding: 0 10px;
+  border: 1px solid var(--border-strong);
+  background: var(--surface);
+  color: var(--muted);
+  font-size: 0.74rem;
+  font-weight: 650;
+}
+
+.format-control label:first-of-type span { border-radius: 5px 0 0 5px; }
+.format-control label:last-of-type span { border-left: 0; border-radius: 0 5px 5px 0; }
+.format-control input:checked + span {
+  border-color: var(--primary);
+  background: var(--ready-bg);
+  color: var(--ready-text);
+}
+.format-control input:focus-visible + span { outline: 2px solid var(--primary); outline-offset: 2px; }
+
+.crop-control {
+  display: flex;
+  align-items: center;
+  gap: 7px;
+  color: var(--text);
+  font-size: 0.73rem;
+  font-weight: 600;
+  cursor: pointer;
+}
+
+.crop-control input { accent-color: var(--primary); }
+.camera-options small { color: var(--muted); font-size: 0.67rem; }
+
 .camera-actions {
   position: absolute;
   bottom: 16px;
   left: 50%;
+  z-index: 2;
   display: flex;
   gap: 8px;
   padding: 7px;
@@ -545,7 +644,9 @@ button, input { font: inherit; }
   .app-shell { width: calc(100% - 28px); }
   .workspace { grid-template-columns: 1fr; }
   .upload-surface { min-height: 420px; }
-  .upload-surface img, .upload-surface video, .viewer-shell { height: 420px; }
+  .upload-surface img, .viewer-shell { height: 420px; }
+  .camera-stage[data-format="landscape"] { width: min(100%, 560px); }
+  .camera-stage[data-format="portrait"] { height: 420px; }
   .examples-grid { grid-template-columns: repeat(6, minmax(0, 1fr)); }
   .example-item { grid-column: span 2 !important; }
 }
@@ -561,7 +662,9 @@ button, input { font: inherit; }
   .panel-content { padding: 9px; }
   .source-actions, .download-bar { padding: 0 9px 9px; }
   .upload-surface { min-height: 360px; }
-  .upload-surface img, .upload-surface video, .viewer-shell { height: 360px; }
+  .upload-surface img, .viewer-shell { height: 360px; }
+  .camera-stage[data-format="landscape"] { width: min(100%, 480px); }
+  .camera-stage[data-format="portrait"] { height: 360px; }
   .examples-section { margin-bottom: 24px; padding: 0; }
   .examples-grid { grid-template-columns: repeat(4, minmax(0, 1fr)); grid-auto-rows: 132px; }
   .example-item { grid-column: span 2 !important; }

tests/test_public_configs.py

@@ -4,7 +4,11 @@ from hydra import compose, initialize_config_dir
 from hydra.core.global_hydra import GlobalHydra
 
 from src.demo.config import load_typed_root_config
-from src.demo.infer_single_image import INFERENCE_HEIGHT, INFERENCE_WIDTH
+from src.demo.infer_single_image import (
+    INFERENCE_HEIGHT,
+    INFERENCE_WIDTH,
+    resolve_inference_shape,
+)
 
 
 def _load_experiment(name: str):
@@ -45,5 +49,11 @@ def test_lidar_experiment_loads() -> None:
     assert cfg.model.encoder.name == "infinisplat_infinidepth"
 
 
-def test_inference_resolution_is_hardcoded() -> None:
+def test_landscape_inference_resolution_is_hardcoded() -> None:
     assert (INFERENCE_HEIGHT, INFERENCE_WIDTH) == (1152, 1536)
+
+
+def test_inference_resolution_preserves_orientation() -> None:
+    assert resolve_inference_shape(image_width=1600, image_height=900) == (1152, 1536)
+    assert resolve_inference_shape(image_width=900, image_height=1600) == (1536, 1152)
+    assert resolve_inference_shape(image_width=1000, image_height=1000) == (1152, 1536)

tests/test_webcam_frontend.py

@@ -12,6 +12,9 @@ def test_custom_frontend_exposes_camera_controls() -> None:
     assert 'id="cameraPreview"' in markup
     assert 'id="captureButton"' in markup
     assert 'id="cancelCameraButton"' in markup
+    assert 'name="cameraFormat" value="landscape"' in markup
+    assert 'name="cameraFormat" value="portrait"' in markup
+    assert 'id="cropToFrame" type="checkbox" checked' in markup
     assert "autoplay playsinline muted" in markup
 
 
@@ -20,6 +23,9 @@ def test_custom_frontend_captures_camera_as_jpeg_file() -> None:
 
     assert "navigator.mediaDevices.getUserMedia" in script
     assert 'canvas.toBlob(resolve, "image/jpeg", 0.92)' in script
+    assert "targetRatio = dimensions.width / dimensions.height" in script
+    assert "cropWidth = sourceHeight * targetRatio" in script
+    assert "cropHeight = sourceWidth / targetRatio" in script
     assert "new File([blob]" in script
     assert "stopCamera();" in script