CodecSplat demo

.gitattributes

@@ -33,3 +33,53 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/00.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/01.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/02.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/03.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/04.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/05.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/06.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/07.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/08.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/09.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/10.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/11.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/12.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/13.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/14.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/15.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/16.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/17.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/18.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/19.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/20.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/21.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/22.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/23.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_a/24.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/00.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/01.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/02.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/03.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/04.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/05.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/06.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/07.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/08.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/09.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/10.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/11.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/12.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/13.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/14.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/15.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/16.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/17.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/18.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/19.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/20.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/21.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/22.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/23.jpg filter=lfs diff=lfs merge=lfs -text
+examples/scene_b/24.jpg filter=lfs diff=lfs merge=lfs -text

LICENSE-CodecSplat

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Pengpeng Yu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,13 +1,50 @@
 ---
-title: Codecsplat
-emoji: 👀
-colorFrom: green
-colorTo: purple
+title: CodecSplat
+emoji: 🗜️
+colorFrom: red
+colorTo: yellow
 sdk: gradio
 sdk_version: 6.30.0
-python_version: '3.12'
 app_file: app.py
-pinned: false
+short_description: Compress photos into a tiny 3D Gaussian splat bitstream
+python_version: "3.12"
+startup_duration_timeout: 1h
+models:
+  - pengpeng-yu/CodecSplat
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# CodecSplat: Ultra-Compact Latent Coding for Feed-Forward 3D Gaussian Splatting
+
+Demo of [**CodecSplat**](https://huggingface.co/papers/2605.25563) (Yu et al.). CodecSplat
+entropy-codes the 2D features that a feed-forward 3DGS model (DepthSplat-style) uses for depth
+and Gaussian prediction. A scene becomes a bitstream of a few tens of KB, which is then decoded
+into renderable 3D Gaussians.
+
+How the Space works:
+
+1. **COLMAP** (`pycolmap`, CPU) estimates intrinsics and camera poses for the uploaded photos.
+   The reconstruction is normalised and undistorted to PINHOLE.
+2. 8 context views are picked by farthest-point sampling over the cameras. They are resized and
+   center-cropped to 256×448 with the authors' crop shim.
+3. CodecSplat (`codecsplat-base-dl3dv-256x448-view8`, the authors' DL3DV evaluation preset)
+   runs the **complete bitstream path** from `ModelWrapper.test_step`: encode context →
+   compress to a real bitstream at the chosen λ → decompress → predict Gaussians, using bf16
+   autocast like `trainer.precision=bf16-mixed`.
+4. The authors' gsplat decoder (antialiased, eps2d=0.1) renders held-out views (PSNR is reported
+   against the real photos) and a fly-through video. The decoded Gaussians are shown in an
+   interactive `gr.Model3D` splat viewer, and the `.bin` bitstream is available to download.
+
+Notes:
+* gsplat's CUDA kernels are JIT-compiled at startup against the CUDA toolkit in the ZeroGPU
+  image, following `hugging-apps/resplat-recurrent-gs`.
+* The viewer gets a compact copy of the Gaussians: DC colour only, near-transparent splats
+  pruned, re-centred so the scene is framed.
+
+Code: https://github.com/pengpeng-yu/CodecSplat (MIT) · Weights: https://huggingface.co/pengpeng-yu/CodecSplat (MIT)
+
+## Example scenes
+
+The example frames are from the DL3DV demo scene that the DepthSplat authors redistribute in
+[`haofeixu/depthsplat`](https://huggingface.co/datasets/haofeixu/depthsplat) (MIT-labelled).
+The imagery itself comes from [DL3DV-10K](https://dl3dv-10k.github.io/DL3DV-10K/), credit to
+the DL3DV authors.

app.py

@@ -0,0 +1,338 @@
+"""CodecSplat: Ultra-Compact Latent Coding for Feed-Forward 3D Gaussian Splatting.
+
+Upload overlapping photos of a static scene. The Space runs COLMAP (pycolmap, CPU) to get
+camera poses, then CodecSplat (authors' DL3DV 8-view 256x448 checkpoint):
+  context views -> 2D latent features -> learned entropy coder -> real bitstream
+  -> decoder -> 3D Gaussians -> gsplat rendering.
+You get an interactive splat viewer, a fly-through video, held-out novel views, the
+size of the compressed scene bitstream (and the bitstream itself).
+"""
+
+from __future__ import annotations
+
+import ctypes
+import glob
+import os
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+APP_DIR = Path(__file__).parent.resolve()
+sys.path.insert(0, str(APP_DIR))
+
+os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "12.0")
+os.environ.setdefault("MAX_JOBS", str(min(32, (os.cpu_count() or 8))))
+
+import spaces  # noqa: E402  (must precede any CUDA-touching import)
+import torch  # noqa: E402
+
+torch.set_float32_matmul_precision("high")
+
+
+# ---------------------------------------------------------------------------
+# 1. gsplat CUDA backend: JIT-compile once at startup against the CUDA toolkit
+#    shipped in the ZeroGPU image (same approach as hugging-apps/resplat-recurrent-gs).
+# ---------------------------------------------------------------------------
+def _pick_cuda_home() -> str | None:
+    torch_cuda = (torch.version.cuda or "").split(".")[0]
+    candidates: list[str] = []
+    for pattern in ("/cuda-image/usr/local/cuda-*", "/usr/local/cuda-*"):
+        candidates += sorted(glob.glob(pattern), reverse=True)
+    candidates += ["/cuda-image/usr/local/cuda", "/usr/local/cuda"]
+    usable = [c for c in candidates if os.path.isfile(os.path.join(c, "bin", "nvcc"))]
+    if not usable:
+        return None
+    for c in usable:
+        if torch_cuda and f"cuda-{torch_cuda}" in c:
+            return c
+    return usable[0]
+
+
+CUDA_HOME = _pick_cuda_home()
+print(f"[setup] torch {torch.__version__} (cuda {torch.version.cuda}) | CUDA_HOME={CUDA_HOME}", flush=True)
+if CUDA_HOME:
+    os.environ["CUDA_HOME"] = CUDA_HOME
+    os.environ["CUDA_PATH"] = CUDA_HOME
+    os.environ["PATH"] = os.path.join(CUDA_HOME, "bin") + os.pathsep + os.environ.get("PATH", "")
+    _libdir = os.path.join(CUDA_HOME, "lib64")
+    os.environ["LD_LIBRARY_PATH"] = _libdir + os.pathsep + os.environ.get("LD_LIBRARY_PATH", "")
+    for _so in sorted(glob.glob(os.path.join(_libdir, "libcudart.so*")), reverse=True):
+        try:
+            ctypes.CDLL(_so, mode=ctypes.RTLD_GLOBAL)
+            break
+        except OSError:
+            continue
+
+import torch.utils.cpp_extension as _cppext  # noqa: E402
+
+_cppext._check_cuda_version = lambda *a, **k: None
+
+print("[setup] compiling gsplat CUDA kernels (one-off, several minutes)...", flush=True)
+_t0 = time.time()
+from gsplat.cuda._backend import _C as _GSPLAT_C  # noqa: E402
+
+if _GSPLAT_C is None:
+    raise RuntimeError(f"gsplat CUDA backend unavailable (CUDA_HOME={CUDA_HOME}).")
+print(f"[setup] gsplat backend ready in {time.time() - _t0:.1f}s", flush=True)
+
+
+# ---------------------------------------------------------------------------
+# 2. Model
+# ---------------------------------------------------------------------------
+import imageio.v2 as imageio  # noqa: E402
+import numpy as np  # noqa: E402
+import gradio as gr  # noqa: E402
+import pycolmap  # noqa: E402
+from huggingface_hub import hf_hub_download  # noqa: E402
+from PIL import Image  # noqa: E402
+
+import codecsplat_pipeline as cp  # noqa: E402
+from colmap_sfm import IMAGE_EXT, run_sfm  # noqa: E402
+
+MODEL_REPO = "pengpeng-yu/CodecSplat"
+CHECKPOINT_FILE = "codecsplat-base-dl3dv-256x448-view8-075062d0.pth"
+NUM_CONTEXT = 8
+NUM_TARGET = 8
+MAX_INPUT_IMAGES = 60
+
+print("[setup] downloading checkpoint...", flush=True)
+ckpt_path = hf_hub_download(MODEL_REPO, CHECKPOINT_FILE)
+print("[setup] building CodecSplat...", flush=True)
+ENCODER = cp.build_encoder(ckpt_path).to("cuda")
+DECODER = cp.build_decoder().to("cuda")
+print("[setup] model ready", flush=True)
+
+
+def _u8(t: torch.Tensor) -> np.ndarray:
+    return (t.clamp(0, 1).permute(1, 2, 0).float().cpu().numpy() * 255).round().astype(np.uint8)
+
+
+# ---------------------------------------------------------------------------
+# 3. GPU part: encode -> bitstream -> decode -> render
+# ---------------------------------------------------------------------------
+@spaces.GPU(duration=60)
+def codecsplat_gpu(image_paths: list[str], c2w: np.ndarray, K: np.ndarray, lmb: float = 1024.0):
+    ctx_idx, tgt_idx = cp.select_views(c2w, NUM_CONTEXT, NUM_TARGET)
+    batch = cp.to_device(cp.build_example(image_paths, c2w, K, ctx_idx, tgt_idx), "cuda")
+    h, w = batch["context"]["image"].shape[-2:]
+
+    torch.cuda.synchronize()
+    t0 = time.perf_counter()
+    out = cp.encode_decode(ENCODER, batch, lmb)
+    torch.cuda.synchronize()
+    t_codec = time.perf_counter() - t0
+    g = out["gaussians"]
+
+    # held-out novel views (+ PSNR against the real photos)
+    renders, score = None, float("nan")
+    if len(tgt_idx) > 0:
+        renders = cp.render(DECODER, g, batch["target"]["extrinsics"][0],
+                            batch["target"]["intrinsics"][0], (h, w))
+        score = cp.psnr(batch["target"]["image"][0].float(), renders)
+
+    # smooth fly-through through the context cameras
+    traj_c2w, traj_K = cp.interpolate_trajectory(
+        batch["context"]["extrinsics"][0], batch["context"]["intrinsics"][0]
+    )
+    video = cp.render(DECODER, g, traj_c2w.cuda(), traj_K.cuda(), (h, w))
+    t_total = time.perf_counter() - t0
+
+    gauss = {
+        "means": g.means[0].float().cpu(),
+        "scales": g.scales[0].float().cpu(),
+        "rotations": g.rotations[0].float().cpu(),
+        "harmonics": g.harmonics[0].float().cpu(),
+        "opacities": g.opacities[0].float().cpu(),
+    }
+    return {
+        "bitstream": out["bitstream"],
+        "num_bytes": float(out["codec_num_bytes"]),
+        "camera_bytes": float(out["codec_camera_num_bytes"]),
+        "feature_bytes": float(out["codec_feature_num_bytes"]),
+        "gaussians": gauss,
+        "context_c2w": batch["context"]["extrinsics"][0].cpu(),
+        "context_images": [_u8(x) for x in batch["context"]["image"][0]],
+        "target_images": [_u8(x) for x in batch["target"]["image"][0]],
+        "renders": [] if renders is None else [_u8(x) for x in renders],
+        "video": [_u8(x) for x in video],
+        "psnr": score,
+        "t_codec": t_codec,
+        "t_total": t_total,
+        "hw": (int(h), int(w)),
+        "ctx_idx": ctx_idx.tolist(),
+        "tgt_idx": tgt_idx.tolist(),
+    }
+
+
+# ---------------------------------------------------------------------------
+# 4. Full pipeline (CPU COLMAP -> GPU CodecSplat -> CPU export)
+# ---------------------------------------------------------------------------
+def _read_colmap_scene(scene_dir: str):
+    rec = pycolmap.Reconstruction(os.path.join(scene_dir, "sparse"))
+    items = []
+    for _, img in rec.images.items():
+        cam = rec.cameras[img.camera_id]
+        p = cam.params
+        if cam.model.name == "PINHOLE":
+            fx, fy, cx, cy = p[:4]
+        elif cam.model.name == "SIMPLE_PINHOLE":
+            fx = fy = p[0]
+            cx, cy = p[1:3]
+        else:
+            continue
+        K = np.eye(3, dtype=np.float32)
+        K[0, 0], K[1, 1] = fx / cam.width, fy / cam.height
+        K[0, 2], K[1, 2] = cx / cam.width, cy / cam.height
+        w2c = np.eye(4, dtype=np.float64)
+        w2c[:3, :4] = img.cam_from_world().matrix() if callable(img.cam_from_world) else img.cam_from_world.matrix()
+        path = os.path.join(scene_dir, "images", img.name)
+        if os.path.exists(path):
+            items.append((img.name, path, np.linalg.inv(w2c).astype(np.float32), K))
+    items.sort(key=lambda x: x[0])
+    if len(items) < 2:
+        raise gr.Error("COLMAP registered fewer than 2 usable images.")
+    return [i[1] for i in items], np.stack([i[2] for i in items]), np.stack([i[3] for i in items])
+
+
+def reconstruct(images, lmb: float = 1024.0, matcher: str = "exhaustive",
+                progress=gr.Progress(track_tqdm=False)):
+    """Reconstruct a compressed 3D Gaussian scene from multi-view photos with CodecSplat.
+
+    Args:
+        images: 3-60 overlapping photos of one static scene (camera poses are estimated with COLMAP).
+        lmb: rate-distortion trade-off lambda (16 = smallest bitstream, 1024 = best quality).
+        matcher: COLMAP matching mode, "exhaustive" (unordered photos) or "sequential" (video frames).
+
+    Returns:
+        Interactive splat (.ply), fly-through video, held-out novel views (render vs. photo),
+        selected input views, the scene bitstream (.bin) and a summary.
+    """
+    if not images:
+        raise gr.Error("Please upload at least 3 overlapping photos of the same scene.")
+    if isinstance(images, (str, Path)):
+        images = [images]
+    paths = [str(im if isinstance(im, (str, Path)) else getattr(im, "name", im)) for im in images]
+    paths = sorted([p for p in paths if p.lower().endswith(IMAGE_EXT)], key=os.path.basename)
+    if len(paths) < 3:
+        raise gr.Error(f"Got {len(paths)} usable image(s); COLMAP needs at least 3 overlapping photos.")
+    note = ""
+    if len(paths) > MAX_INPUT_IMAGES:
+        paths = [paths[i] for i in np.linspace(0, len(paths) - 1, MAX_INPUT_IMAGES).astype(int)]
+        note = f" (sub-sampled to {MAX_INPUT_IMAGES} images)"
+
+    t_start = time.time()
+    workdir = Path(tempfile.mkdtemp(prefix="codecsplat_"))
+    progress(0.05, desc=f"COLMAP on {len(paths)} images")
+    try:
+        sfm = run_sfm(paths, workdir, max_image_size=1024, matcher=matcher,
+                      log=lambda m: print(m, flush=True))
+    except ValueError as exc:
+        raise gr.Error(str(exc)) from exc
+    t_sfm = time.time() - t_start
+    img_paths, c2w, K = _read_colmap_scene(sfm["scene_dir"])
+
+    progress(0.55, desc="CodecSplat: encode -> bitstream -> decode -> render")
+    r = codecsplat_gpu(img_paths, c2w, K, float(lmb))
+
+    progress(0.85, desc="Exporting Gaussians")
+    out = workdir / "out"
+    out.mkdir(exist_ok=True)
+    bin_path = out / f"codecsplat_scene_lmb{int(round(lmb))}.bin"
+    bin_path.write_bytes(r["bitstream"])
+    viewer_ply = cp.export_gaussians(r["gaussians"], r["context_c2w"], out)
+
+    video_path = str(out / "flythrough.mp4")
+    imageio.mimwrite(video_path, r["video"], fps=24, quality=8, macro_block_size=16)
+
+    gallery_ctx = [(im, f"input view {i}") for im, i in zip(r["context_images"], r["ctx_idx"])]
+    gallery_nv = []
+    for rend, gt, i in zip(r["renders"], r["target_images"], r["tgt_idx"]):
+        gallery_nv.append((np.concatenate([rend, gt], axis=1), f"held-out view {i}: render | photo"))
+
+    n_g = int(r["gaussians"]["means"].shape[0])
+    raw_bytes = n_g * (3 + 3 + 4 + 1 + r["gaussians"]["harmonics"][0].numel()) * 4
+    kb = r["num_bytes"] / 1024
+    summary = (
+        f"**Compressed scene: {kb:.1f} KB** (features {r['feature_bytes']/1024:.1f} KB + cameras "
+        f"{r['camera_bytes']/1024:.2f} KB) at λ = {lmb:g} — decodes to **{n_g:,} Gaussians** "
+        f"(≈{raw_bytes/2**20:.0f} MB as raw float32, **{raw_bytes/max(r['num_bytes'],1):,.0f}× smaller**).\n\n"
+        f"Held-out novel views: **PSNR {r['psnr']:.2f} dB** · {len(r['ctx_idx'])} input views at "
+        f"{r['hw'][0]}×{r['hw'][1]} · COLMAP registered {sfm['num_registered']}/{sfm['num_input']} "
+        f"images{note} in {t_sfm:.1f}s · CodecSplat encode+decode {r['t_codec']:.2f}s · "
+        f"total {time.time() - t_start:.1f}s"
+    )
+    return viewer_ply, video_path, gallery_nv, gallery_ctx, str(bin_path), summary
+
+
+# ---------------------------------------------------------------------------
+# 5. UI
+# ---------------------------------------------------------------------------
+EX_DIR = APP_DIR / "examples"
+EXAMPLES = [sorted(str(p) for p in (EX_DIR / s).glob("*.jpg")) for s in ("scene_a", "scene_b")]
+
+DESCRIPTION = """
+# CodecSplat: Ultra-Compact Latent Coding for Feed-Forward 3D Gaussian Splatting
+
+CodecSplat turns a few photos into 3D Gaussians **through a real compressed bitstream**: the 2D
+features used for depth and Gaussian prediction are entropy-coded (tens of KB per scene), then
+decoded into a renderable 3D Gaussian scene. Upload overlapping photos of a static scene —
+camera poses are estimated with COLMAP right here — and pick a rate (λ).
+
+[Paper](https://huggingface.co/papers/2605.25563) · [Code](https://github.com/pengpeng-yu/CodecSplat) · [Weights](https://huggingface.co/pengpeng-yu/CodecSplat)
+"""
+
+CSS = """
+#col-container { max-width: 1280px; margin: 0 auto; }
+.dark .gradio-container { color: var(--body-text-color); }
+"""
+
+with gr.Blocks(title="CodecSplat") as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.Markdown(DESCRIPTION)
+        with gr.Row():
+            with gr.Column(scale=1):
+                image_files = gr.File(
+                    label="Photos of one static scene (8-30 overlapping shots work best)",
+                    file_count="multiple", file_types=["image"], type="filepath",
+                )
+                lmb = gr.Slider(16, 1024, value=1024, step=1, label="Rate λ",
+                                info="16 = smallest bitstream, 1024 = highest quality")
+                run_btn = gr.Button("Compress & reconstruct", variant="primary")
+                with gr.Accordion("Advanced options", open=False):
+                    matcher = gr.Radio(["exhaustive", "sequential"], value="exhaustive",
+                                       label="COLMAP matching",
+                                       info="exhaustive = unordered photos; sequential = video frames")
+                gr.Markdown(
+                    "8 input views are picked by farthest-point sampling over the COLMAP cameras and "
+                    "resized/cropped to 256×448 (the checkpoint's training resolution); up to 8 other "
+                    "photos are held out to measure novel-view quality.\n\n"
+                    "**Viewer**: drag to orbit, right-drag to pan, scroll to zoom."
+                )
+            with gr.Column(scale=2):
+                summary = gr.Markdown()
+                splat = gr.Model3D(label="Decoded 3D Gaussians (interactive)", height=440,
+                                   clear_color=[0.0, 0.0, 0.0, 1.0])
+                video = gr.Video(label="Fly-through rendered from the decoded Gaussians", autoplay=True, loop=True)
+                bin_out = gr.File(label="Compressed scene bitstream (.bin)")
+        novel = gr.Gallery(label="Held-out novel views: render | real photo", columns=2, height=320,
+                           object_fit="contain")
+        inputs_gal = gr.Gallery(label="Selected input views (after crop)", columns=4, height=200,
+                                object_fit="contain")
+
+        outputs = [splat, video, novel, inputs_gal, bin_out, summary]
+        run_btn.click(reconstruct, inputs=[image_files, lmb, matcher], outputs=outputs,
+                      api_name="reconstruct")
+
+        gr.Examples(
+            examples=[[EXAMPLES[0], 1024], [EXAMPLES[1], 64]],
+            inputs=[image_files, lmb],
+            outputs=outputs,
+            fn=reconstruct,
+            cache_examples=True,
+            cache_mode="lazy",
+            label="Frames from the DL3DV demo scene (via the DepthSplat authors)",
+        )
+
+if __name__ == "__main__":
+    demo.queue(max_size=12).launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True, show_error=True)

codecsplat_pipeline.py

@@ -0,0 +1,336 @@
+"""CodecSplat inference helpers for the Space.
+
+Everything here mirrors the authors' evaluation path
+(`scripts/test_dl3dv_view8_256x448.sh` -> `src/main.py` -> `ModelWrapper.test_step`):
+the encoder runs the *complete* learned-codec path (encode context -> entropy-code the
+2D latent features into a real bitstream -> decode the bitstream -> predict Gaussians),
+and the gsplat decoder renders novel views, all under bf16 autocast like
+`trainer.precision=bf16-mixed`.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from types import SimpleNamespace
+
+import numpy as np
+import torch
+from dacite import Config as DaciteConfig, from_dict
+from einops import rearrange, repeat
+from omegaconf import OmegaConf
+from plyfile import PlyData, PlyElement
+from scipy.spatial.transform import Rotation as R
+from scipy.spatial.transform import Slerp
+
+from src.dataset.shims.crop_shim import apply_crop_shim
+from src.dataset.shims.patch_shim import apply_patch_shim
+from src.misc.sh_rotation import rotate_sh
+from src.model.encoder import get_encoder
+from src.model.encoder.encoder_codecsplat import EncoderCodecSplatCfg
+
+APP_DIR = Path(__file__).parent.resolve()
+
+# ---- authors' DL3DV 8-view 256x448 evaluation preset ---------------------------------
+IMAGE_SHAPE = (256, 448)
+NEAR, FAR = 0.5, 200.0  # config/experiment/dl3dv.yaml
+ENCODER_OVERRIDES = {
+    "num_depth_candidates": 128,
+    "shim_patch_size": 16,
+    "num_scales": 2,
+    "upsample_factor": 4,
+    "lowest_feature_resolution": 8,
+    "monodepth_vit_type": "vitb",
+    "enable_feature_codec": True,
+    "enable_depth_guided_mv_refiner": True,
+    "enable_post_vae_mv_refiner": True,
+    "codec_lmb_range": [16.0, 1024.0],
+    "supervise_intermediate_depth": False,
+}
+DECODER_OVERRIDES = {
+    "use_covariances": True,
+    "rasterize_mode": "antialiased",
+    "eps2d": 0.1,
+    "packed": True,
+    "scale_invariant": False,
+}
+
+
+def build_encoder(checkpoint_path: str):
+    enc_cfg = OmegaConf.load(APP_DIR / "config/model/encoder/codecsplat.yaml")
+    enc_cfg = OmegaConf.merge(enc_cfg, ENCODER_OVERRIDES)
+    enc_cfg = from_dict(
+        EncoderCodecSplatCfg,
+        OmegaConf.to_container(enc_cfg, resolve=True),
+        config=DaciteConfig(cast=[tuple], strict=False),
+    )
+    encoder = get_encoder(enc_cfg)
+
+    ckpt = torch.load(checkpoint_path, map_location="cpu", weights_only=False)
+    state = ckpt.get("state_dict", ckpt)
+    state = {k[len("encoder."):]: v for k, v in state.items() if k.startswith("encoder.")}
+    encoder.load_state_dict(state, strict=True)
+    # eval() also runs `compress_mode` -> builds the entropy-coder CDF tables (on CPU).
+    encoder.eval()
+    return encoder
+
+
+def build_decoder():
+    from src.model.decoder.decoder_splatting_gsplat import (
+        DecoderSplattingGsplat,
+        DecoderSplattingGsplatCfg,
+    )
+
+    dec_cfg = OmegaConf.load(APP_DIR / "config/model/decoder/splatting_gsplat.yaml")
+    dec_cfg = OmegaConf.merge(dec_cfg, DECODER_OVERRIDES)
+    dec_cfg = from_dict(
+        DecoderSplattingGsplatCfg,
+        OmegaConf.to_container(dec_cfg, resolve=True),
+        config=DaciteConfig(strict=False),
+    )
+    decoder = DecoderSplattingGsplat(dec_cfg, SimpleNamespace(background_color=[0.0, 0.0, 0.0]))
+    return decoder.eval()
+
+
+# ---- view selection ------------------------------------------------------------------
+def farthest_point_sample(xyz: np.ndarray, npoint: int) -> np.ndarray:
+    n = len(xyz)
+    if npoint >= n:
+        return np.arange(n)
+    dist = np.full(n, 1e10)
+    farthest = int(np.argmax(((xyz - xyz.mean(0)) ** 2).sum(-1)))
+    out = []
+    for _ in range(npoint):
+        out.append(farthest)
+        d = ((xyz - xyz[farthest]) ** 2).sum(-1)
+        dist = np.minimum(dist, d)
+        farthest = int(np.argmax(dist))
+    return np.sort(np.array(out))
+
+
+def select_views(c2w: np.ndarray, num_context: int = 8, num_target: int = 8):
+    n = len(c2w)
+    ctx = farthest_point_sample(c2w[:, :3, 3], min(num_context, n))
+    rest = np.setdiff1d(np.arange(n), ctx)
+    if len(rest) > num_target:
+        rest = rest[np.linspace(0, len(rest) - 1, num_target).astype(int)]
+    return ctx, rest
+
+
+# ---- batch construction (same conventions as the dataset loaders) --------------------
+def _load_images(paths):
+    from PIL import Image
+
+    imgs = []
+    for p in paths:
+        im = np.asarray(Image.open(p).convert("RGB"), dtype=np.float32) / 255.0
+        imgs.append(torch.from_numpy(im).permute(2, 0, 1))
+    return torch.stack(imgs)
+
+
+def image_shape_for(h: int, w: int) -> tuple[int, int]:
+    # The checkpoint is trained at 256x448 (landscape). Portrait inputs use 448x256.
+    return IMAGE_SHAPE if w >= h else (IMAGE_SHAPE[1], IMAGE_SHAPE[0])
+
+
+def build_example(image_paths, c2w, K, ctx_idx, tgt_idx):
+    """Build a single (un-batched) example, apply the authors' crop + patch shims."""
+    images = _load_images(image_paths)  # [N, 3, H, W]
+    shape = image_shape_for(images.shape[-2], images.shape[-1])
+    c2w = torch.as_tensor(c2w, dtype=torch.float32)
+    K = torch.as_tensor(K, dtype=torch.float32)
+
+    # Express all poses relative to the middle context camera (rigid change of frame).
+    pivot = c2w[ctx_idx[len(ctx_idx) // 2]]
+    c2w = torch.linalg.inv(pivot)[None] @ c2w
+
+    def views(idx):
+        idx = torch.as_tensor(np.asarray(idx), dtype=torch.long)
+        return {
+            "extrinsics": c2w[idx],
+            "intrinsics": K[idx],
+            "image": images[idx],
+            "near": torch.full((len(idx),), NEAR),
+            "far": torch.full((len(idx),), FAR),
+            "index": idx,
+        }
+
+    example = {"context": views(ctx_idx), "target": views(tgt_idx), "scene": "user_scene"}
+    example = apply_crop_shim(example, shape)
+    batch = {
+        "context": {k: v[None] for k, v in example["context"].items()},
+        "target": {k: v[None] for k, v in example["target"].items()},
+        "scene": ["user_scene"],
+    }
+    # shim_patch_size * downscale_factor = 16 * 4
+    return apply_patch_shim(batch, patch_size=64)
+
+
+def to_device(batch, device):
+    out = {}
+    for k, v in batch.items():
+        if isinstance(v, dict):
+            out[k] = {kk: vv.to(device) for kk, vv in v.items()}
+        else:
+            out[k] = v
+    return out
+
+
+# ---- smooth fly-through trajectory through the context cameras ----------------------
+def order_cameras(c2w: torch.Tensor) -> list[int]:
+    """Greedy nearest-neighbour ordering of cameras (start from an extreme one)."""
+    pos = c2w[:, :3, 3].cpu().numpy()
+    n = len(pos)
+    if n <= 2:
+        return list(range(n))
+    start = int(np.argmax(((pos - pos.mean(0)) ** 2).sum(-1)))
+    order, left = [start], set(range(n)) - {start}
+    while left:
+        last = pos[order[-1]]
+        nxt = min(left, key=lambda i: float(((pos[i] - last) ** 2).sum()))
+        order.append(nxt)
+        left.remove(nxt)
+    return order
+
+
+def interpolate_trajectory(c2w: torch.Tensor, K: torch.Tensor, frames_per_segment: int = 12):
+    order = order_cameras(c2w)
+    c2w = c2w[order].cpu().double().numpy()
+    K = K[order].cpu()
+    poses, intrs = [], []
+    for i in range(len(order) - 1):
+        rots = R.from_matrix(c2w[i : i + 2, :3, :3])
+        slerp = Slerp([0.0, 1.0], rots)
+        for t in np.linspace(0, 1, frames_per_segment, endpoint=False):
+            m = np.eye(4)
+            m[:3, :3] = slerp([t]).as_matrix()[0]
+            m[:3, 3] = (1 - t) * c2w[i, :3, 3] + t * c2w[i + 1, :3, 3]
+            poses.append(m)
+            intrs.append((1 - t) * K[i] + t * K[i + 1])
+    last = np.eye(4)
+    last[:3] = c2w[-1, :3]
+    poses.append(last)
+    intrs.append(K[-1])
+    poses = poses + poses[::-1][1:]  # ping-pong loop
+    intrs = intrs + intrs[::-1][1:]
+    return (
+        torch.tensor(np.stack(poses), dtype=torch.float32),
+        torch.stack([torch.as_tensor(k, dtype=torch.float32) for k in intrs]),
+    )
+
+
+# ---- core inference ------------------------------------------------------------------
+@torch.no_grad()
+def encode_decode(encoder, batch, lmb: float):
+    """Full learned-codec round trip: context -> bitstream -> Gaussians."""
+    encoder.feature_codec.cfg.lmb = float(lmb)
+    with torch.autocast("cuda", dtype=torch.bfloat16, enabled=batch["context"]["image"].is_cuda):
+        out = encoder(batch["context"], 0, visualization_dump={}, return_bitstream=True)
+    return out
+
+
+@torch.no_grad()
+def render(decoder, gaussians, extrinsics, intrinsics, hw, chunk: int = 16):
+    """Render [V] views. `extrinsics`/`intrinsics` are [V,4,4]/[V,3,3] on device."""
+    v = extrinsics.shape[0]
+    near = torch.full((1, v), NEAR, device=extrinsics.device)
+    far = torch.full((1, v), FAR, device=extrinsics.device)
+    colors = []
+    with torch.autocast("cuda", dtype=torch.bfloat16, enabled=extrinsics.is_cuda):
+        for s in range(0, v, chunk):
+            o = decoder.forward(
+                gaussians,
+                extrinsics[None, s : s + chunk],
+                intrinsics[None, s : s + chunk],
+                near[:, s : s + chunk],
+                far[:, s : s + chunk],
+                hw,
+                depth_mode=None,
+            )
+            colors.append(o.color[0].float().clamp(0, 1))
+    return torch.cat(colors, dim=0)
+
+
+def psnr(gt: torch.Tensor, pred: torch.Tensor) -> float:
+    mse = ((gt - pred) ** 2).mean(dim=(1, 2, 3)).clamp_min(1e-10)
+    return float((-10 * torch.log10(mse)).mean())
+
+
+# ---- Gaussian export -----------------------------------------------------------------
+def world_rotations(rotations: torch.Tensor, context_c2w: torch.Tensor) -> torch.Tensor:
+    """Per-pixel Gaussian rotations are predicted in each context camera's frame."""
+    v = context_c2w.shape[0]
+    n = rotations.shape[0]
+    c2w_rot = repeat(context_c2w[:, :3, :3], "v i j -> (v r) i j", r=n // v)
+    local = R.from_quat(rotations.cpu().double().numpy()).as_matrix()
+    world = c2w_rot.cpu().double().numpy() @ local
+    return torch.from_numpy(R.from_matrix(world).as_quat()).float()
+
+
+def write_ply(
+    path,
+    pivot_c2w: torch.Tensor,
+    means: torch.Tensor,
+    scales: torch.Tensor,
+    rotations_xyzw_world: torch.Tensor,
+    harmonics: torch.Tensor,
+    opacities: torch.Tensor,
+):
+    """Same output as the authors' `export_ply(..., align_to_view=True)`, vectorised."""
+    means = means.float().cpu()
+    scales = scales.float().cpu()
+    harmonics = harmonics.float().cpu()
+    opacities = opacities.float().cpu()
+    pivot = pivot_c2w.float().cpu()
+
+    view_rot = pivot[:3, :3].T
+    means = (means - pivot[:3, 3]) @ view_rot.T
+    harmonics = rotate_sh(harmonics, view_rot)
+    rot = R.from_quat(rotations_xyzw_world.double().numpy()).as_matrix()
+    rot = R.from_matrix(view_rot.double().numpy() @ rot).as_quat()  # xyzw
+    rot = np.concatenate([rot[:, 3:4], rot[:, :3]], axis=1)  # wxyz
+
+    dc = harmonics[..., 0].numpy()
+    rest = harmonics[..., 1:].flatten(start_dim=1).numpy()
+    logit_op = torch.logit(opacities.clamp(1e-6, 1 - 1e-6))[:, None].numpy()
+    log_scales = scales.clamp_min(1e-12).log().numpy()
+    m = means.numpy()
+
+    cols = [m, np.zeros_like(m), dc, rest, logit_op, log_scales, rot]
+    data = np.concatenate(cols, axis=1).astype(np.float32)
+    names = ["x", "y", "z", "nx", "ny", "nz", "f_dc_0", "f_dc_1", "f_dc_2"]
+    names += [f"f_rest_{i}" for i in range(rest.shape[1])]
+    names += ["opacity", "scale_0", "scale_1", "scale_2", "rot_0", "rot_1", "rot_2", "rot_3"]
+    arr = np.empty(data.shape[0], dtype=[(n, "f4") for n in names])
+    for i, n in enumerate(names):
+        arr[n] = data[:, i]
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    PlyData([PlyElement.describe(arr, "vertex")]).write(str(path))
+    return str(path)
+
+
+def export_gaussians(g: dict, context_c2w: torch.Tensor, outdir: Path,
+                     viewer_max: int = 600_000, viewer_min_opacity: float = 0.02):
+    """Write a compact (DC colour, pruned), re-framed .ply of the Gaussians for the web viewer."""
+    pivot = context_c2w[len(context_c2w) // 2]
+    rot_world = world_rotations(g["rotations"], context_c2w)
+
+    keep = g["opacities"] > viewer_min_opacity
+    if int(keep.sum()) < 1000:
+        keep = torch.ones_like(keep)
+    idx = torch.nonzero(keep).squeeze(1)
+    if idx.numel() > viewer_max:
+        idx = idx[torch.randperm(idx.numel())[:viewer_max]]
+    means, scales = g["means"][idx], g["scales"][idx]
+
+    # Re-centre / re-scale (in the pivot camera frame) so the viewer's orbit camera frames it.
+    view_rot = pivot[:3, :3].T
+    local = (means - pivot[:3, 3]) @ view_rot.T
+    center = local.median(dim=0).values
+    radius = torch.quantile((local - center).norm(dim=-1).float(), 0.7)
+    s = float(3.0 / max(float(radius), 1e-6))
+    # shift in world coordinates so that the pivot-frame centre maps to the origin
+    means = (means - pivot[:3, 3] - center @ view_rot) * s + pivot[:3, 3]
+    viewer_path = write_ply(outdir / "codecsplat_viewer.ply", pivot, means, scales * s,
+                            rot_world[idx], g["harmonics"][idx][..., :1], g["opacities"][idx])
+    return viewer_path

colmap_sfm.py

@@ -0,0 +1,177 @@
+"""COLMAP structure-from-motion preprocessing (runs inside the Space, on CPU).
+
+Takes a set of raw multi-view images of a scene, estimates camera intrinsics /
+poses / a sparse point cloud with COLMAP (via `pycolmap`), undistorts the images
+to a PINHOLE camera model and writes a standard COLMAP scene layout
+
+    <out>/images/*.jpg
+    <out>/sparse/{cameras,images,points3D}.bin
+
+which is exactly what CodecSplat's reference COLMAP loader expects.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+import pycolmap
+from PIL import Image, ImageOps
+
+IMAGE_EXT = (".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff")
+
+
+def prepare_images(image_paths, dest: Path, max_image_size: int = 1280) -> list[str]:
+    """Copy the uploaded images into `dest`, EXIF-rotated, RGB, size-capped."""
+    dest.mkdir(parents=True, exist_ok=True)
+    names: list[str] = []
+    for i, src in enumerate(str(p) for p in image_paths):  # caller decides the order
+        if not src.lower().endswith(IMAGE_EXT):
+            continue
+        try:
+            img = Image.open(src)
+            img = ImageOps.exif_transpose(img).convert("RGB")
+        except Exception as exc:
+            print(f"[colmap] skipping unreadable image {src}: {exc}", flush=True)
+            continue
+        scale = max_image_size / max(img.size)
+        if scale < 1.0:
+            img = img.resize(
+                (max(1, int(round(img.width * scale))), max(1, int(round(img.height * scale)))),
+                Image.LANCZOS,
+            )
+        name = f"{i:04d}.jpg"
+        img.save(dest / name, quality=95)
+        names.append(name)
+    return names
+
+
+def _largest_reconstruction(maps) -> pycolmap.Reconstruction | None:
+    recs = list(maps.values()) if isinstance(maps, dict) else list(maps)
+    if not recs:
+        return None
+    return max(recs, key=lambda r: r.num_reg_images())
+
+
+def run_sfm(
+    image_paths,
+    workdir: Path,
+    max_image_size: int = 1280,
+    matcher: str = "exhaustive",
+    max_num_features: int = 8192,
+    log=print,
+) -> dict:
+    """Run COLMAP SfM on `image_paths`; return info about the written scene.
+
+    Returns a dict with `scene_dir`, `sparse_sub`, `images_sub`, `num_input`,
+    `num_registered`, `num_points3D`, `camera_model`.
+    """
+    workdir = Path(workdir)
+    image_dir = workdir / "sfm_images"
+    names = prepare_images(image_paths, image_dir, max_image_size)
+    if len(names) < 3:
+        raise ValueError(
+            f"COLMAP needs at least 3 usable images, got {len(names)}. "
+            "Upload several overlapping photos of the same scene."
+        )
+    log(f"[colmap] {len(names)} images prepared at <= {max_image_size}px")
+
+    database_path = workdir / "colmap.db"
+    if database_path.exists():
+        database_path.unlink()
+
+    extraction_options = pycolmap.FeatureExtractionOptions()
+    extraction_options.sift.max_num_features = int(max_num_features)
+    extraction_options.sift.estimate_affine_shape = False
+    extraction_options.sift.domain_size_pooling = False
+    extraction_options.use_gpu = False
+    extraction_options.max_image_size = int(max_image_size)
+
+    reader_options = pycolmap.ImageReaderOptions()
+    reader_options.camera_model = "SIMPLE_RADIAL"
+
+    pycolmap.extract_features(
+        database_path=database_path,
+        image_path=image_dir,
+        camera_mode=pycolmap.CameraMode.SINGLE,
+        reader_options=reader_options,
+        extraction_options=extraction_options,
+        device=pycolmap.Device.cpu,
+    )
+    log("[colmap] features extracted")
+
+    if matcher == "sequential":
+        pairing = pycolmap.SequentialPairingOptions()
+        pairing.overlap = 10
+        pairing.quadratic_overlap = True
+        pycolmap.match_sequential(
+            database_path=database_path,
+            pairing_options=pairing,
+            device=pycolmap.Device.cpu,
+        )
+    else:
+        pycolmap.match_exhaustive(database_path=database_path, device=pycolmap.Device.cpu)
+    log(f"[colmap] {matcher} matching done")
+
+    sparse_dir = workdir / "sparse_raw"
+    sparse_dir.mkdir(parents=True, exist_ok=True)
+    options = pycolmap.IncrementalPipelineOptions()
+    options.min_num_matches = 15
+    options.multiple_models = False
+    options.extract_colors = True
+    maps = pycolmap.incremental_mapping(
+        database_path=database_path,
+        image_path=image_dir,
+        output_path=sparse_dir,
+        options=options,
+    )
+    rec = _largest_reconstruction(maps)
+    if rec is None or rec.num_reg_images() < 3:
+        raise ValueError(
+            "COLMAP could not reconstruct this image set (only "
+            f"{0 if rec is None else rec.num_reg_images()} of {len(names)} images registered). "
+            "Use photos with plenty of visual overlap between neighbours, avoid pure "
+            "rotation / blurry frames, and keep the scene static."
+        )
+    # Deterministic world scale (same convention COLMAP itself uses), so that the
+    # fixed near/far planes of the CodecSplat checkpoint stay meaningful.
+    rec.normalize(fixed_scale=False, extent=10.0)
+    log(f"[colmap] mapped {rec.num_reg_images()}/{len(names)} images, {rec.num_points3D()} points")
+
+    model_dir = workdir / "sparse_model"
+    if model_dir.exists():
+        shutil.rmtree(model_dir)
+    model_dir.mkdir(parents=True)
+    rec.write(str(model_dir))
+
+    scene_dir = workdir / "colmap_scene"
+    if scene_dir.exists():
+        shutil.rmtree(scene_dir)
+    pycolmap.undistort_images(
+        output_path=scene_dir,
+        input_path=model_dir,
+        image_path=image_dir,
+    )
+    log("[colmap] undistorted to PINHOLE")
+
+    # Keep only what CodecSplat reads (the undistorter also writes stereo/ scaffolding).
+    for extra in ("stereo",):
+        p = scene_dir / extra
+        if p.is_dir():
+            shutil.rmtree(p, ignore_errors=True)
+
+    cam_model = "PINHOLE"
+    try:
+        cam_model = list(rec.cameras.values())[0].model.name
+    except Exception:
+        pass
+
+    return {
+        "scene_dir": str(scene_dir),
+        "sparse_sub": "sparse",
+        "images_sub": "images",
+        "num_input": len(names),
+        "num_registered": int(rec.num_reg_images()),
+        "num_points3D": int(rec.num_points3D()),
+        "camera_model": cam_model,
+    }

config/dataset/dl3dv.yaml

@@ -0,0 +1,27 @@
+defaults:
+  - view_sampler: boundedv2_360
+
+name: dl3dv
+roots: [datasets/DL3DV-ALL-480P-Processed]
+augment: true
+
+image_shape: [270, 480]
+background_color: [0.0, 0.0, 0.0]
+cameras_are_circular: false
+
+max_fov: 100.0
+
+skip_bad_shape: true
+near: -1.
+far: -1.
+shuffle_val: true
+test_len: -1
+test_chunk_interval: 1
+sort_target_index: true
+sort_context_index: true
+
+train_times_per_scene: 1
+test_times_per_scene: 1
+ori_image_shape: [270, 480]
+overfit_max_views: 148
+use_index_to_load_chunk: false

config/dataset/re10k.yaml

@@ -0,0 +1,21 @@
+defaults:
+  - view_sampler: bounded
+
+name: re10k
+roots: [datasets/RE10K-Torch-360p]
+augment: true
+
+image_shape: [180, 320]
+background_color: [0.0, 0.0, 0.0]
+cameras_are_circular: false
+
+max_fov: 100.0
+
+skip_bad_shape: true
+near: -1.
+far: -1.
+shuffle_val: true
+test_len: -1
+test_chunk_interval: 1
+
+use_index_to_load_chunk: false

config/dataset/view_sampler/all.yaml

@@ -0,0 +1 @@
+name: all

config/dataset/view_sampler/arbitrary.yaml

@@ -0,0 +1,7 @@
+name: arbitrary
+
+num_target_views: 1
+num_context_views: 2
+
+# If you want to hard-code context views, do so here.
+context_views: null

config/dataset/view_sampler/bounded.yaml

@@ -0,0 +1,12 @@
+name: bounded
+
+num_target_views: 1
+num_context_views: 2
+
+min_distance_between_context_views: 2
+max_distance_between_context_views: 6
+min_distance_to_context_views: 0
+
+warm_up_steps: 0
+initial_min_distance_between_context_views: 2
+initial_max_distance_between_context_views: 6

config/dataset/view_sampler/boundedv2.yaml

@@ -0,0 +1,15 @@
+name: boundedv2
+
+num_target_views: 1
+num_context_views: 2
+
+min_distance_between_context_views: 2
+max_distance_between_context_views: 6
+max_distance_to_context_views: 0
+
+context_gap_warm_up_steps: 0
+target_gap_warm_up_steps: 0
+
+initial_min_distance_between_context_views: 2
+initial_max_distance_between_context_views: 6
+initial_max_distance_to_context_views: 0

config/dataset/view_sampler/boundedv2_360.yaml

@@ -0,0 +1,17 @@
+name: boundedv2
+
+num_target_views: 4
+num_context_views: 4
+
+min_distance_between_context_views: 20
+max_distance_between_context_views: 50
+max_distance_to_context_views: 0
+
+context_gap_warm_up_steps: 10000
+target_gap_warm_up_steps: 0
+
+initial_min_distance_between_context_views: 15
+initial_max_distance_between_context_views: 30
+initial_max_distance_to_context_views: 0
+extra_views_sampling_strategy: farthest_point
+target_views_replace_sample: false

config/dataset/view_sampler/evaluation.yaml

@@ -0,0 +1,4 @@
+name: evaluation
+
+index_path: assets/evaluation_index_re10k_video.json
+num_context_views: 2

config/dataset/view_sampler_dataset_specific_config/bounded_re10k.yaml

@@ -0,0 +1,11 @@
+# @package _global_
+
+dataset:
+  view_sampler:
+    min_distance_between_context_views: 45
+    max_distance_between_context_views: 135
+    min_distance_to_context_views: 0
+    warm_up_steps: 30000
+    initial_min_distance_between_context_views: 25
+    initial_max_distance_between_context_views: 45
+    num_target_views: 4

config/dataset/view_sampler_dataset_specific_config/boundedv2_dl3dv.yaml

@@ -0,0 +1,14 @@
+# @package _global_
+
+dataset:
+  view_sampler:
+    min_distance_between_context_views: 20
+    max_distance_between_context_views: 50
+    max_distance_to_context_views: 0
+    context_gap_warm_up_steps: 10000
+    target_gap_warm_up_steps: 0
+    initial_min_distance_between_context_views: 15
+    initial_max_distance_between_context_views: 30
+    initial_max_distance_to_context_views: 0
+    extra_views_sampling_strategy: farthest_point
+    num_target_views: 4

config/dataset/view_sampler_dataset_specific_config/evaluation_dl3dv.yaml

@@ -0,0 +1,7 @@
+# @package _global_
+
+dataset:
+  image_shape: [256, 448]
+  view_sampler:
+    index_path: assets/dl3dv_evaluation/dl3dv_start_0_distance_40_ctx_8v_tgt_8v.json
+    num_context_views: 8

config/dataset/view_sampler_dataset_specific_config/evaluation_re10k.yaml

@@ -0,0 +1,5 @@
+# @package _global_
+
+dataset:
+  view_sampler:
+    index_path: assets/evaluation_index_re10k.json

config/experiment/dl3dv.yaml

@@ -0,0 +1,41 @@
+# @package _global_
+
+defaults:
+  - override /dataset: dl3dv
+  - override /model/encoder: codecsplat
+  - override /loss: [mse, lpips]
+  - override /dataset/view_sampler: boundedv2_360
+
+wandb:
+  name: dl3dv
+  tags: [dl3dv, 270x480]
+
+data_loader:
+  train:
+    batch_size: 1
+
+trainer:
+  max_steps: 300_001
+  num_nodes: 1
+
+model:
+  encoder:
+    num_depth_candidates: 128
+    shim_patch_size: 16
+
+# lpips loss
+loss:
+  lpips:
+    apply_after_step: 0
+    weight: 0.05
+
+dataset:
+  near: 0.5
+  far: 200.
+  min_views: 0
+  max_views: 0
+  highres: false
+
+test:
+  eval_time_skip_steps: 0
+  compute_scores: true

config/experiment/re10k.yaml

@@ -0,0 +1,40 @@
+# @package _global_
+
+defaults:
+  - override /dataset: re10k
+  - override /model/encoder: codecsplat
+  - override /loss: [mse, lpips]
+
+wandb:
+  name: re10k
+  tags: [re10k, 256x256]
+
+data_loader:
+  train:
+    batch_size: 14
+
+trainer:
+  max_steps: 300_001
+  num_nodes: 1
+
+model:
+  encoder:
+    num_depth_candidates: 128
+
+# lpips loss
+loss:
+  lpips:
+    apply_after_step: 0
+    weight: 0.05
+
+dataset:
+  image_shape: [256, 256]
+  roots: [datasets/RE10K-Torch-360p]
+  near: 0.5
+  far: 100.
+  train_times_per_scene: 1
+  highres: false
+
+test:
+  eval_time_skip_steps: 0
+  compute_scores: true

config/loss/lpips.yaml

@@ -0,0 +1,3 @@
+lpips:
+  weight: 0.05
+  apply_after_step: 150_000

config/loss/mse.yaml

@@ -0,0 +1,2 @@
+mse:
+  weight: 1.0

config/main.yaml

@@ -0,0 +1,115 @@
+defaults:
+  - dataset: re10k
+  - optional dataset/view_sampler_dataset_specific_config: ${dataset/view_sampler}_${dataset}
+  - model/encoder: codecsplat
+  - model/decoder: splatting_gsplat
+  - loss: [mse]
+  - _self_
+
+wandb:
+  project: codecsplat
+  entity: null
+  name: placeholder
+  mode: online
+  id: null
+
+mode: train
+
+dataset:
+  overfit_to_scene: null
+
+data_loader:
+  train:
+    num_workers: 10
+    persistent_workers: true
+    batch_size: 4
+    seed: 1234
+  test:
+    num_workers: 4
+    persistent_workers: false
+    batch_size: 1
+    seed: 2345
+  val:
+    num_workers: 1
+    persistent_workers: true
+    batch_size: 1
+    seed: 3456
+
+optimizer:
+  lr: 2.e-4
+  lr_monodepth: 2.e-6
+  warm_up_steps: 2000
+  weight_decay: 0.01
+  div_factor: 25.
+  final_div_factor: 1.e4
+
+checkpointing:
+  load: null
+  every_n_train_steps: 5000
+  save_top_k: 5
+  pretrained_model: null
+  pretrained_monodepth: null
+  pretrained_mvdepth: null
+  pretrained_depth: null
+  no_strict_load: false
+  resume: false
+
+train:
+  depth_mode: null
+  extended_visualization: false
+  print_log_every_n_steps: 100
+  eval_model_every_n_val: 2  # quantitative evaluation every n val
+  eval_data_length: 999999
+  eval_time_skip_steps: 3
+  eval_save_model: true
+  l1_loss: false
+  intermediate_loss_weight: 0.9
+  no_viz_video: false
+  train_ignore_large_loss: 0.
+
+test:
+  compute_scores: true
+  eval_time_skip_steps: 0
+  save_image: false
+  save_video: false
+  save_gt_image: false
+  save_input_images: false
+  save_depth: false
+  save_depth_npy: false
+  save_depth_concat_img: false
+  save_gaussian: false
+  save_bitstream: false
+  render_chunk_size: null
+  stabilize_camera: false
+  stab_camera_kernel: 50
+  gaussian_export_all_points: true
+  gaussian_export_align_to_view: false
+
+seed: 111123
+
+trainer:
+  max_steps: -1
+  val_check_interval: 0.5
+  gradient_clip_val: 0.5
+  accumulate_grad_batches: 1
+  precision: 32-true
+  num_sanity_val_steps: 2
+  num_nodes: 1
+
+output_dir: checkpoints/tmp
+
+use_plugins: false
+
+hydra:
+  run:
+    dir: ${output_dir}/hydra/${now:%Y-%m-%d-%H-%M-%S}
+  job:
+    chdir: false
+  output_subdir: .
+  job_logging:
+    handlers:
+      file:
+        filename: /dev/null
+    root:
+      handlers:
+        - console

config/model/decoder/splatting_cuda.yaml

@@ -0,0 +1 @@
+name: splatting_cuda

config/model/decoder/splatting_gsplat.yaml

@@ -0,0 +1,10 @@
+name: splatting_gsplat
+render_mode: RGB
+rasterize_mode: antialiased
+use_covariances: true
+packed: true
+eps2d: 0.1
+sparse_grad: false
+absgrad: false
+channel_chunk: 32
+scale_invariant: false

config/model/encoder/codecsplat.yaml

@@ -0,0 +1,48 @@
+name: codecsplat
+
+num_depth_candidates: 128
+num_surfaces: 1
+
+gaussian_adapter:
+  gaussian_scale_min: 1e-10
+  gaussian_scale_max: 3.
+  sh_degree: 2
+  exp_scale_bias: 4.
+  no_rotate_sh: false
+  no_sh_mask: false
+
+# Input alignment
+downscale_factor: 4
+shim_patch_size: 4
+
+# monodepth
+monodepth_vit_type: vits
+
+# return depth
+supervise_intermediate_depth: true
+return_depth: true
+
+# mv_unimatch
+num_scales: 1
+upsample_factor: 4
+lowest_feature_resolution: 4
+depth_unet_channels: 128
+grid_sample_disable_cudnn: false
+local_mv_match: 2
+
+# color branch
+init_sh_input_img: false
+gaussian_regressor_channels: 64
+
+# feature codec path
+enable_feature_codec: false
+enable_depth_guided_mv_refiner: false
+enable_post_vae_mv_refiner: false
+codec_loss_weight: 1.0
+codec_depth_loss_weight: 0.1
+codec_lmb_range: [16.0, 1024.0]
+freeze_legacy_modules_for_codec: false
+codec_estimated_feature_num_bytes: false
+deflate_feature_ablation: false
+deflate_feature_dtype: float16
+deflate_feature_level: 9