Use the general checkpoint for uploads; drop "MLP" from the title
Browse filesUploaded images and videos are now reconstructed with
DVSM-TTB-p8-general-960p. The gallery examples were baked with
DVSM-TTB-p8-DL3DV-960p, so both models are loaded and a live session
renders each artifact with the model named in its hf_variant.
APP_MODEL_DIR now points at a directory with one sub-directory per
variant (the nvidia/DVSM layout) instead of a single model directory.
Wording: the subtitle reads "Test-time Baking ..." and the tagline says
"TTB bakes the scene ...".
Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
README.md
CHANGED
|
@@ -16,7 +16,7 @@ license: other
|
|
| 16 |
Upload images / a short orbit video → π³ pose estimation + TTB-DVSM
|
| 17 |
reconstruction (one GPU call) → a time-boxed interactive viser viewer.
|
| 18 |
|
| 19 |
-
|
| 20 |
|
| 21 |
## ZeroGPU notes
|
| 22 |
Select **ZeroGPU** hardware in the Space Settings (it is not set via this
|
|
|
|
| 16 |
Upload images / a short orbit video → π³ pose estimation + TTB-DVSM
|
| 17 |
reconstruction (one GPU call) → a time-boxed interactive viser viewer.
|
| 18 |
|
| 19 |
+
Models (`nvidia/DVSM`): uploads are reconstructed with `DVSM-TTB-p8-general-960p`; the gallery examples were baked with `DVSM-TTB-p8-DL3DV-960p`.
|
| 20 |
|
| 21 |
## ZeroGPU notes
|
| 22 |
Select **ZeroGPU** hardware in the Space Settings (it is not set via this
|
app.py
CHANGED
|
@@ -17,7 +17,9 @@ Why this shape (ZeroGPU constraints, see README):
|
|
| 17 |
"Start / Continue" to fork a new worker and rebuild the viewer from the
|
| 18 |
already-computed artifact on disk (no re-reconstruction).
|
| 19 |
|
| 20 |
-
|
|
|
|
|
|
|
| 21 |
"""
|
| 22 |
from __future__ import annotations
|
| 23 |
|
|
@@ -61,7 +63,11 @@ from viser_app.zerogpu_compat import repair_torch_cuda_state
|
|
| 61 |
# ---------------------------------------------------------------------------
|
| 62 |
# Config
|
| 63 |
# ---------------------------------------------------------------------------
|
| 64 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
# How long one held-GPU interactive session lasts before it must be renewed.
|
| 66 |
# Bounded by the per-call duration cap and the user's daily quota
|
| 67 |
# (anon 2min / free 5min / PRO 40min). Override via env for PRO accounts.
|
|
@@ -102,18 +108,25 @@ EXAMPLE_SCENES = _discover_examples()
|
|
| 102 |
# Module-level model load: download + keep on CPU (ZeroGPU convention).
|
| 103 |
# Real `.cuda()` happens inside @spaces.GPU workers.
|
| 104 |
# ---------------------------------------------------------------------------
|
| 105 |
-
print(f"[app] loading {
|
| 106 |
from dvsm.models.DVSM import DecoderViewSynthesisModel
|
| 107 |
from viser_app.pi3_estimator import load_pi3_model
|
| 108 |
|
| 109 |
-
# APP_MODEL_DIR: load the DVSM
|
| 110 |
-
#
|
|
|
|
| 111 |
_LOCAL_MODEL_DIR = os.environ.get("APP_MODEL_DIR")
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 117 |
PI3_MODEL_CPU = load_pi3_model(flavor="pi3x", device="cpu").eval()
|
| 118 |
print("[app] models ready (CPU)", flush=True)
|
| 119 |
|
|
@@ -167,11 +180,11 @@ def reconstruct_gpu(img_dir: str, target_h: int, target_w: int, max_views: int)
|
|
| 167 |
# Reuse the module-level CPU model (moved to cuda inside reconstruct()).
|
| 168 |
renderer, batch, ishape, c2w_r, c2w_t, intr = reconstruct(
|
| 169 |
ckpt_root=None, ckpt_name="", scene_data=scene_data, cmd_lst=[],
|
| 170 |
-
hf_variant=
|
| 171 |
)
|
| 172 |
out_path = ARTIFACTS_DIR / f"{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}.recon.pt"
|
| 173 |
save_artifact(renderer, batch, ishape, c2w_r, c2w_t, intr, out_path,
|
| 174 |
-
hf_variant=
|
| 175 |
return str(out_path)
|
| 176 |
|
| 177 |
|
|
@@ -206,8 +219,13 @@ def _live_session_worker(artifact_path: str):
|
|
| 206 |
|
| 207 |
repair_torch_cuda_state() # torch.compile vs ZeroGPU, see zerogpu_compat
|
| 208 |
yield ("prep", 0.15, "Loading scene…")
|
| 209 |
-
model = DVSM_MODEL_CPU.cuda().eval()
|
| 210 |
artifact = torch.load(artifact_path, weights_only=False, map_location="cpu")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
renderer = build_renderer(model, artifact)
|
| 212 |
scene_meta = artifact["scene_meta"]
|
| 213 |
|
|
@@ -605,14 +623,14 @@ with gr.Blocks(title="TTB-DVSM (ZeroGPU)", delete_cache=(3600, 3600)) as demo:
|
|
| 605 |
"<h2 style='font-style:normal;font-weight:500;font-size:1.6em;"
|
| 606 |
"margin-top:0;color:var(--body-text-color)'>"
|
| 607 |
"<b style='color:#76B900'>T</b>est-<b style='color:#76B900'>t</b>ime "
|
| 608 |
-
"
|
| 609 |
"<b style='color:#76B900'>D</b>ecoder-only "
|
| 610 |
"<b style='color:#76B900'>V</b>iew "
|
| 611 |
"<b style='color:#76B900'>S</b>ynthesis "
|
| 612 |
"<b style='color:#76B900'>M</b>odels"
|
| 613 |
"</h2>"
|
| 614 |
"<p style='color:var(--body-text-color);font-size:1.1em;margin-top:0.5em;line-height:1.6'>"
|
| 615 |
-
"TTB
|
| 616 |
"and the result is browseable in the viser viewer."
|
| 617 |
"</p>"
|
| 618 |
"<div style='color:var(--body-text-color);font-size:1.1em;margin-top:0.8em;line-height:1.7'>"
|
|
|
|
| 17 |
"Start / Continue" to fork a new worker and rebuild the viewer from the
|
| 18 |
already-computed artifact on disk (no re-reconstruction).
|
| 19 |
|
| 20 |
+
Models (nvidia/DVSM): uploads are reconstructed with DVSM-TTB-p8-general-960p; the
|
| 21 |
+
gallery examples were baked with DVSM-TTB-p8-DL3DV-960p. A live session renders an
|
| 22 |
+
artifact with the model that baked it.
|
| 23 |
"""
|
| 24 |
from __future__ import annotations
|
| 25 |
|
|
|
|
| 63 |
# ---------------------------------------------------------------------------
|
| 64 |
# Config
|
| 65 |
# ---------------------------------------------------------------------------
|
| 66 |
+
# Uploaded images / videos are reconstructed with the general model. The gallery
|
| 67 |
+
# examples were baked with the DL3DV model. An artifact records the variant that
|
| 68 |
+
# baked it ("hf_variant") and has to be rendered with that same model.
|
| 69 |
+
UPLOAD_VARIANT = "DVSM-TTB-p8-general-960p"
|
| 70 |
+
EXAMPLE_VARIANT = "DVSM-TTB-p8-DL3DV-960p"
|
| 71 |
# How long one held-GPU interactive session lasts before it must be renewed.
|
| 72 |
# Bounded by the per-call duration cap and the user's daily quota
|
| 73 |
# (anon 2min / free 5min / PRO 40min). Override via env for PRO accounts.
|
|
|
|
| 108 |
# Module-level model load: download + keep on CPU (ZeroGPU convention).
|
| 109 |
# Real `.cuda()` happens inside @spaces.GPU workers.
|
| 110 |
# ---------------------------------------------------------------------------
|
| 111 |
+
print(f"[app] loading {UPLOAD_VARIANT}, {EXAMPLE_VARIANT} + π³ to CPU ...", flush=True)
|
| 112 |
from dvsm.models.DVSM import DecoderViewSynthesisModel
|
| 113 |
from viser_app.pi3_estimator import load_pi3_model
|
| 114 |
|
| 115 |
+
# APP_MODEL_DIR: load the DVSM models from a local directory instead of the Hub
|
| 116 |
+
# (faster / offline; used for local GPU-node testing). It holds one HF-format
|
| 117 |
+
# sub-directory per variant, i.e. the layout of the nvidia/DVSM repo.
|
| 118 |
_LOCAL_MODEL_DIR = os.environ.get("APP_MODEL_DIR")
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def _load_dvsm(variant):
|
| 122 |
+
if _LOCAL_MODEL_DIR:
|
| 123 |
+
path = Path(_LOCAL_MODEL_DIR) / variant
|
| 124 |
+
print(f"[app] loading {variant} from local dir {path}", flush=True)
|
| 125 |
+
return DecoderViewSynthesisModel.from_pretrained(str(path)).eval()
|
| 126 |
+
return DecoderViewSynthesisModel.from_hf_pretrained(variant).eval()
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
DVSM_MODELS_CPU = {v: _load_dvsm(v) for v in (UPLOAD_VARIANT, EXAMPLE_VARIANT)}
|
| 130 |
PI3_MODEL_CPU = load_pi3_model(flavor="pi3x", device="cpu").eval()
|
| 131 |
print("[app] models ready (CPU)", flush=True)
|
| 132 |
|
|
|
|
| 180 |
# Reuse the module-level CPU model (moved to cuda inside reconstruct()).
|
| 181 |
renderer, batch, ishape, c2w_r, c2w_t, intr = reconstruct(
|
| 182 |
ckpt_root=None, ckpt_name="", scene_data=scene_data, cmd_lst=[],
|
| 183 |
+
hf_variant=UPLOAD_VARIANT, model=DVSM_MODELS_CPU[UPLOAD_VARIANT],
|
| 184 |
)
|
| 185 |
out_path = ARTIFACTS_DIR / f"{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}.recon.pt"
|
| 186 |
save_artifact(renderer, batch, ishape, c2w_r, c2w_t, intr, out_path,
|
| 187 |
+
hf_variant=UPLOAD_VARIANT)
|
| 188 |
return str(out_path)
|
| 189 |
|
| 190 |
|
|
|
|
| 219 |
|
| 220 |
repair_torch_cuda_state() # torch.compile vs ZeroGPU, see zerogpu_compat
|
| 221 |
yield ("prep", 0.15, "Loading scene…")
|
|
|
|
| 222 |
artifact = torch.load(artifact_path, weights_only=False, map_location="cpu")
|
| 223 |
+
# Render with the model that baked this artifact (uploads vs gallery examples).
|
| 224 |
+
variant = artifact.get("hf_variant") or EXAMPLE_VARIANT
|
| 225 |
+
if variant not in DVSM_MODELS_CPU:
|
| 226 |
+
raise RuntimeError(f"artifact was baked with {variant}, which this app does not load")
|
| 227 |
+
print(f"[session] rendering with {variant}", flush=True)
|
| 228 |
+
model = DVSM_MODELS_CPU[variant].cuda().eval()
|
| 229 |
renderer = build_renderer(model, artifact)
|
| 230 |
scene_meta = artifact["scene_meta"]
|
| 231 |
|
|
|
|
| 623 |
"<h2 style='font-style:normal;font-weight:500;font-size:1.6em;"
|
| 624 |
"margin-top:0;color:var(--body-text-color)'>"
|
| 625 |
"<b style='color:#76B900'>T</b>est-<b style='color:#76B900'>t</b>ime "
|
| 626 |
+
"<b style='color:#76B900'>B</b>aking for Efficient Rendering of "
|
| 627 |
"<b style='color:#76B900'>D</b>ecoder-only "
|
| 628 |
"<b style='color:#76B900'>V</b>iew "
|
| 629 |
"<b style='color:#76B900'>S</b>ynthesis "
|
| 630 |
"<b style='color:#76B900'>M</b>odels"
|
| 631 |
"</h2>"
|
| 632 |
"<p style='color:var(--body-text-color);font-size:1.1em;margin-top:0.5em;line-height:1.6'>"
|
| 633 |
+
"TTB bakes the scene into fast weights in a single forward pass, "
|
| 634 |
"and the result is browseable in the viser viewer."
|
| 635 |
"</p>"
|
| 636 |
"<div style='color:var(--body-text-color);font-size:1.1em;margin-top:0.8em;line-height:1.7'>"
|