linjohnss Claude Fable 5.2 commited on
Commit
3f9d2ed
·
1 Parent(s): 191284b

Use the general checkpoint for uploads; drop "MLP" from the title

Browse files

Uploaded images and videos are now reconstructed with
DVSM-TTB-p8-general-960p. The gallery examples were baked with
DVSM-TTB-p8-DL3DV-960p, so both models are loaded and a live session
renders each artifact with the model named in its hf_variant.

APP_MODEL_DIR now points at a directory with one sub-directory per
variant (the nvidia/DVSM layout) instead of a single model directory.

Wording: the subtitle reads "Test-time Baking ..." and the tagline says
"TTB bakes the scene ...".

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>

Files changed (2) hide show
  1. README.md +1 -1
  2. app.py +33 -15
README.md CHANGED
@@ -16,7 +16,7 @@ license: other
16
  Upload images / a short orbit video → π³ pose estimation + TTB-DVSM
17
  reconstruction (one GPU call) → a time-boxed interactive viser viewer.
18
 
19
- Fixed model: `DVSM-TTB-p8-DL3DV-960p` (`nvidia/DVSM`).
20
 
21
  ## ZeroGPU notes
22
  Select **ZeroGPU** hardware in the Space Settings (it is not set via this
 
16
  Upload images / a short orbit video → π³ pose estimation + TTB-DVSM
17
  reconstruction (one GPU call) → a time-boxed interactive viser viewer.
18
 
19
+ Models (`nvidia/DVSM`): uploads are reconstructed with `DVSM-TTB-p8-general-960p`; the gallery examples were baked with `DVSM-TTB-p8-DL3DV-960p`.
20
 
21
  ## ZeroGPU notes
22
  Select **ZeroGPU** hardware in the Space Settings (it is not set via this
app.py CHANGED
@@ -17,7 +17,9 @@ Why this shape (ZeroGPU constraints, see README):
17
  "Start / Continue" to fork a new worker and rebuild the viewer from the
18
  already-computed artifact on disk (no re-reconstruction).
19
 
20
- Fixed model: DVSM-TTB-p8-DL3DV-960p (nvidia/DVSM).
 
 
21
  """
22
  from __future__ import annotations
23
 
@@ -61,7 +63,11 @@ from viser_app.zerogpu_compat import repair_torch_cuda_state
61
  # ---------------------------------------------------------------------------
62
  # Config
63
  # ---------------------------------------------------------------------------
64
- HF_VARIANT = "DVSM-TTB-p8-DL3DV-960p"
 
 
 
 
65
  # How long one held-GPU interactive session lasts before it must be renewed.
66
  # Bounded by the per-call duration cap and the user's daily quota
67
  # (anon 2min / free 5min / PRO 40min). Override via env for PRO accounts.
@@ -102,18 +108,25 @@ EXAMPLE_SCENES = _discover_examples()
102
  # Module-level model load: download + keep on CPU (ZeroGPU convention).
103
  # Real `.cuda()` happens inside @spaces.GPU workers.
104
  # ---------------------------------------------------------------------------
105
- print(f"[app] loading {HF_VARIANT} + π³ to CPU ...", flush=True)
106
  from dvsm.models.DVSM import DecoderViewSynthesisModel
107
  from viser_app.pi3_estimator import load_pi3_model
108
 
109
- # APP_MODEL_DIR: load the DVSM model from a local HF-format directory instead of
110
- # downloading from the Hub (faster / offline; used for local GPU-node testing).
 
111
  _LOCAL_MODEL_DIR = os.environ.get("APP_MODEL_DIR")
112
- if _LOCAL_MODEL_DIR:
113
- print(f"[app] loading DVSM from local dir {_LOCAL_MODEL_DIR}", flush=True)
114
- DVSM_MODEL_CPU = DecoderViewSynthesisModel.from_pretrained(_LOCAL_MODEL_DIR).eval()
115
- else:
116
- DVSM_MODEL_CPU = DecoderViewSynthesisModel.from_hf_pretrained(HF_VARIANT).eval()
 
 
 
 
 
 
117
  PI3_MODEL_CPU = load_pi3_model(flavor="pi3x", device="cpu").eval()
118
  print("[app] models ready (CPU)", flush=True)
119
 
@@ -167,11 +180,11 @@ def reconstruct_gpu(img_dir: str, target_h: int, target_w: int, max_views: int)
167
  # Reuse the module-level CPU model (moved to cuda inside reconstruct()).
168
  renderer, batch, ishape, c2w_r, c2w_t, intr = reconstruct(
169
  ckpt_root=None, ckpt_name="", scene_data=scene_data, cmd_lst=[],
170
- hf_variant=HF_VARIANT, model=DVSM_MODEL_CPU,
171
  )
172
  out_path = ARTIFACTS_DIR / f"{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}.recon.pt"
173
  save_artifact(renderer, batch, ishape, c2w_r, c2w_t, intr, out_path,
174
- hf_variant=HF_VARIANT)
175
  return str(out_path)
176
 
177
 
@@ -206,8 +219,13 @@ def _live_session_worker(artifact_path: str):
206
 
207
  repair_torch_cuda_state() # torch.compile vs ZeroGPU, see zerogpu_compat
208
  yield ("prep", 0.15, "Loading scene…")
209
- model = DVSM_MODEL_CPU.cuda().eval()
210
  artifact = torch.load(artifact_path, weights_only=False, map_location="cpu")
 
 
 
 
 
 
211
  renderer = build_renderer(model, artifact)
212
  scene_meta = artifact["scene_meta"]
213
 
@@ -605,14 +623,14 @@ with gr.Blocks(title="TTB-DVSM (ZeroGPU)", delete_cache=(3600, 3600)) as demo:
605
  "<h2 style='font-style:normal;font-weight:500;font-size:1.6em;"
606
  "margin-top:0;color:var(--body-text-color)'>"
607
  "<b style='color:#76B900'>T</b>est-<b style='color:#76B900'>t</b>ime "
608
- "MLP <b style='color:#76B900'>B</b>aking for Efficient Rendering of "
609
  "<b style='color:#76B900'>D</b>ecoder-only "
610
  "<b style='color:#76B900'>V</b>iew "
611
  "<b style='color:#76B900'>S</b>ynthesis "
612
  "<b style='color:#76B900'>M</b>odels"
613
  "</h2>"
614
  "<p style='color:var(--body-text-color);font-size:1.1em;margin-top:0.5em;line-height:1.6'>"
615
- "TTB-DVSM bakes the scene into fast weights in a single forward pass, "
616
  "and the result is browseable in the viser viewer."
617
  "</p>"
618
  "<div style='color:var(--body-text-color);font-size:1.1em;margin-top:0.8em;line-height:1.7'>"
 
17
  "Start / Continue" to fork a new worker and rebuild the viewer from the
18
  already-computed artifact on disk (no re-reconstruction).
19
 
20
+ Models (nvidia/DVSM): uploads are reconstructed with DVSM-TTB-p8-general-960p; the
21
+ gallery examples were baked with DVSM-TTB-p8-DL3DV-960p. A live session renders an
22
+ artifact with the model that baked it.
23
  """
24
  from __future__ import annotations
25
 
 
63
  # ---------------------------------------------------------------------------
64
  # Config
65
  # ---------------------------------------------------------------------------
66
+ # Uploaded images / videos are reconstructed with the general model. The gallery
67
+ # examples were baked with the DL3DV model. An artifact records the variant that
68
+ # baked it ("hf_variant") and has to be rendered with that same model.
69
+ UPLOAD_VARIANT = "DVSM-TTB-p8-general-960p"
70
+ EXAMPLE_VARIANT = "DVSM-TTB-p8-DL3DV-960p"
71
  # How long one held-GPU interactive session lasts before it must be renewed.
72
  # Bounded by the per-call duration cap and the user's daily quota
73
  # (anon 2min / free 5min / PRO 40min). Override via env for PRO accounts.
 
108
  # Module-level model load: download + keep on CPU (ZeroGPU convention).
109
  # Real `.cuda()` happens inside @spaces.GPU workers.
110
  # ---------------------------------------------------------------------------
111
+ print(f"[app] loading {UPLOAD_VARIANT}, {EXAMPLE_VARIANT} + π³ to CPU ...", flush=True)
112
  from dvsm.models.DVSM import DecoderViewSynthesisModel
113
  from viser_app.pi3_estimator import load_pi3_model
114
 
115
+ # APP_MODEL_DIR: load the DVSM models from a local directory instead of the Hub
116
+ # (faster / offline; used for local GPU-node testing). It holds one HF-format
117
+ # sub-directory per variant, i.e. the layout of the nvidia/DVSM repo.
118
  _LOCAL_MODEL_DIR = os.environ.get("APP_MODEL_DIR")
119
+
120
+
121
+ def _load_dvsm(variant):
122
+ if _LOCAL_MODEL_DIR:
123
+ path = Path(_LOCAL_MODEL_DIR) / variant
124
+ print(f"[app] loading {variant} from local dir {path}", flush=True)
125
+ return DecoderViewSynthesisModel.from_pretrained(str(path)).eval()
126
+ return DecoderViewSynthesisModel.from_hf_pretrained(variant).eval()
127
+
128
+
129
+ DVSM_MODELS_CPU = {v: _load_dvsm(v) for v in (UPLOAD_VARIANT, EXAMPLE_VARIANT)}
130
  PI3_MODEL_CPU = load_pi3_model(flavor="pi3x", device="cpu").eval()
131
  print("[app] models ready (CPU)", flush=True)
132
 
 
180
  # Reuse the module-level CPU model (moved to cuda inside reconstruct()).
181
  renderer, batch, ishape, c2w_r, c2w_t, intr = reconstruct(
182
  ckpt_root=None, ckpt_name="", scene_data=scene_data, cmd_lst=[],
183
+ hf_variant=UPLOAD_VARIANT, model=DVSM_MODELS_CPU[UPLOAD_VARIANT],
184
  )
185
  out_path = ARTIFACTS_DIR / f"{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}.recon.pt"
186
  save_artifact(renderer, batch, ishape, c2w_r, c2w_t, intr, out_path,
187
+ hf_variant=UPLOAD_VARIANT)
188
  return str(out_path)
189
 
190
 
 
219
 
220
  repair_torch_cuda_state() # torch.compile vs ZeroGPU, see zerogpu_compat
221
  yield ("prep", 0.15, "Loading scene…")
 
222
  artifact = torch.load(artifact_path, weights_only=False, map_location="cpu")
223
+ # Render with the model that baked this artifact (uploads vs gallery examples).
224
+ variant = artifact.get("hf_variant") or EXAMPLE_VARIANT
225
+ if variant not in DVSM_MODELS_CPU:
226
+ raise RuntimeError(f"artifact was baked with {variant}, which this app does not load")
227
+ print(f"[session] rendering with {variant}", flush=True)
228
+ model = DVSM_MODELS_CPU[variant].cuda().eval()
229
  renderer = build_renderer(model, artifact)
230
  scene_meta = artifact["scene_meta"]
231
 
 
623
  "<h2 style='font-style:normal;font-weight:500;font-size:1.6em;"
624
  "margin-top:0;color:var(--body-text-color)'>"
625
  "<b style='color:#76B900'>T</b>est-<b style='color:#76B900'>t</b>ime "
626
+ "<b style='color:#76B900'>B</b>aking for Efficient Rendering of "
627
  "<b style='color:#76B900'>D</b>ecoder-only "
628
  "<b style='color:#76B900'>V</b>iew "
629
  "<b style='color:#76B900'>S</b>ynthesis "
630
  "<b style='color:#76B900'>M</b>odels"
631
  "</h2>"
632
  "<p style='color:var(--body-text-color);font-size:1.1em;margin-top:0.5em;line-height:1.6'>"
633
+ "TTB bakes the scene into fast weights in a single forward pass, "
634
  "and the result is browseable in the viser viewer."
635
  "</p>"
636
  "<div style='color:var(--body-text-color);font-size:1.1em;margin-top:0.8em;line-height:1.7'>"