Ronaldo-GOAT commited on
Commit
3cee115
·
verified ·
1 Parent(s): 09df272

forgebench: Pixal3D MV uses upstream f7cf384 inference_mv (drop stale local copy), env-overridable paths, SETUP.md

Browse files
forgebench/code/SETUP.md ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Setup: the drivers are NOT self-contained
2
+
3
+ Each driver wraps an official repo + its conda env + HF weights. Install each official repo per its README, then point
4
+ the drivers at it (edit the path constants at the top of the file, or the env vars below).
5
+
6
+ | method | official repo @ commit | weights (HF) | driver | notes |
7
+ |---|---|---|---|---|
8
+ | Pixal3D 1v | TencentARC/Pixal3D @ `cdbb2bb` | `TencentARC/Pixal3D` (single-view `ckpts/*`, `pipeline.json`) | `baselines/pixal3d_ours/batch_pixal3d.py` | copy `mv_common.py`, `mv_proper.py` into the repo root; `PIXAL3D_REPO=<repo>` |
9
+ | Pixal3D 4v | TencentARC/Pixal3D @ **`f7cf384`** ("support multi-view image input", 2026-09-01), unmodified | same HF repo, **`ckpts/*_bf16_mv.safetensors` + `pipeline_mv.json`** (+ NAF upsampler, auto-downloaded) | `baselines/pixal3d_ours/batch_pixal3d_mv.py` | uses upstream `inference_mv.py` exactly; `PIXAL3D_UPSTREAM=<f7cf384 checkout>`, `PIXAL3D_OURS=<dir with mv_common.py>`. Same env as Pixal3D 1v (o_voxel, utils3d wheel per README). |
10
+ | ReconViaGen | GAP-LAB-CUHK-SZ/ReconViaGen @ `f672092` + `baselines/reconviagen_local.diff` | per its README | `baselines/batch_reconviagen.py` | edit `REPO`, `ENV`, `HF` |
11
+ | Amodal3R | Amodal3R official (ReconViaGen env) | per its README | `baselines/batch_amodal3r.py` | edit `AMODAL_ROOT`, `ENV`, `HF` |
12
+ | Hunyuan3D-2mv | Tencent Hunyuan3D-2 (2mv) | `tencent/Hunyuan3D-2mv` | `baselines/batch_hy3d_2mv.py` | `--seed 42 --simplify-faces 40000`; edit `HY_ROOT`, `HF` |
13
+ | Cupid | cupid3d/Cupid @ `10af9b2` | per its README | `baselines/batch_cupid.py` (1v) | multi-view driver pending |
14
+
15
+ Exact argument lists: `baselines/commands_reference.py` (`cmd_for`). Every driver is skip-existing and takes `--shard i --nshards n`.
16
+ Ours: `ours/` needs the training repo (`Ronaldo-GOAT/bert_simpson: migrator/code/mv-sam3d-for-6d-v2-ssflow/`), the SAM3D env, SS-flow 80k + SLAT 32k checkpoints (README.md).
forgebench/code/baselines/pixal3d_ours/autotune_cache.json.lock DELETED
File without changes
forgebench/code/baselines/pixal3d_ours/batch_pixal3d.py CHANGED
@@ -15,7 +15,7 @@ os.environ['OPENCV_IO_ENABLE_OPENEXR'] = '1'
15
  os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
16
  os.environ.setdefault("ATTN_BACKEND", "sdpa")
17
 
18
- REPO = "/lp-dev/jonghoon/mv-mesh/pixal3d"
19
  os.chdir(REPO)
20
  sys.path.insert(0, REPO)
21
  os.environ["FLEX_GEMM_AUTOTUNE_CACHE_PATH"] = os.path.join(REPO, 'autotune_cache.json')
 
15
  os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
16
  os.environ.setdefault("ATTN_BACKEND", "sdpa")
17
 
18
+ REPO = os.environ.get("PIXAL3D_REPO", "/lp-dev/jonghoon/mv-mesh/pixal3d") # TencentARC/Pixal3D @ cdbb2bb + mv_common.py, mv_proper.py
19
  os.chdir(REPO)
20
  sys.path.insert(0, REPO)
21
  os.environ["FLEX_GEMM_AUTOTUNE_CACHE_PATH"] = os.path.join(REPO, 'autotune_cache.json')
forgebench/code/baselines/pixal3d_ours/batch_pixal3d.py.pre8v DELETED
@@ -1,266 +0,0 @@
1
- #!/usr/bin/env python
2
- """Batch Pixal3D driver: load pipeline ONCE, process an object list.
3
-
4
- Usage:
5
- python batch_pixal3d.py --selection SEL.json --inputs DIR --out OUTDIR \
6
- --views 1|2 [--seed 42] [--limit N] [--resolution 1536]
7
-
8
- 1v -> single front image; 2v -> front+side mean-pool (inference_mv style).
9
- Inputs are RGBA crops (alpha = mask), so preprocess_image uses alpha directly
10
- (no rembg download). Idempotent: existing OUTDIR/<object>.glb is skipped.
11
- """
12
- import os, sys, argparse, json, time, traceback
13
-
14
- os.environ['OPENCV_IO_ENABLE_OPENEXR'] = '1'
15
- os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
16
- os.environ.setdefault("ATTN_BACKEND", "sdpa")
17
-
18
- REPO = "/lp-dev/jonghoon/mv-mesh/pixal3d"
19
- os.chdir(REPO)
20
- sys.path.insert(0, REPO)
21
- os.environ["FLEX_GEMM_AUTOTUNE_CACHE_PATH"] = os.path.join(REPO, 'autotune_cache.json')
22
-
23
- import torch
24
- import numpy as np
25
- from PIL import Image
26
-
27
- import mv_common
28
- mv_common.patch_rembg_lazy()
29
- import o_voxel
30
- import inference as _inf
31
- # NOTE: 'hack' MV (inference_mv.build_mv_cond_functions) is REMOVED — it projected
32
- # every view through the front camera (geometrically wrong). Only corrected 'proper' remains.
33
- from mv_proper import build_mv_cond_functions_proper
34
-
35
-
36
- def export_vertex_colored(mesh, out, pbr_attr_layout, simplify_target=150000):
37
- """FAST path: run the AUTHORS' own GPU quadric simplifier (cumesh, via
38
- Mesh.simplify -> the same primitive o_voxel.postprocess.to_glb uses for its
39
- decimation_target) to lower the res-1536 dual-grid mesh (~3-32M faces) down to
40
- ~simplify_target faces, THEN sample the decoder's native per-vertex base_color
41
- from the attr volume at the SIMPLIFIED vertices (one trilinear grid_sample) and
42
- write a vertex-colored GLB. Skips only the expensive UV-parameterize + 4096
43
- texture bake (eval renders vertex colors). Colors stay exact because they are
44
- re-sampled from the volume at the new vertex positions (not interpolated).
45
- Orientation matches the to_glb path (same Y/Z swap + rot)."""
46
- import trimesh
47
- if simplify_target and mesh.faces.shape[0] > simplify_target:
48
- mesh.simplify(int(simplify_target)) # authors' cumesh quadric decimate
49
- va = mesh.query_vertex_attrs() # [N, C] trilinear at verts
50
- bc = va[:, pbr_attr_layout['base_color']]
51
- rgb = (torch.clamp(bc, 0, 1) * 255).to(torch.uint8).cpu().numpy()
52
- verts = mesh.vertices.detach().cpu().numpy().astype(np.float64)
53
- faces = mesh.faces.detach().cpu().numpy()
54
- v1 = verts[:, 2].copy(); v2 = (-verts[:, 1]).copy()
55
- verts[:, 1] = v1; verts[:, 2] = v2 # Y/Z swap (as in to_glb)
56
- vc = np.concatenate([rgb, np.full((len(rgb), 1), 255, np.uint8)], axis=1)
57
- tm = trimesh.Trimesh(vertices=verts, faces=faces, vertex_colors=vc, process=False)
58
- rot = np.array([[-1,0,0,0],[0,0,-1,0],[0,-1,0,0],[0,0,0,1]], dtype=np.float64)
59
- tm.apply_transform(rot)
60
- tmp = out + ".tmp.glb"; tm.export(tmp); os.replace(tmp, out)
61
-
62
-
63
- def wait_for_vram(min_free_mib=16000, timeout_s=10800, poll_s=20):
64
- """Block until the current CUDA device has >= min_free_mib free (after releasing
65
- our own transient cache), so the res-1536 sampling spike (~13.5GB) can run without
66
- OOMing against co-located sibling jobs sharing the GPU. Our resident model weights
67
- are NOT counted as free, so min_free just needs to cover the spike + margin."""
68
- import torch, time as _t
69
- t0 = _t.time()
70
- while True:
71
- torch.cuda.empty_cache()
72
- free, total = torch.cuda.mem_get_info()
73
- free_mib = free / (1024 * 1024)
74
- if free_mib >= min_free_mib:
75
- return
76
- if _t.time() - t0 > timeout_s:
77
- print(f"[vram] timeout waiting for {min_free_mib}MiB (free={free_mib:.0f}); proceeding", flush=True)
78
- return
79
- print(f"[vram] free={free_mib:.0f}MiB < {min_free_mib}MiB, waiting for GPU headroom...", flush=True)
80
- _t.sleep(poll_s)
81
-
82
-
83
- def run_one_resilient(*args, retries=6, **kw):
84
- """Run one object, retrying on CUDA OOM after clearing cache + waiting for VRAM
85
- headroom (co-located sibling jobs release memory between their own objects)."""
86
- import torch
87
- for attempt in range(retries):
88
- try:
89
- wait_for_vram()
90
- return run_one(*args, **kw)
91
- except torch.cuda.OutOfMemoryError:
92
- torch.cuda.empty_cache()
93
- import time as _t; _t.sleep(15)
94
- print(f"[oom] attempt {attempt+1}/{retries} hit CUDA OOM; cleared cache, retrying", flush=True)
95
- # last attempt without swallowing
96
- wait_for_vram(min_free_mib=18000)
97
- return run_one(*args, **kw)
98
-
99
-
100
- def _load_c2w_for_views(renders_dir, obj_name, view_tags):
101
- """Load GT camera-to-world (c2w_cv, 4x4) for each view from renders/<obj>_<tag>.npz."""
102
- import numpy as _np
103
- mats = []
104
- for t in view_tags:
105
- npz = os.path.join(renders_dir, f"{obj_name}_{t}.npz")
106
- z = _np.load(npz)
107
- mats.append(_np.asarray(z["c2w_cv"], dtype=_np.float64))
108
- return mats
109
-
110
-
111
- def run_one(pipeline, moge, imgs, out, seed, resolution, image_resolution, low_vram,
112
- vertex_color=True, mv_mode="proper", side_azimuth=90.0, simplify_target=150000,
113
- renders_dir=None, obj_name=None, view_tags=None):
114
- outdir = os.path.dirname(os.path.abspath(out))
115
- os.makedirs(outdir, exist_ok=True)
116
- pre_imgs, tmp_paths = [], []
117
- for i, p in enumerate(imgs):
118
- img = Image.open(p) # RGBA -> preprocess uses alpha directly
119
- pi = pipeline.preprocess_image(img)
120
- pre_imgs.append(pi)
121
- tp = os.path.join(outdir, f"_tmp_mv_{i}_{int(time.time()*1000)}.png")
122
- pi.save(tp); tmp_paths.append(tp)
123
-
124
- cams = []
125
- for tp in tmp_paths:
126
- cp = _inf.get_camera_params_wild_moge(tp, moge, device="cuda",
127
- mesh_scale=1.0, extend_pixel=0,
128
- image_resolution=image_resolution)
129
- cams.append(cp)
130
- for tp in tmp_paths:
131
- try: os.remove(tp)
132
- except OSError: pass
133
-
134
- if len(pre_imgs) > 1:
135
- # CORRECTED proper MV: per-view back-projection through each view's REAL
136
- # camera, read per-object from GT renders (renders/<obj>_<tag>.npz c2w_cv),
137
- # then simple averaging (paper Sec 3.2.3). Fixes the previous zero-elevation
138
- # / hard-coded-azimuth-sign rig. 'hack' mode is REMOVED.
139
- if mv_mode != "proper":
140
- raise ValueError(f"mv_mode={mv_mode!r} removed; only 'proper' is supported")
141
- if renders_dir is None or obj_name is None or view_tags is None:
142
- raise ValueError("proper MV requires renders_dir, obj_name, view_tags")
143
- c2w_list = _load_c2w_for_views(renders_dir, obj_name, view_tags)
144
- from mv_proper import real_view_transforms
145
- vt = real_view_transforms(c2w_list, cams[0]['distance'], device="cuda")
146
- build_mv_cond_functions_proper(pipeline, pre_imgs, cams, view_transforms=vt)
147
- # len==1: single view -> pipeline uses its native official get_proj_cond (no patch)
148
-
149
- torch.manual_seed(seed)
150
- pipeline_type = f"{resolution if resolution > 0 else (1024 if low_vram else 1536)}_cascade"
151
- mesh_list, (shape_slat, tex_slat, res) = pipeline.run(
152
- pre_imgs[0], camera_params=cams[0], seed=seed,
153
- sparse_structure_sampler_params={"steps": 12, "guidance_strength": 7.5, "guidance_rescale": 0.7, "rescale_t": 5.0},
154
- shape_slat_sampler_params={"steps": 12, "guidance_strength": 7.5, "guidance_rescale": 0.5, "rescale_t": 3.0},
155
- tex_slat_sampler_params={"steps": 12, "guidance_strength": 1.0, "guidance_rescale": 0.0, "rescale_t": 3.0},
156
- preprocess_image=False, return_latent=True, pipeline_type=pipeline_type, max_num_tokens=49152,
157
- )
158
- mesh = mesh_list[0]
159
- if vertex_color:
160
- export_vertex_colored(mesh, out, pipeline.pbr_attr_layout, simplify_target=simplify_target)
161
- return
162
- glb = o_voxel.postprocess.to_glb(
163
- vertices=mesh.vertices, faces=mesh.faces, attr_volume=mesh.attrs,
164
- coords=mesh.coords, attr_layout=pipeline.pbr_attr_layout,
165
- grid_size=res, aabb=[[-0.5, -0.5, -0.5], [0.5, 0.5, 0.5]],
166
- decimation_target=1000000, texture_size=4096,
167
- remesh=True, remesh_band=1, remesh_project=0, use_tqdm=False,
168
- )
169
- rot = np.array([[-1,0,0,0],[0,0,-1,0],[0,-1,0,0],[0,0,0,1]], dtype=np.float64)
170
- glb.apply_transform(rot)
171
- tmp = out + ".tmp.glb"
172
- glb.export(tmp, extension_webp=True)
173
- os.replace(tmp, out)
174
-
175
-
176
- def main():
177
- ap = argparse.ArgumentParser()
178
- ap.add_argument("--selection", required=True)
179
- ap.add_argument("--inputs", required=True)
180
- ap.add_argument("--out", required=True)
181
- ap.add_argument("--views", type=int, required=True, choices=[1, 2, 4])
182
- ap.add_argument("--seed", type=int, default=42)
183
- ap.add_argument("--limit", type=int, default=None)
184
- ap.add_argument("--resolution", type=int, default=1536)
185
- ap.add_argument("--image_resolution", type=int, default=512)
186
- ap.add_argument("--low_vram", action="store_true")
187
- ap.add_argument("--to_glb", dest="vertex_color", action="store_false", default=True,
188
- help="use slow textured to_glb export (default: fast vertex-color)")
189
- ap.add_argument("--shard", type=int, default=0)
190
- ap.add_argument("--nshards", type=int, default=1)
191
- ap.add_argument("--reverse", action="store_true",
192
- help="process objects in reverse order (convergent co-located "
193
- "helper against a forward run; skip-existing + atomic write)")
194
- ap.add_argument("--mv_mode", choices=["proper"], default="proper",
195
- help="proper=paper-faithful per-view back-projection through REAL "
196
- "per-object GT extrinsics + average ('hack' removed)")
197
- ap.add_argument("--renders", default=None,
198
- help="dir with GT renders/<obj>_<view>.npz (c2w_cv); "
199
- "default: <inputs>/../renders")
200
- ap.add_argument("--side_azimuth", type=float, default=90.0,
201
- help="azimuth offset (deg) of the side view from front, about up-axis")
202
- ap.add_argument("--simplify_target", type=int, default=150000,
203
- help="target face count for the authors' cumesh quadric decimate "
204
- "in the vertex-color export (0=off); ~150k renders/evals fast "
205
- "and avoids the nvdiffrast CUDA-700 crash on the raw dual-grid mesh")
206
- args = ap.parse_args()
207
-
208
- with open(args.selection) as f:
209
- sel = json.load(f)["selections"]
210
- objects = [s["object"] for s in sel]
211
- if args.limit:
212
- objects = objects[:args.limit]
213
- if args.nshards > 1:
214
- objects = objects[args.shard::args.nshards]
215
- if args.reverse:
216
- objects = objects[::-1]
217
- outdir = os.path.abspath(args.out)
218
- os.makedirs(outdir, exist_ok=True)
219
- renders_dir = args.renders or os.path.join(os.path.dirname(os.path.abspath(args.inputs)), "renders")
220
- view_tags = {1: ["front"], 2: ["front", "side"],
221
- 4: ["front", "side", "back", "oside"]}[args.views]
222
-
223
- jobs = []
224
- for name in objects:
225
- out = os.path.join(outdir, f"{name}.glb")
226
- imgs = [os.path.join(args.inputs, f"{name}_{t}.png") for t in view_tags]
227
- jobs.append((name, imgs, out))
228
- todo = [j for j in jobs if not os.path.isfile(j[2])]
229
- print(f"[pixal] {len(jobs)} obj, {len(jobs)-len(todo)} done, {len(todo)} to run "
230
- f"| views={args.views} res={args.resolution}", flush=True)
231
-
232
- pipeline = _inf.init_pipeline(_inf.MODEL_PATH, low_vram=args.low_vram)
233
- moge = _inf.load_moge_model(device="cuda")
234
-
235
- n_ok = n_fail = n_skip = 0
236
- for i, (name, imgs, out) in enumerate(jobs, 1):
237
- t0 = time.time()
238
- try:
239
- if os.path.isfile(out):
240
- n_skip += 1
241
- print(f"[{i}/{len(jobs)}] {name} SKIP", flush=True)
242
- continue
243
- for p in imgs:
244
- if not os.path.isfile(p):
245
- raise FileNotFoundError(p)
246
- run_one_resilient(pipeline, moge, imgs, out, args.seed, args.resolution,
247
- args.image_resolution, args.low_vram, vertex_color=args.vertex_color,
248
- mv_mode=args.mv_mode, side_azimuth=args.side_azimuth,
249
- simplify_target=args.simplify_target,
250
- renders_dir=renders_dir, obj_name=name, view_tags=view_tags)
251
- torch.cuda.empty_cache()
252
- n_ok += 1
253
- print(f"[{i}/{len(jobs)}] {name} OK {time.time()-t0:.1f}s -> {out}", flush=True)
254
- except Exception:
255
- n_fail += 1
256
- traceback.print_exc()
257
- print(f"[{i}/{len(jobs)}] {name} FAIL {time.time()-t0:.1f}s", flush=True)
258
- finally:
259
- try: torch.cuda.empty_cache()
260
- except Exception: pass
261
- print(f"BATCH DONE ok={n_ok} fail={n_fail} skip={n_skip} total={len(jobs)}", flush=True)
262
- return 1 if n_fail else 0
263
-
264
-
265
- if __name__ == "__main__":
266
- sys.exit(main())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
forgebench/code/baselines/pixal3d_ours/batch_pixal3d_mv.py CHANGED
@@ -32,8 +32,8 @@ os.environ['OPENCV_IO_ENABLE_OPENEXR'] = '1'
32
  os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
33
  os.environ.setdefault("ATTN_BACKEND", "sdpa")
34
 
35
- UPSTREAM = "/lp-dev/jonghoon/mv-mesh/.debug_pixal3d_verify/pixal3d_upstream"
36
- OURREPO = "/lp-dev/jonghoon/mv-mesh/pixal3d"
37
  os.chdir(UPSTREAM)
38
  sys.path.insert(0, UPSTREAM)
39
  sys.path.insert(1, OURREPO) # mv_common only
 
32
  os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
33
  os.environ.setdefault("ATTN_BACKEND", "sdpa")
34
 
35
+ UPSTREAM = os.environ.get("PIXAL3D_UPSTREAM", "/lp-dev/jonghoon/mv-mesh/.debug_pixal3d_verify/pixal3d_upstream") # TencentARC/Pixal3D @ f7cf384 (official MV)
36
+ OURREPO = os.environ.get("PIXAL3D_OURS", "/lp-dev/jonghoon/mv-mesh/pixal3d") # dir holding mv_common.py
37
  os.chdir(UPSTREAM)
38
  sys.path.insert(0, UPSTREAM)
39
  sys.path.insert(1, OURREPO) # mv_common only
forgebench/code/baselines/pixal3d_ours/inference_mv.py DELETED
@@ -1,178 +0,0 @@
1
- """
2
- EXPERIMENTAL multi-view inference for Pixal3D.
3
-
4
- IMPORTANT / HONEST CAVEAT:
5
- Pixal3D's released `proj` pipeline + weights are architecturally SINGLE-IMAGE.
6
- There is NO native multi-view inference. This script constructs a multi-view
7
- variant by the ONLY mechanism the code allows: it runs the DinoV3ProjFeatureExtractor
8
- on V views (each with its own MoGe-estimated camera), then MEAN-POOLS the projected
9
- grid features (and global tokens) across views before the sparse coords indexing.
10
-
11
- Two known limitations (do not report this as "true" multi-view):
12
- 1. ProjGrid.forward asserts transform_matrix is None -> every view is projected
13
- through the SAME canonical front-view camera. Non-frontal views (back/top/45deg)
14
- therefore paint their features onto geometrically wrong grid cells.
15
- 2. The DiT was trained on ONE view per object, so averaged multi-view features are
16
- out-of-distribution; there is no learned view-fusion to exploit.
17
-
18
- Usage:
19
- python inference_mv.py --images v1.jpg v2.jpg v3.jpg --output out.glb --seed 42
20
- """
21
- import os, argparse, math, time, types
22
- import torch, numpy as np
23
- from PIL import Image
24
-
25
- os.environ['OPENCV_IO_ENABLE_OPENEXR'] = '1'
26
- os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
27
- os.environ.setdefault("ATTN_BACKEND", "sdpa")
28
- os.environ["FLEX_GEMM_AUTOTUNE_CACHE_PATH"] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'autotune_cache.json')
29
-
30
- import mv_common
31
- mv_common.patch_rembg_lazy()
32
-
33
- from pixal3d.pipelines import Pixal3DImageTo3DPipeline
34
- from pixal3d.modules.sparse import SparseTensor
35
- import o_voxel
36
- import inference as _inf # reuse init_pipeline / camera helpers
37
-
38
-
39
- def build_mv_cond_functions(pipeline, images, cams):
40
- """Monkeypatch get_proj_cond_ss / get_proj_cond_shape to use V views + mean-pool.
41
-
42
- images: list of V preprocessed PIL images.
43
- cams: list of V dicts {camera_angle_x, distance, mesh_scale}.
44
- """
45
- device_getter = lambda: pipeline.device
46
-
47
- def _cam_tensors(dev):
48
- cax = torch.tensor([c['camera_angle_x'] for c in cams], device=dev)
49
- dist = torch.tensor([c['distance'] for c in cams], device=dev)
50
- scal = torch.tensor([c.get('mesh_scale', 1.0) for c in cams], device=dev)
51
- return cax, dist, scal
52
-
53
- @torch.no_grad()
54
- def get_proj_cond_ss(self, image, camera_angle_x=None, distance=None, mesh_scale=None):
55
- dev = self.device
56
- m = self.image_cond_model_ss
57
- if self.low_vram: m.to(dev)
58
- cax, dist, scal = _cam_tensors(dev)
59
- z_global, z_proj = m(images, camera_angle_x=cax, distance=dist, mesh_scale=scal) # [V,..],[V,R3,C]
60
- if self.low_vram: m.cpu()
61
- z_global = z_global.mean(dim=0, keepdim=True) # [1,tok,C]
62
- z_proj = z_proj.mean(dim=0, keepdim=True) # [1,R3,C]
63
- return {
64
- 'cond': {'global': z_global, 'proj': z_proj},
65
- 'neg_cond': {'global': torch.zeros_like(z_global), 'proj': torch.zeros_like(z_proj)},
66
- }
67
-
68
- @torch.no_grad()
69
- def get_proj_cond_shape(self, image_cond_model, image, coords,
70
- camera_angle_x=None, distance=None, mesh_scale=None,
71
- grid_resolution_override=None):
72
- dev = self.device
73
- if self.low_vram: image_cond_model.to(dev)
74
- orig_grid_res = image_cond_model.grid_resolution
75
- if grid_resolution_override is not None and grid_resolution_override != orig_grid_res:
76
- image_cond_model.grid_resolution = grid_resolution_override
77
- image_cond_model.proj_grid = image_cond_model.proj_grid.__class__(
78
- grid_resolution=grid_resolution_override,
79
- image_resolution=image_cond_model.proj_grid.image_resolution,
80
- ).to(dev)
81
- cax, dist, scal = _cam_tensors(dev)
82
- z_global, z_proj = image_cond_model(images, camera_angle_x=cax, distance=dist, mesh_scale=scal)
83
- # mean-pool over views
84
- z_global = z_global.mean(dim=0, keepdim=True) # [1,tok,C]
85
- z_proj = z_proj.mean(dim=0, keepdim=True) # [1,R3,C]
86
- grid_res = image_cond_model.grid_resolution
87
- z_proj_grid = z_proj.reshape(1, grid_res, grid_res, grid_res, -1)
88
- bi = coords[:, 0].long(); xc = coords[:, 1].long(); yc = coords[:, 2].long(); zc = coords[:, 3].long()
89
- z_proj_sparse = z_proj_grid[bi, xc, yc, zc]
90
- z_proj_st = SparseTensor(feats=z_proj_sparse, coords=coords)
91
- if grid_resolution_override is not None and grid_resolution_override != orig_grid_res:
92
- image_cond_model.grid_resolution = orig_grid_res
93
- image_cond_model.proj_grid = image_cond_model.proj_grid.__class__(
94
- grid_resolution=orig_grid_res,
95
- image_resolution=image_cond_model.proj_grid.image_resolution,
96
- ).to(dev)
97
- if self.low_vram: image_cond_model.cpu()
98
- return {
99
- 'cond': {'global': z_global, 'proj': z_proj_st},
100
- 'neg_cond': {'global': torch.zeros_like(z_global),
101
- 'proj': SparseTensor(feats=torch.zeros_like(z_proj_sparse), coords=coords)},
102
- }
103
-
104
- pipeline.get_proj_cond_ss = types.MethodType(get_proj_cond_ss, pipeline)
105
- pipeline.get_proj_cond_shape = types.MethodType(get_proj_cond_shape, pipeline)
106
-
107
-
108
- def main():
109
- ap = argparse.ArgumentParser()
110
- ap.add_argument("--images", nargs="+", required=True)
111
- ap.add_argument("--masks", nargs="+", default=None, help="optional per-view masks (aligned with --images)")
112
- ap.add_argument("--output", required=True)
113
- ap.add_argument("--seed", type=int, default=42)
114
- ap.add_argument("--low_vram", action="store_true")
115
- ap.add_argument("--resolution", type=int, default=-1)
116
- ap.add_argument("--image_resolution", type=int, default=512)
117
- args = ap.parse_args()
118
-
119
- pipeline = _inf.init_pipeline(_inf.MODEL_PATH, low_vram=args.low_vram)
120
-
121
- # preprocess each view (bg removal + crop) and estimate per-view camera via MoGe
122
- pre_imgs, tmp_paths = [], []
123
- outdir = os.path.dirname(os.path.abspath(args.output))
124
- os.makedirs(outdir, exist_ok=True)
125
- masks = args.masks if args.masks else [None] * len(args.images)
126
- assert len(masks) == len(args.images), "masks must align with images"
127
- for i, p in enumerate(args.images):
128
- img = mv_common.load_rgba(p, masks[i])
129
- pi = pipeline.preprocess_image(img)
130
- pre_imgs.append(pi)
131
- tp = os.path.join(outdir, f"_tmp_mv_{i}_{int(time.time()*1000)}.png")
132
- pi.save(tp); tmp_paths.append(tp)
133
-
134
- print("[MoGe-2] estimating per-view cameras...")
135
- moge = _inf.load_moge_model(device="cuda")
136
- cams = []
137
- for tp in tmp_paths:
138
- cp = _inf.get_camera_params_wild_moge(tp, moge, device="cuda",
139
- mesh_scale=1.0, extend_pixel=0,
140
- image_resolution=args.image_resolution)
141
- cams.append(cp)
142
- print(f" view cam: angle_x={cp['camera_angle_x']:.4f} dist={cp['distance']:.4f}")
143
- moge.cpu(); del moge; torch.cuda.empty_cache()
144
- for tp in tmp_paths:
145
- try: os.remove(tp)
146
- except OSError: pass
147
-
148
- build_mv_cond_functions(pipeline, pre_imgs, cams)
149
-
150
- seed = args.seed
151
- torch.manual_seed(seed)
152
- pipeline_type = f"{args.resolution if args.resolution > 0 else (1024 if args.low_vram else 1536)}_cascade"
153
- print(f"[MV-Inference] {len(pre_imgs)} views, pipeline_type={pipeline_type}")
154
- # camera_params passed to run() are ignored by patched cond fns; supply view-0's as placeholder
155
- mesh_list, (shape_slat, tex_slat, res) = pipeline.run(
156
- pre_imgs[0], camera_params=cams[0], seed=seed,
157
- sparse_structure_sampler_params={"steps": 12, "guidance_strength": 7.5, "guidance_rescale": 0.7, "rescale_t": 5.0},
158
- shape_slat_sampler_params={"steps": 12, "guidance_strength": 7.5, "guidance_rescale": 0.5, "rescale_t": 3.0},
159
- tex_slat_sampler_params={"steps": 12, "guidance_strength": 1.0, "guidance_rescale": 0.0, "rescale_t": 3.0},
160
- preprocess_image=False, return_latent=True, pipeline_type=pipeline_type, max_num_tokens=49152,
161
- )
162
- mesh = mesh_list[0]
163
- print("[MV-Inference] extracting GLB...")
164
- glb = o_voxel.postprocess.to_glb(
165
- vertices=mesh.vertices, faces=mesh.faces, attr_volume=mesh.attrs,
166
- coords=mesh.coords, attr_layout=pipeline.pbr_attr_layout,
167
- grid_size=res, aabb=[[-0.5, -0.5, -0.5], [0.5, 0.5, 0.5]],
168
- decimation_target=1000000, texture_size=4096,
169
- remesh=True, remesh_band=1, remesh_project=0, use_tqdm=True,
170
- )
171
- rot = np.array([[-1,0,0,0],[0,0,-1,0],[0,-1,0,0],[0,0,0,1]], dtype=np.float64)
172
- glb.apply_transform(rot)
173
- glb.export(args.output, extension_webp=True)
174
- print(f"[Done] GLB saved to: {args.output}")
175
-
176
-
177
- if __name__ == "__main__":
178
- main()