forgebench: Pixal3D MV uses upstream f7cf384 inference_mv (drop stale local copy), env-overridable paths, SETUP.md
Browse files- forgebench/code/SETUP.md +16 -0
- forgebench/code/baselines/pixal3d_ours/autotune_cache.json.lock +0 -0
- forgebench/code/baselines/pixal3d_ours/batch_pixal3d.py +1 -1
- forgebench/code/baselines/pixal3d_ours/batch_pixal3d.py.pre8v +0 -266
- forgebench/code/baselines/pixal3d_ours/batch_pixal3d_mv.py +2 -2
- forgebench/code/baselines/pixal3d_ours/inference_mv.py +0 -178
forgebench/code/SETUP.md
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Setup: the drivers are NOT self-contained
|
| 2 |
+
|
| 3 |
+
Each driver wraps an official repo + its conda env + HF weights. Install each official repo per its README, then point
|
| 4 |
+
the drivers at it (edit the path constants at the top of the file, or the env vars below).
|
| 5 |
+
|
| 6 |
+
| method | official repo @ commit | weights (HF) | driver | notes |
|
| 7 |
+
|---|---|---|---|---|
|
| 8 |
+
| Pixal3D 1v | TencentARC/Pixal3D @ `cdbb2bb` | `TencentARC/Pixal3D` (single-view `ckpts/*`, `pipeline.json`) | `baselines/pixal3d_ours/batch_pixal3d.py` | copy `mv_common.py`, `mv_proper.py` into the repo root; `PIXAL3D_REPO=<repo>` |
|
| 9 |
+
| Pixal3D 4v | TencentARC/Pixal3D @ **`f7cf384`** ("support multi-view image input", 2026-09-01), unmodified | same HF repo, **`ckpts/*_bf16_mv.safetensors` + `pipeline_mv.json`** (+ NAF upsampler, auto-downloaded) | `baselines/pixal3d_ours/batch_pixal3d_mv.py` | uses upstream `inference_mv.py` exactly; `PIXAL3D_UPSTREAM=<f7cf384 checkout>`, `PIXAL3D_OURS=<dir with mv_common.py>`. Same env as Pixal3D 1v (o_voxel, utils3d wheel per README). |
|
| 10 |
+
| ReconViaGen | GAP-LAB-CUHK-SZ/ReconViaGen @ `f672092` + `baselines/reconviagen_local.diff` | per its README | `baselines/batch_reconviagen.py` | edit `REPO`, `ENV`, `HF` |
|
| 11 |
+
| Amodal3R | Amodal3R official (ReconViaGen env) | per its README | `baselines/batch_amodal3r.py` | edit `AMODAL_ROOT`, `ENV`, `HF` |
|
| 12 |
+
| Hunyuan3D-2mv | Tencent Hunyuan3D-2 (2mv) | `tencent/Hunyuan3D-2mv` | `baselines/batch_hy3d_2mv.py` | `--seed 42 --simplify-faces 40000`; edit `HY_ROOT`, `HF` |
|
| 13 |
+
| Cupid | cupid3d/Cupid @ `10af9b2` | per its README | `baselines/batch_cupid.py` (1v) | multi-view driver pending |
|
| 14 |
+
|
| 15 |
+
Exact argument lists: `baselines/commands_reference.py` (`cmd_for`). Every driver is skip-existing and takes `--shard i --nshards n`.
|
| 16 |
+
Ours: `ours/` needs the training repo (`Ronaldo-GOAT/bert_simpson: migrator/code/mv-sam3d-for-6d-v2-ssflow/`), the SAM3D env, SS-flow 80k + SLAT 32k checkpoints (README.md).
|
forgebench/code/baselines/pixal3d_ours/autotune_cache.json.lock
DELETED
|
File without changes
|
forgebench/code/baselines/pixal3d_ours/batch_pixal3d.py
CHANGED
|
@@ -15,7 +15,7 @@ os.environ['OPENCV_IO_ENABLE_OPENEXR'] = '1'
|
|
| 15 |
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
|
| 16 |
os.environ.setdefault("ATTN_BACKEND", "sdpa")
|
| 17 |
|
| 18 |
-
REPO = "/lp-dev/jonghoon/mv-mesh/pixal3d"
|
| 19 |
os.chdir(REPO)
|
| 20 |
sys.path.insert(0, REPO)
|
| 21 |
os.environ["FLEX_GEMM_AUTOTUNE_CACHE_PATH"] = os.path.join(REPO, 'autotune_cache.json')
|
|
|
|
| 15 |
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
|
| 16 |
os.environ.setdefault("ATTN_BACKEND", "sdpa")
|
| 17 |
|
| 18 |
+
REPO = os.environ.get("PIXAL3D_REPO", "/lp-dev/jonghoon/mv-mesh/pixal3d") # TencentARC/Pixal3D @ cdbb2bb + mv_common.py, mv_proper.py
|
| 19 |
os.chdir(REPO)
|
| 20 |
sys.path.insert(0, REPO)
|
| 21 |
os.environ["FLEX_GEMM_AUTOTUNE_CACHE_PATH"] = os.path.join(REPO, 'autotune_cache.json')
|
forgebench/code/baselines/pixal3d_ours/batch_pixal3d.py.pre8v
DELETED
|
@@ -1,266 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python
|
| 2 |
-
"""Batch Pixal3D driver: load pipeline ONCE, process an object list.
|
| 3 |
-
|
| 4 |
-
Usage:
|
| 5 |
-
python batch_pixal3d.py --selection SEL.json --inputs DIR --out OUTDIR \
|
| 6 |
-
--views 1|2 [--seed 42] [--limit N] [--resolution 1536]
|
| 7 |
-
|
| 8 |
-
1v -> single front image; 2v -> front+side mean-pool (inference_mv style).
|
| 9 |
-
Inputs are RGBA crops (alpha = mask), so preprocess_image uses alpha directly
|
| 10 |
-
(no rembg download). Idempotent: existing OUTDIR/<object>.glb is skipped.
|
| 11 |
-
"""
|
| 12 |
-
import os, sys, argparse, json, time, traceback
|
| 13 |
-
|
| 14 |
-
os.environ['OPENCV_IO_ENABLE_OPENEXR'] = '1'
|
| 15 |
-
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
|
| 16 |
-
os.environ.setdefault("ATTN_BACKEND", "sdpa")
|
| 17 |
-
|
| 18 |
-
REPO = "/lp-dev/jonghoon/mv-mesh/pixal3d"
|
| 19 |
-
os.chdir(REPO)
|
| 20 |
-
sys.path.insert(0, REPO)
|
| 21 |
-
os.environ["FLEX_GEMM_AUTOTUNE_CACHE_PATH"] = os.path.join(REPO, 'autotune_cache.json')
|
| 22 |
-
|
| 23 |
-
import torch
|
| 24 |
-
import numpy as np
|
| 25 |
-
from PIL import Image
|
| 26 |
-
|
| 27 |
-
import mv_common
|
| 28 |
-
mv_common.patch_rembg_lazy()
|
| 29 |
-
import o_voxel
|
| 30 |
-
import inference as _inf
|
| 31 |
-
# NOTE: 'hack' MV (inference_mv.build_mv_cond_functions) is REMOVED — it projected
|
| 32 |
-
# every view through the front camera (geometrically wrong). Only corrected 'proper' remains.
|
| 33 |
-
from mv_proper import build_mv_cond_functions_proper
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
def export_vertex_colored(mesh, out, pbr_attr_layout, simplify_target=150000):
|
| 37 |
-
"""FAST path: run the AUTHORS' own GPU quadric simplifier (cumesh, via
|
| 38 |
-
Mesh.simplify -> the same primitive o_voxel.postprocess.to_glb uses for its
|
| 39 |
-
decimation_target) to lower the res-1536 dual-grid mesh (~3-32M faces) down to
|
| 40 |
-
~simplify_target faces, THEN sample the decoder's native per-vertex base_color
|
| 41 |
-
from the attr volume at the SIMPLIFIED vertices (one trilinear grid_sample) and
|
| 42 |
-
write a vertex-colored GLB. Skips only the expensive UV-parameterize + 4096
|
| 43 |
-
texture bake (eval renders vertex colors). Colors stay exact because they are
|
| 44 |
-
re-sampled from the volume at the new vertex positions (not interpolated).
|
| 45 |
-
Orientation matches the to_glb path (same Y/Z swap + rot)."""
|
| 46 |
-
import trimesh
|
| 47 |
-
if simplify_target and mesh.faces.shape[0] > simplify_target:
|
| 48 |
-
mesh.simplify(int(simplify_target)) # authors' cumesh quadric decimate
|
| 49 |
-
va = mesh.query_vertex_attrs() # [N, C] trilinear at verts
|
| 50 |
-
bc = va[:, pbr_attr_layout['base_color']]
|
| 51 |
-
rgb = (torch.clamp(bc, 0, 1) * 255).to(torch.uint8).cpu().numpy()
|
| 52 |
-
verts = mesh.vertices.detach().cpu().numpy().astype(np.float64)
|
| 53 |
-
faces = mesh.faces.detach().cpu().numpy()
|
| 54 |
-
v1 = verts[:, 2].copy(); v2 = (-verts[:, 1]).copy()
|
| 55 |
-
verts[:, 1] = v1; verts[:, 2] = v2 # Y/Z swap (as in to_glb)
|
| 56 |
-
vc = np.concatenate([rgb, np.full((len(rgb), 1), 255, np.uint8)], axis=1)
|
| 57 |
-
tm = trimesh.Trimesh(vertices=verts, faces=faces, vertex_colors=vc, process=False)
|
| 58 |
-
rot = np.array([[-1,0,0,0],[0,0,-1,0],[0,-1,0,0],[0,0,0,1]], dtype=np.float64)
|
| 59 |
-
tm.apply_transform(rot)
|
| 60 |
-
tmp = out + ".tmp.glb"; tm.export(tmp); os.replace(tmp, out)
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
def wait_for_vram(min_free_mib=16000, timeout_s=10800, poll_s=20):
|
| 64 |
-
"""Block until the current CUDA device has >= min_free_mib free (after releasing
|
| 65 |
-
our own transient cache), so the res-1536 sampling spike (~13.5GB) can run without
|
| 66 |
-
OOMing against co-located sibling jobs sharing the GPU. Our resident model weights
|
| 67 |
-
are NOT counted as free, so min_free just needs to cover the spike + margin."""
|
| 68 |
-
import torch, time as _t
|
| 69 |
-
t0 = _t.time()
|
| 70 |
-
while True:
|
| 71 |
-
torch.cuda.empty_cache()
|
| 72 |
-
free, total = torch.cuda.mem_get_info()
|
| 73 |
-
free_mib = free / (1024 * 1024)
|
| 74 |
-
if free_mib >= min_free_mib:
|
| 75 |
-
return
|
| 76 |
-
if _t.time() - t0 > timeout_s:
|
| 77 |
-
print(f"[vram] timeout waiting for {min_free_mib}MiB (free={free_mib:.0f}); proceeding", flush=True)
|
| 78 |
-
return
|
| 79 |
-
print(f"[vram] free={free_mib:.0f}MiB < {min_free_mib}MiB, waiting for GPU headroom...", flush=True)
|
| 80 |
-
_t.sleep(poll_s)
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
def run_one_resilient(*args, retries=6, **kw):
|
| 84 |
-
"""Run one object, retrying on CUDA OOM after clearing cache + waiting for VRAM
|
| 85 |
-
headroom (co-located sibling jobs release memory between their own objects)."""
|
| 86 |
-
import torch
|
| 87 |
-
for attempt in range(retries):
|
| 88 |
-
try:
|
| 89 |
-
wait_for_vram()
|
| 90 |
-
return run_one(*args, **kw)
|
| 91 |
-
except torch.cuda.OutOfMemoryError:
|
| 92 |
-
torch.cuda.empty_cache()
|
| 93 |
-
import time as _t; _t.sleep(15)
|
| 94 |
-
print(f"[oom] attempt {attempt+1}/{retries} hit CUDA OOM; cleared cache, retrying", flush=True)
|
| 95 |
-
# last attempt without swallowing
|
| 96 |
-
wait_for_vram(min_free_mib=18000)
|
| 97 |
-
return run_one(*args, **kw)
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
def _load_c2w_for_views(renders_dir, obj_name, view_tags):
|
| 101 |
-
"""Load GT camera-to-world (c2w_cv, 4x4) for each view from renders/<obj>_<tag>.npz."""
|
| 102 |
-
import numpy as _np
|
| 103 |
-
mats = []
|
| 104 |
-
for t in view_tags:
|
| 105 |
-
npz = os.path.join(renders_dir, f"{obj_name}_{t}.npz")
|
| 106 |
-
z = _np.load(npz)
|
| 107 |
-
mats.append(_np.asarray(z["c2w_cv"], dtype=_np.float64))
|
| 108 |
-
return mats
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
def run_one(pipeline, moge, imgs, out, seed, resolution, image_resolution, low_vram,
|
| 112 |
-
vertex_color=True, mv_mode="proper", side_azimuth=90.0, simplify_target=150000,
|
| 113 |
-
renders_dir=None, obj_name=None, view_tags=None):
|
| 114 |
-
outdir = os.path.dirname(os.path.abspath(out))
|
| 115 |
-
os.makedirs(outdir, exist_ok=True)
|
| 116 |
-
pre_imgs, tmp_paths = [], []
|
| 117 |
-
for i, p in enumerate(imgs):
|
| 118 |
-
img = Image.open(p) # RGBA -> preprocess uses alpha directly
|
| 119 |
-
pi = pipeline.preprocess_image(img)
|
| 120 |
-
pre_imgs.append(pi)
|
| 121 |
-
tp = os.path.join(outdir, f"_tmp_mv_{i}_{int(time.time()*1000)}.png")
|
| 122 |
-
pi.save(tp); tmp_paths.append(tp)
|
| 123 |
-
|
| 124 |
-
cams = []
|
| 125 |
-
for tp in tmp_paths:
|
| 126 |
-
cp = _inf.get_camera_params_wild_moge(tp, moge, device="cuda",
|
| 127 |
-
mesh_scale=1.0, extend_pixel=0,
|
| 128 |
-
image_resolution=image_resolution)
|
| 129 |
-
cams.append(cp)
|
| 130 |
-
for tp in tmp_paths:
|
| 131 |
-
try: os.remove(tp)
|
| 132 |
-
except OSError: pass
|
| 133 |
-
|
| 134 |
-
if len(pre_imgs) > 1:
|
| 135 |
-
# CORRECTED proper MV: per-view back-projection through each view's REAL
|
| 136 |
-
# camera, read per-object from GT renders (renders/<obj>_<tag>.npz c2w_cv),
|
| 137 |
-
# then simple averaging (paper Sec 3.2.3). Fixes the previous zero-elevation
|
| 138 |
-
# / hard-coded-azimuth-sign rig. 'hack' mode is REMOVED.
|
| 139 |
-
if mv_mode != "proper":
|
| 140 |
-
raise ValueError(f"mv_mode={mv_mode!r} removed; only 'proper' is supported")
|
| 141 |
-
if renders_dir is None or obj_name is None or view_tags is None:
|
| 142 |
-
raise ValueError("proper MV requires renders_dir, obj_name, view_tags")
|
| 143 |
-
c2w_list = _load_c2w_for_views(renders_dir, obj_name, view_tags)
|
| 144 |
-
from mv_proper import real_view_transforms
|
| 145 |
-
vt = real_view_transforms(c2w_list, cams[0]['distance'], device="cuda")
|
| 146 |
-
build_mv_cond_functions_proper(pipeline, pre_imgs, cams, view_transforms=vt)
|
| 147 |
-
# len==1: single view -> pipeline uses its native official get_proj_cond (no patch)
|
| 148 |
-
|
| 149 |
-
torch.manual_seed(seed)
|
| 150 |
-
pipeline_type = f"{resolution if resolution > 0 else (1024 if low_vram else 1536)}_cascade"
|
| 151 |
-
mesh_list, (shape_slat, tex_slat, res) = pipeline.run(
|
| 152 |
-
pre_imgs[0], camera_params=cams[0], seed=seed,
|
| 153 |
-
sparse_structure_sampler_params={"steps": 12, "guidance_strength": 7.5, "guidance_rescale": 0.7, "rescale_t": 5.0},
|
| 154 |
-
shape_slat_sampler_params={"steps": 12, "guidance_strength": 7.5, "guidance_rescale": 0.5, "rescale_t": 3.0},
|
| 155 |
-
tex_slat_sampler_params={"steps": 12, "guidance_strength": 1.0, "guidance_rescale": 0.0, "rescale_t": 3.0},
|
| 156 |
-
preprocess_image=False, return_latent=True, pipeline_type=pipeline_type, max_num_tokens=49152,
|
| 157 |
-
)
|
| 158 |
-
mesh = mesh_list[0]
|
| 159 |
-
if vertex_color:
|
| 160 |
-
export_vertex_colored(mesh, out, pipeline.pbr_attr_layout, simplify_target=simplify_target)
|
| 161 |
-
return
|
| 162 |
-
glb = o_voxel.postprocess.to_glb(
|
| 163 |
-
vertices=mesh.vertices, faces=mesh.faces, attr_volume=mesh.attrs,
|
| 164 |
-
coords=mesh.coords, attr_layout=pipeline.pbr_attr_layout,
|
| 165 |
-
grid_size=res, aabb=[[-0.5, -0.5, -0.5], [0.5, 0.5, 0.5]],
|
| 166 |
-
decimation_target=1000000, texture_size=4096,
|
| 167 |
-
remesh=True, remesh_band=1, remesh_project=0, use_tqdm=False,
|
| 168 |
-
)
|
| 169 |
-
rot = np.array([[-1,0,0,0],[0,0,-1,0],[0,-1,0,0],[0,0,0,1]], dtype=np.float64)
|
| 170 |
-
glb.apply_transform(rot)
|
| 171 |
-
tmp = out + ".tmp.glb"
|
| 172 |
-
glb.export(tmp, extension_webp=True)
|
| 173 |
-
os.replace(tmp, out)
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
def main():
|
| 177 |
-
ap = argparse.ArgumentParser()
|
| 178 |
-
ap.add_argument("--selection", required=True)
|
| 179 |
-
ap.add_argument("--inputs", required=True)
|
| 180 |
-
ap.add_argument("--out", required=True)
|
| 181 |
-
ap.add_argument("--views", type=int, required=True, choices=[1, 2, 4])
|
| 182 |
-
ap.add_argument("--seed", type=int, default=42)
|
| 183 |
-
ap.add_argument("--limit", type=int, default=None)
|
| 184 |
-
ap.add_argument("--resolution", type=int, default=1536)
|
| 185 |
-
ap.add_argument("--image_resolution", type=int, default=512)
|
| 186 |
-
ap.add_argument("--low_vram", action="store_true")
|
| 187 |
-
ap.add_argument("--to_glb", dest="vertex_color", action="store_false", default=True,
|
| 188 |
-
help="use slow textured to_glb export (default: fast vertex-color)")
|
| 189 |
-
ap.add_argument("--shard", type=int, default=0)
|
| 190 |
-
ap.add_argument("--nshards", type=int, default=1)
|
| 191 |
-
ap.add_argument("--reverse", action="store_true",
|
| 192 |
-
help="process objects in reverse order (convergent co-located "
|
| 193 |
-
"helper against a forward run; skip-existing + atomic write)")
|
| 194 |
-
ap.add_argument("--mv_mode", choices=["proper"], default="proper",
|
| 195 |
-
help="proper=paper-faithful per-view back-projection through REAL "
|
| 196 |
-
"per-object GT extrinsics + average ('hack' removed)")
|
| 197 |
-
ap.add_argument("--renders", default=None,
|
| 198 |
-
help="dir with GT renders/<obj>_<view>.npz (c2w_cv); "
|
| 199 |
-
"default: <inputs>/../renders")
|
| 200 |
-
ap.add_argument("--side_azimuth", type=float, default=90.0,
|
| 201 |
-
help="azimuth offset (deg) of the side view from front, about up-axis")
|
| 202 |
-
ap.add_argument("--simplify_target", type=int, default=150000,
|
| 203 |
-
help="target face count for the authors' cumesh quadric decimate "
|
| 204 |
-
"in the vertex-color export (0=off); ~150k renders/evals fast "
|
| 205 |
-
"and avoids the nvdiffrast CUDA-700 crash on the raw dual-grid mesh")
|
| 206 |
-
args = ap.parse_args()
|
| 207 |
-
|
| 208 |
-
with open(args.selection) as f:
|
| 209 |
-
sel = json.load(f)["selections"]
|
| 210 |
-
objects = [s["object"] for s in sel]
|
| 211 |
-
if args.limit:
|
| 212 |
-
objects = objects[:args.limit]
|
| 213 |
-
if args.nshards > 1:
|
| 214 |
-
objects = objects[args.shard::args.nshards]
|
| 215 |
-
if args.reverse:
|
| 216 |
-
objects = objects[::-1]
|
| 217 |
-
outdir = os.path.abspath(args.out)
|
| 218 |
-
os.makedirs(outdir, exist_ok=True)
|
| 219 |
-
renders_dir = args.renders or os.path.join(os.path.dirname(os.path.abspath(args.inputs)), "renders")
|
| 220 |
-
view_tags = {1: ["front"], 2: ["front", "side"],
|
| 221 |
-
4: ["front", "side", "back", "oside"]}[args.views]
|
| 222 |
-
|
| 223 |
-
jobs = []
|
| 224 |
-
for name in objects:
|
| 225 |
-
out = os.path.join(outdir, f"{name}.glb")
|
| 226 |
-
imgs = [os.path.join(args.inputs, f"{name}_{t}.png") for t in view_tags]
|
| 227 |
-
jobs.append((name, imgs, out))
|
| 228 |
-
todo = [j for j in jobs if not os.path.isfile(j[2])]
|
| 229 |
-
print(f"[pixal] {len(jobs)} obj, {len(jobs)-len(todo)} done, {len(todo)} to run "
|
| 230 |
-
f"| views={args.views} res={args.resolution}", flush=True)
|
| 231 |
-
|
| 232 |
-
pipeline = _inf.init_pipeline(_inf.MODEL_PATH, low_vram=args.low_vram)
|
| 233 |
-
moge = _inf.load_moge_model(device="cuda")
|
| 234 |
-
|
| 235 |
-
n_ok = n_fail = n_skip = 0
|
| 236 |
-
for i, (name, imgs, out) in enumerate(jobs, 1):
|
| 237 |
-
t0 = time.time()
|
| 238 |
-
try:
|
| 239 |
-
if os.path.isfile(out):
|
| 240 |
-
n_skip += 1
|
| 241 |
-
print(f"[{i}/{len(jobs)}] {name} SKIP", flush=True)
|
| 242 |
-
continue
|
| 243 |
-
for p in imgs:
|
| 244 |
-
if not os.path.isfile(p):
|
| 245 |
-
raise FileNotFoundError(p)
|
| 246 |
-
run_one_resilient(pipeline, moge, imgs, out, args.seed, args.resolution,
|
| 247 |
-
args.image_resolution, args.low_vram, vertex_color=args.vertex_color,
|
| 248 |
-
mv_mode=args.mv_mode, side_azimuth=args.side_azimuth,
|
| 249 |
-
simplify_target=args.simplify_target,
|
| 250 |
-
renders_dir=renders_dir, obj_name=name, view_tags=view_tags)
|
| 251 |
-
torch.cuda.empty_cache()
|
| 252 |
-
n_ok += 1
|
| 253 |
-
print(f"[{i}/{len(jobs)}] {name} OK {time.time()-t0:.1f}s -> {out}", flush=True)
|
| 254 |
-
except Exception:
|
| 255 |
-
n_fail += 1
|
| 256 |
-
traceback.print_exc()
|
| 257 |
-
print(f"[{i}/{len(jobs)}] {name} FAIL {time.time()-t0:.1f}s", flush=True)
|
| 258 |
-
finally:
|
| 259 |
-
try: torch.cuda.empty_cache()
|
| 260 |
-
except Exception: pass
|
| 261 |
-
print(f"BATCH DONE ok={n_ok} fail={n_fail} skip={n_skip} total={len(jobs)}", flush=True)
|
| 262 |
-
return 1 if n_fail else 0
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
if __name__ == "__main__":
|
| 266 |
-
sys.exit(main())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
forgebench/code/baselines/pixal3d_ours/batch_pixal3d_mv.py
CHANGED
|
@@ -32,8 +32,8 @@ os.environ['OPENCV_IO_ENABLE_OPENEXR'] = '1'
|
|
| 32 |
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
|
| 33 |
os.environ.setdefault("ATTN_BACKEND", "sdpa")
|
| 34 |
|
| 35 |
-
UPSTREAM = "/lp-dev/jonghoon/mv-mesh/.debug_pixal3d_verify/pixal3d_upstream"
|
| 36 |
-
OURREPO = "/lp-dev/jonghoon/mv-mesh/pixal3d"
|
| 37 |
os.chdir(UPSTREAM)
|
| 38 |
sys.path.insert(0, UPSTREAM)
|
| 39 |
sys.path.insert(1, OURREPO) # mv_common only
|
|
|
|
| 32 |
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
|
| 33 |
os.environ.setdefault("ATTN_BACKEND", "sdpa")
|
| 34 |
|
| 35 |
+
UPSTREAM = os.environ.get("PIXAL3D_UPSTREAM", "/lp-dev/jonghoon/mv-mesh/.debug_pixal3d_verify/pixal3d_upstream") # TencentARC/Pixal3D @ f7cf384 (official MV)
|
| 36 |
+
OURREPO = os.environ.get("PIXAL3D_OURS", "/lp-dev/jonghoon/mv-mesh/pixal3d") # dir holding mv_common.py
|
| 37 |
os.chdir(UPSTREAM)
|
| 38 |
sys.path.insert(0, UPSTREAM)
|
| 39 |
sys.path.insert(1, OURREPO) # mv_common only
|
forgebench/code/baselines/pixal3d_ours/inference_mv.py
DELETED
|
@@ -1,178 +0,0 @@
|
|
| 1 |
-
"""
|
| 2 |
-
EXPERIMENTAL multi-view inference for Pixal3D.
|
| 3 |
-
|
| 4 |
-
IMPORTANT / HONEST CAVEAT:
|
| 5 |
-
Pixal3D's released `proj` pipeline + weights are architecturally SINGLE-IMAGE.
|
| 6 |
-
There is NO native multi-view inference. This script constructs a multi-view
|
| 7 |
-
variant by the ONLY mechanism the code allows: it runs the DinoV3ProjFeatureExtractor
|
| 8 |
-
on V views (each with its own MoGe-estimated camera), then MEAN-POOLS the projected
|
| 9 |
-
grid features (and global tokens) across views before the sparse coords indexing.
|
| 10 |
-
|
| 11 |
-
Two known limitations (do not report this as "true" multi-view):
|
| 12 |
-
1. ProjGrid.forward asserts transform_matrix is None -> every view is projected
|
| 13 |
-
through the SAME canonical front-view camera. Non-frontal views (back/top/45deg)
|
| 14 |
-
therefore paint their features onto geometrically wrong grid cells.
|
| 15 |
-
2. The DiT was trained on ONE view per object, so averaged multi-view features are
|
| 16 |
-
out-of-distribution; there is no learned view-fusion to exploit.
|
| 17 |
-
|
| 18 |
-
Usage:
|
| 19 |
-
python inference_mv.py --images v1.jpg v2.jpg v3.jpg --output out.glb --seed 42
|
| 20 |
-
"""
|
| 21 |
-
import os, argparse, math, time, types
|
| 22 |
-
import torch, numpy as np
|
| 23 |
-
from PIL import Image
|
| 24 |
-
|
| 25 |
-
os.environ['OPENCV_IO_ENABLE_OPENEXR'] = '1'
|
| 26 |
-
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
|
| 27 |
-
os.environ.setdefault("ATTN_BACKEND", "sdpa")
|
| 28 |
-
os.environ["FLEX_GEMM_AUTOTUNE_CACHE_PATH"] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'autotune_cache.json')
|
| 29 |
-
|
| 30 |
-
import mv_common
|
| 31 |
-
mv_common.patch_rembg_lazy()
|
| 32 |
-
|
| 33 |
-
from pixal3d.pipelines import Pixal3DImageTo3DPipeline
|
| 34 |
-
from pixal3d.modules.sparse import SparseTensor
|
| 35 |
-
import o_voxel
|
| 36 |
-
import inference as _inf # reuse init_pipeline / camera helpers
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
def build_mv_cond_functions(pipeline, images, cams):
|
| 40 |
-
"""Monkeypatch get_proj_cond_ss / get_proj_cond_shape to use V views + mean-pool.
|
| 41 |
-
|
| 42 |
-
images: list of V preprocessed PIL images.
|
| 43 |
-
cams: list of V dicts {camera_angle_x, distance, mesh_scale}.
|
| 44 |
-
"""
|
| 45 |
-
device_getter = lambda: pipeline.device
|
| 46 |
-
|
| 47 |
-
def _cam_tensors(dev):
|
| 48 |
-
cax = torch.tensor([c['camera_angle_x'] for c in cams], device=dev)
|
| 49 |
-
dist = torch.tensor([c['distance'] for c in cams], device=dev)
|
| 50 |
-
scal = torch.tensor([c.get('mesh_scale', 1.0) for c in cams], device=dev)
|
| 51 |
-
return cax, dist, scal
|
| 52 |
-
|
| 53 |
-
@torch.no_grad()
|
| 54 |
-
def get_proj_cond_ss(self, image, camera_angle_x=None, distance=None, mesh_scale=None):
|
| 55 |
-
dev = self.device
|
| 56 |
-
m = self.image_cond_model_ss
|
| 57 |
-
if self.low_vram: m.to(dev)
|
| 58 |
-
cax, dist, scal = _cam_tensors(dev)
|
| 59 |
-
z_global, z_proj = m(images, camera_angle_x=cax, distance=dist, mesh_scale=scal) # [V,..],[V,R3,C]
|
| 60 |
-
if self.low_vram: m.cpu()
|
| 61 |
-
z_global = z_global.mean(dim=0, keepdim=True) # [1,tok,C]
|
| 62 |
-
z_proj = z_proj.mean(dim=0, keepdim=True) # [1,R3,C]
|
| 63 |
-
return {
|
| 64 |
-
'cond': {'global': z_global, 'proj': z_proj},
|
| 65 |
-
'neg_cond': {'global': torch.zeros_like(z_global), 'proj': torch.zeros_like(z_proj)},
|
| 66 |
-
}
|
| 67 |
-
|
| 68 |
-
@torch.no_grad()
|
| 69 |
-
def get_proj_cond_shape(self, image_cond_model, image, coords,
|
| 70 |
-
camera_angle_x=None, distance=None, mesh_scale=None,
|
| 71 |
-
grid_resolution_override=None):
|
| 72 |
-
dev = self.device
|
| 73 |
-
if self.low_vram: image_cond_model.to(dev)
|
| 74 |
-
orig_grid_res = image_cond_model.grid_resolution
|
| 75 |
-
if grid_resolution_override is not None and grid_resolution_override != orig_grid_res:
|
| 76 |
-
image_cond_model.grid_resolution = grid_resolution_override
|
| 77 |
-
image_cond_model.proj_grid = image_cond_model.proj_grid.__class__(
|
| 78 |
-
grid_resolution=grid_resolution_override,
|
| 79 |
-
image_resolution=image_cond_model.proj_grid.image_resolution,
|
| 80 |
-
).to(dev)
|
| 81 |
-
cax, dist, scal = _cam_tensors(dev)
|
| 82 |
-
z_global, z_proj = image_cond_model(images, camera_angle_x=cax, distance=dist, mesh_scale=scal)
|
| 83 |
-
# mean-pool over views
|
| 84 |
-
z_global = z_global.mean(dim=0, keepdim=True) # [1,tok,C]
|
| 85 |
-
z_proj = z_proj.mean(dim=0, keepdim=True) # [1,R3,C]
|
| 86 |
-
grid_res = image_cond_model.grid_resolution
|
| 87 |
-
z_proj_grid = z_proj.reshape(1, grid_res, grid_res, grid_res, -1)
|
| 88 |
-
bi = coords[:, 0].long(); xc = coords[:, 1].long(); yc = coords[:, 2].long(); zc = coords[:, 3].long()
|
| 89 |
-
z_proj_sparse = z_proj_grid[bi, xc, yc, zc]
|
| 90 |
-
z_proj_st = SparseTensor(feats=z_proj_sparse, coords=coords)
|
| 91 |
-
if grid_resolution_override is not None and grid_resolution_override != orig_grid_res:
|
| 92 |
-
image_cond_model.grid_resolution = orig_grid_res
|
| 93 |
-
image_cond_model.proj_grid = image_cond_model.proj_grid.__class__(
|
| 94 |
-
grid_resolution=orig_grid_res,
|
| 95 |
-
image_resolution=image_cond_model.proj_grid.image_resolution,
|
| 96 |
-
).to(dev)
|
| 97 |
-
if self.low_vram: image_cond_model.cpu()
|
| 98 |
-
return {
|
| 99 |
-
'cond': {'global': z_global, 'proj': z_proj_st},
|
| 100 |
-
'neg_cond': {'global': torch.zeros_like(z_global),
|
| 101 |
-
'proj': SparseTensor(feats=torch.zeros_like(z_proj_sparse), coords=coords)},
|
| 102 |
-
}
|
| 103 |
-
|
| 104 |
-
pipeline.get_proj_cond_ss = types.MethodType(get_proj_cond_ss, pipeline)
|
| 105 |
-
pipeline.get_proj_cond_shape = types.MethodType(get_proj_cond_shape, pipeline)
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
def main():
|
| 109 |
-
ap = argparse.ArgumentParser()
|
| 110 |
-
ap.add_argument("--images", nargs="+", required=True)
|
| 111 |
-
ap.add_argument("--masks", nargs="+", default=None, help="optional per-view masks (aligned with --images)")
|
| 112 |
-
ap.add_argument("--output", required=True)
|
| 113 |
-
ap.add_argument("--seed", type=int, default=42)
|
| 114 |
-
ap.add_argument("--low_vram", action="store_true")
|
| 115 |
-
ap.add_argument("--resolution", type=int, default=-1)
|
| 116 |
-
ap.add_argument("--image_resolution", type=int, default=512)
|
| 117 |
-
args = ap.parse_args()
|
| 118 |
-
|
| 119 |
-
pipeline = _inf.init_pipeline(_inf.MODEL_PATH, low_vram=args.low_vram)
|
| 120 |
-
|
| 121 |
-
# preprocess each view (bg removal + crop) and estimate per-view camera via MoGe
|
| 122 |
-
pre_imgs, tmp_paths = [], []
|
| 123 |
-
outdir = os.path.dirname(os.path.abspath(args.output))
|
| 124 |
-
os.makedirs(outdir, exist_ok=True)
|
| 125 |
-
masks = args.masks if args.masks else [None] * len(args.images)
|
| 126 |
-
assert len(masks) == len(args.images), "masks must align with images"
|
| 127 |
-
for i, p in enumerate(args.images):
|
| 128 |
-
img = mv_common.load_rgba(p, masks[i])
|
| 129 |
-
pi = pipeline.preprocess_image(img)
|
| 130 |
-
pre_imgs.append(pi)
|
| 131 |
-
tp = os.path.join(outdir, f"_tmp_mv_{i}_{int(time.time()*1000)}.png")
|
| 132 |
-
pi.save(tp); tmp_paths.append(tp)
|
| 133 |
-
|
| 134 |
-
print("[MoGe-2] estimating per-view cameras...")
|
| 135 |
-
moge = _inf.load_moge_model(device="cuda")
|
| 136 |
-
cams = []
|
| 137 |
-
for tp in tmp_paths:
|
| 138 |
-
cp = _inf.get_camera_params_wild_moge(tp, moge, device="cuda",
|
| 139 |
-
mesh_scale=1.0, extend_pixel=0,
|
| 140 |
-
image_resolution=args.image_resolution)
|
| 141 |
-
cams.append(cp)
|
| 142 |
-
print(f" view cam: angle_x={cp['camera_angle_x']:.4f} dist={cp['distance']:.4f}")
|
| 143 |
-
moge.cpu(); del moge; torch.cuda.empty_cache()
|
| 144 |
-
for tp in tmp_paths:
|
| 145 |
-
try: os.remove(tp)
|
| 146 |
-
except OSError: pass
|
| 147 |
-
|
| 148 |
-
build_mv_cond_functions(pipeline, pre_imgs, cams)
|
| 149 |
-
|
| 150 |
-
seed = args.seed
|
| 151 |
-
torch.manual_seed(seed)
|
| 152 |
-
pipeline_type = f"{args.resolution if args.resolution > 0 else (1024 if args.low_vram else 1536)}_cascade"
|
| 153 |
-
print(f"[MV-Inference] {len(pre_imgs)} views, pipeline_type={pipeline_type}")
|
| 154 |
-
# camera_params passed to run() are ignored by patched cond fns; supply view-0's as placeholder
|
| 155 |
-
mesh_list, (shape_slat, tex_slat, res) = pipeline.run(
|
| 156 |
-
pre_imgs[0], camera_params=cams[0], seed=seed,
|
| 157 |
-
sparse_structure_sampler_params={"steps": 12, "guidance_strength": 7.5, "guidance_rescale": 0.7, "rescale_t": 5.0},
|
| 158 |
-
shape_slat_sampler_params={"steps": 12, "guidance_strength": 7.5, "guidance_rescale": 0.5, "rescale_t": 3.0},
|
| 159 |
-
tex_slat_sampler_params={"steps": 12, "guidance_strength": 1.0, "guidance_rescale": 0.0, "rescale_t": 3.0},
|
| 160 |
-
preprocess_image=False, return_latent=True, pipeline_type=pipeline_type, max_num_tokens=49152,
|
| 161 |
-
)
|
| 162 |
-
mesh = mesh_list[0]
|
| 163 |
-
print("[MV-Inference] extracting GLB...")
|
| 164 |
-
glb = o_voxel.postprocess.to_glb(
|
| 165 |
-
vertices=mesh.vertices, faces=mesh.faces, attr_volume=mesh.attrs,
|
| 166 |
-
coords=mesh.coords, attr_layout=pipeline.pbr_attr_layout,
|
| 167 |
-
grid_size=res, aabb=[[-0.5, -0.5, -0.5], [0.5, 0.5, 0.5]],
|
| 168 |
-
decimation_target=1000000, texture_size=4096,
|
| 169 |
-
remesh=True, remesh_band=1, remesh_project=0, use_tqdm=True,
|
| 170 |
-
)
|
| 171 |
-
rot = np.array([[-1,0,0,0],[0,0,-1,0],[0,-1,0,0],[0,0,0,1]], dtype=np.float64)
|
| 172 |
-
glb.apply_transform(rot)
|
| 173 |
-
glb.export(args.output, extension_webp=True)
|
| 174 |
-
print(f"[Done] GLB saved to: {args.output}")
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
if __name__ == "__main__":
|
| 178 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|