Spaces:
Running on Zero
fix: rebuild for ZeroGPU torch whitelist (2.8-2.11)
Browse filesZeroGPU rejects any torch pin outside 2.8.0/2.9.1/2.10.0/2.11.0, and there
are no kaolin/pytorch3d/gsplat wheels for those versions. Instead of pinning
old torch:
- drop torch from requirements — use the ZeroGPU-provided build
- stub kaolin/pytorch3d/flash_attn (adopted from jboth/sam3d-objects-fixed);
only their pure-python surface is used since texture baking, mesh
postprocess and layout postprocess are disabled in Inference.__call__
- install spconv-cu124 (torch-independent), gsplat (PyPI JIT mode, kernels
never compiled), sam2/MoGe/utils3d at runtime against the live torch
- force sdpa attention (patch inference_pipeline.py H200 flash_attn autodetect)
- checkpoints via symlink to snapshot_download, config = pipeline.yaml
- add Diagnose tab for module/GPU health checks
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
- README.md +2 -2
- app.py +237 -170
- flash_attn_stub/flash_attn/__init__.py +1 -0
- kaolin_stub/kaolin/__init__.py +1 -0
- kaolin_stub/kaolin/ops/__init__.py +1 -0
- kaolin_stub/kaolin/ops/mesh.py +1 -0
- kaolin_stub/kaolin/render/__init__.py +1 -0
- kaolin_stub/kaolin/render/camera.py +1 -0
- kaolin_stub/kaolin/utils/__init__.py +1 -0
- kaolin_stub/kaolin/utils/testing.py +1 -0
- kaolin_stub/kaolin/visualize/__init__.py +1 -0
- pytorch3d_stub/pytorch3d/__init__.py +1 -0
- pytorch3d_stub/pytorch3d/renderer/__init__.py +1 -0
- pytorch3d_stub/pytorch3d/renderer/camera_utils.py +1 -0
- pytorch3d_stub/pytorch3d/renderer/cameras.py +1 -0
- pytorch3d_stub/pytorch3d/renderer/mesh/__init__.py +1 -0
- pytorch3d_stub/pytorch3d/renderer/mesh/textures.py +1 -0
- pytorch3d_stub/pytorch3d/structures/__init__.py +1 -0
- pytorch3d_stub/pytorch3d/transforms/__init__.py +1 -0
- pytorch3d_stub/pytorch3d/vis/__init__.py +1 -0
- pytorch3d_stub/pytorch3d/vis/plotly_vis.py +1 -0
- requirements.txt +48 -23
|
@@ -4,8 +4,8 @@ emoji: 📦
|
|
| 4 |
colorFrom: purple
|
| 5 |
colorTo: blue
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version: 6.
|
| 8 |
-
python_version: "3.
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
| 11 |
license: other
|
|
|
|
| 4 |
colorFrom: purple
|
| 5 |
colorTo: blue
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 6.13.0
|
| 8 |
+
python_version: "3.12"
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
| 11 |
license: other
|
|
@@ -2,101 +2,209 @@
|
|
| 2 |
SAM 3D Objects MCP Server
|
| 3 |
Image → 3D Object (GLB)
|
| 4 |
|
| 5 |
-
|
|
|
|
|
|
|
|
|
|
| 6 |
"""
|
| 7 |
import os
|
| 8 |
import sys
|
| 9 |
import subprocess
|
| 10 |
import tempfile
|
| 11 |
-
import uuid
|
| 12 |
from pathlib import Path
|
| 13 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
import gradio as gr
|
| 15 |
import numpy as np
|
| 16 |
-
import spaces
|
| 17 |
from huggingface_hub import snapshot_download, login
|
| 18 |
-
from PIL import Image
|
| 19 |
|
| 20 |
-
# Login with HF_TOKEN if available
|
| 21 |
if os.environ.get("HF_TOKEN"):
|
| 22 |
-
login(token=os.environ
|
| 23 |
-
|
| 24 |
-
#
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
if not SAM3D_PATH.exists():
|
| 33 |
-
print("Cloning sam-3d-objects
|
| 34 |
-
subprocess.run([
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
sys.path.insert(0, str(SAM3D_PATH))
|
| 42 |
sys.path.insert(0, str(SAM3D_PATH / "notebook"))
|
| 43 |
|
| 44 |
-
#
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
import torch
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 94 |
|
| 95 |
|
| 96 |
-
@spaces.GPU(duration=
|
| 97 |
def reconstruct_objects(image: np.ndarray):
|
| 98 |
"""
|
| 99 |
-
Automatically detect
|
|
|
|
| 100 |
|
| 101 |
Args:
|
| 102 |
image: Input RGB image
|
|
@@ -106,94 +214,56 @@ def reconstruct_objects(image: np.ndarray):
|
|
| 106 |
"""
|
| 107 |
if image is None:
|
| 108 |
return None, None, "❌ No image provided"
|
| 109 |
-
|
| 110 |
try:
|
| 111 |
import torch
|
| 112 |
-
import
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
if isinstance(image, np.ndarray)
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
masks = generator.generate(image)
|
| 129 |
-
|
| 130 |
-
if not masks or len(masks) == 0:
|
| 131 |
-
return None, image, "⚠️ No objects detected"
|
| 132 |
-
|
| 133 |
-
# Sort by area, take largest object
|
| 134 |
-
masks = sorted(masks, key=lambda x: x['area'], reverse=True)
|
| 135 |
-
best_mask = masks[0]['segmentation']
|
| 136 |
-
|
| 137 |
-
# Create preview with mask overlay
|
| 138 |
-
preview = image.copy()
|
| 139 |
preview[best_mask] = (preview[best_mask] * 0.5 + np.array([0, 255, 0]) * 0.5).astype(np.uint8)
|
|
|
|
| 140 |
|
| 141 |
-
#
|
| 142 |
-
|
|
|
|
| 143 |
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
|
|
|
|
|
|
|
| 148 |
if result is None:
|
| 149 |
-
return None, preview, "⚠️
|
| 150 |
-
|
| 151 |
-
# Export as GLB
|
| 152 |
-
output_dir = tempfile.mkdtemp()
|
| 153 |
-
glb_path = f"{output_dir}/object_{uuid.uuid4().hex[:8]}.glb"
|
| 154 |
-
|
| 155 |
-
# Extract point cloud from result and convert to mesh
|
| 156 |
-
if hasattr(result, 'save_ply'):
|
| 157 |
-
# Save temp PLY then convert
|
| 158 |
-
ply_path = f"{output_dir}/temp.ply"
|
| 159 |
-
result.save_ply(ply_path)
|
| 160 |
-
# Load and convert to mesh using Open3D
|
| 161 |
-
import open3d as o3d
|
| 162 |
-
pcd = o3d.io.read_point_cloud(ply_path)
|
| 163 |
-
# Estimate normals and create mesh via Poisson reconstruction
|
| 164 |
-
pcd.estimate_normals()
|
| 165 |
-
mesh, _ = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(pcd, depth=8)
|
| 166 |
-
o3d.io.write_triangle_mesh(glb_path, mesh)
|
| 167 |
-
elif 'gaussians' in result:
|
| 168 |
-
ply_path = f"{output_dir}/temp.ply"
|
| 169 |
-
result['gaussians'].save_ply(ply_path)
|
| 170 |
-
import open3d as o3d
|
| 171 |
-
pcd = o3d.io.read_point_cloud(ply_path)
|
| 172 |
-
pcd.estimate_normals()
|
| 173 |
-
mesh, _ = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(pcd, depth=8)
|
| 174 |
-
o3d.io.write_triangle_mesh(glb_path, mesh)
|
| 175 |
-
else:
|
| 176 |
-
# Try to extract vertices
|
| 177 |
-
vertices = result.get('xyz', result.get('points', None))
|
| 178 |
-
if vertices is not None:
|
| 179 |
-
if torch.is_tensor(vertices):
|
| 180 |
-
vertices = vertices.cpu().numpy()
|
| 181 |
-
# Create mesh from points
|
| 182 |
-
import open3d as o3d
|
| 183 |
-
pcd = o3d.geometry.PointCloud()
|
| 184 |
-
pcd.points = o3d.utility.Vector3dVector(vertices)
|
| 185 |
-
pcd.estimate_normals()
|
| 186 |
-
mesh, _ = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(pcd, depth=8)
|
| 187 |
-
o3d.io.write_triangle_mesh(glb_path, mesh)
|
| 188 |
-
else:
|
| 189 |
-
return None, preview, "⚠️ Could not extract 3D data"
|
| 190 |
-
|
| 191 |
-
return glb_path, preview, f"✓ Detected {len(masks)} objects, reconstructed largest"
|
| 192 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 193 |
except Exception as e:
|
| 194 |
import traceback
|
| 195 |
-
traceback.
|
| 196 |
-
|
|
|
|
| 197 |
|
| 198 |
|
| 199 |
# Gradio Interface
|
|
@@ -205,27 +275,24 @@ with gr.Blocks(title="SAM 3D Objects MCP") as demo:
|
|
| 205 |
Automatically detects objects and reconstructs the largest one in 3D.
|
| 206 |
""")
|
| 207 |
|
| 208 |
-
with gr.
|
| 209 |
-
with gr.
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
with gr.Row():
|
| 218 |
-
with gr.Column():
|
| 219 |
output_model = gr.Model3D(label="3D Preview")
|
| 220 |
-
with gr.Column():
|
| 221 |
output_file = gr.File(label="Download GLB")
|
|
|
|
|
|
|
| 222 |
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
outputs=[
|
| 227 |
-
)
|
| 228 |
-
output_model.change(lambda x: x, inputs=[output_model], outputs=[output_file])
|
| 229 |
|
| 230 |
gr.Markdown("""
|
| 231 |
---
|
|
|
|
| 2 |
SAM 3D Objects MCP Server
|
| 3 |
Image → 3D Object (GLB)
|
| 4 |
|
| 5 |
+
SAM2 auto-detection + SAM 3D Objects reconstruction on ZeroGPU.
|
| 6 |
+
kaolin/pytorch3d/flash_attn are stubbed (only their pure-python surface is
|
| 7 |
+
used since texture baking and mesh postprocessing are disabled); spconv and
|
| 8 |
+
gsplat are installed at runtime against the ZeroGPU-provided torch.
|
| 9 |
"""
|
| 10 |
import os
|
| 11 |
import sys
|
| 12 |
import subprocess
|
| 13 |
import tempfile
|
|
|
|
| 14 |
from pathlib import Path
|
| 15 |
|
| 16 |
+
os.environ.setdefault("CUDA_HOME", "/usr/local/cuda")
|
| 17 |
+
os.environ.setdefault("CONDA_PREFIX", "/usr/local")
|
| 18 |
+
os.environ["LIDRA_SKIP_INIT"] = "true"
|
| 19 |
+
os.environ["ATTN_BACKEND"] = "sdpa"
|
| 20 |
+
os.environ["SPARSE_ATTN_BACKEND"] = "sdpa"
|
| 21 |
+
os.environ["SPARSE_BACKEND"] = "spconv"
|
| 22 |
+
|
| 23 |
+
# MUST import spaces before torch
|
| 24 |
+
import spaces
|
| 25 |
import gradio as gr
|
| 26 |
import numpy as np
|
|
|
|
| 27 |
from huggingface_hub import snapshot_download, login
|
|
|
|
| 28 |
|
|
|
|
| 29 |
if os.environ.get("HF_TOKEN"):
|
| 30 |
+
login(token=os.environ["HF_TOKEN"])
|
| 31 |
+
|
| 32 |
+
# --- Stubs (must be on sys.path before sam3d imports) ---
|
| 33 |
+
APP_ROOT = Path(__file__).parent
|
| 34 |
+
for stub in ["kaolin_stub", "pytorch3d_stub", "flash_attn_stub"]:
|
| 35 |
+
stub_path = APP_ROOT / stub
|
| 36 |
+
if stub_path.exists():
|
| 37 |
+
sys.path.insert(0, str(stub_path))
|
| 38 |
+
print(f"Stub added: {stub}")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
# --- Runtime pip installs (need torch present, so not in requirements.txt) ---
|
| 42 |
+
def _pip(*a):
|
| 43 |
+
r = subprocess.run(
|
| 44 |
+
[sys.executable, "-m", "pip", "install", "--no-cache-dir"] + list(a),
|
| 45 |
+
capture_output=True, text=True, timeout=1200,
|
| 46 |
+
)
|
| 47 |
+
tag = a[-1][:60]
|
| 48 |
+
print(f" pip {'OK' if r.returncode == 0 else 'FAIL'}: {tag}")
|
| 49 |
+
if r.returncode != 0:
|
| 50 |
+
print(f" {r.stderr[-300:]}")
|
| 51 |
+
return r.returncode == 0
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
print("=== Runtime installs ===")
|
| 55 |
+
_pip("open3d>=0.18.0")
|
| 56 |
+
_pip("--no-deps", "git+https://github.com/EasternJournalist/utils3d.git")
|
| 57 |
+
_pip("iopath")
|
| 58 |
+
_pip("--no-deps", "sam2>=1.1.0")
|
| 59 |
+
_pip("--no-deps", "git+https://github.com/microsoft/MoGe.git@a8c37341bc0325ca99b9d57981cc3bb2bd3e255b")
|
| 60 |
+
# gsplat: no prebuilt wheels beyond pt24 — PyPI sdist installs in JIT mode;
|
| 61 |
+
# kernels never compile here because rendering paths are disabled.
|
| 62 |
+
_pip("--no-deps", "gsplat")
|
| 63 |
+
# spconv cu124 wheel is forward-compatible with the cu128 runtime
|
| 64 |
+
_pip("spconv-cu124==2.3.8")
|
| 65 |
+
|
| 66 |
+
# --- Clone sam-3d-objects ---
|
| 67 |
+
SAM3D_PATH = APP_ROOT / "sam-3d-objects"
|
| 68 |
if not SAM3D_PATH.exists():
|
| 69 |
+
print("Cloning sam-3d-objects...")
|
| 70 |
+
subprocess.run(["git", "clone", "--depth", "1",
|
| 71 |
+
"https://github.com/facebookresearch/sam-3d-objects.git",
|
| 72 |
+
str(SAM3D_PATH)], check=True)
|
| 73 |
+
|
| 74 |
+
subprocess.run([sys.executable, "-m", "pip", "install", "-e", str(SAM3D_PATH), "--no-deps"],
|
| 75 |
+
capture_output=True, text=True)
|
| 76 |
+
|
| 77 |
+
hydra_patch = SAM3D_PATH / "patching" / "hydra"
|
| 78 |
+
if hydra_patch.exists():
|
| 79 |
+
subprocess.run(["bash", str(hydra_patch)], capture_output=True, cwd=str(SAM3D_PATH))
|
| 80 |
+
|
| 81 |
+
# Patch: inference_pipeline.py forces flash_attn on A100/H100/H200,
|
| 82 |
+
# but only the stub is installed — keep sdpa.
|
| 83 |
+
ip_file = SAM3D_PATH / "sam3d_objects" / "pipeline" / "inference_pipeline.py"
|
| 84 |
+
if ip_file.exists():
|
| 85 |
+
src = ip_file.read_text()
|
| 86 |
+
old = ('if "A100" in gpu_name or "H100" in gpu_name or "H200" in gpu_name:\n'
|
| 87 |
+
' # logger.info("Use flash_attn")\n'
|
| 88 |
+
' os.environ["ATTN_BACKEND"] = "flash_attn"\n'
|
| 89 |
+
' os.environ["SPARSE_ATTN_BACKEND"] = "flash_attn"')
|
| 90 |
+
if old in src:
|
| 91 |
+
ip_file.write_text(src.replace(
|
| 92 |
+
old,
|
| 93 |
+
'# PATCHED: flash_attn not available on ZeroGPU, keep sdpa\n'
|
| 94 |
+
' os.environ.setdefault("ATTN_BACKEND", "sdpa")\n'
|
| 95 |
+
' os.environ.setdefault("SPARSE_ATTN_BACKEND", "sdpa")'
|
| 96 |
+
))
|
| 97 |
+
print("PATCHED: inference_pipeline.py — forced sdpa backend")
|
| 98 |
+
else:
|
| 99 |
+
print("INFO: inference_pipeline.py patch marker not found (upstream changed?)")
|
| 100 |
+
|
| 101 |
sys.path.insert(0, str(SAM3D_PATH))
|
| 102 |
sys.path.insert(0, str(SAM3D_PATH / "notebook"))
|
| 103 |
|
| 104 |
+
# --- Patch: utils3d from git lacks depth_edge/normals_edge (needed by layout code) ---
|
| 105 |
+
try:
|
| 106 |
+
import utils3d.numpy as _u3d_np
|
| 107 |
+
if not hasattr(_u3d_np, "depth_edge"):
|
| 108 |
+
def _depth_edge(depth, rtol=0.03, mask=None):
|
| 109 |
+
from scipy.ndimage import sobel
|
| 110 |
+
d = np.where(mask, depth, 0.0) if mask is not None else depth.copy()
|
| 111 |
+
grad = np.sqrt(sobel(d, axis=1) ** 2 + sobel(d, axis=0) ** 2)
|
| 112 |
+
denom = np.abs(d)
|
| 113 |
+
denom[denom < 1e-6] = 1e-6
|
| 114 |
+
edge = (grad / denom) > rtol
|
| 115 |
+
return edge & mask if mask is not None else edge
|
| 116 |
+
_u3d_np.depth_edge = _depth_edge
|
| 117 |
+
|
| 118 |
+
def _normals_edge(normals, tol=0.1, mask=None):
|
| 119 |
+
from scipy.ndimage import sobel
|
| 120 |
+
edges = np.zeros(normals.shape[:2], dtype=bool)
|
| 121 |
+
for c in range(normals.shape[-1]):
|
| 122 |
+
ch = np.where(mask, normals[..., c], 0.0) if mask is not None else normals[..., c]
|
| 123 |
+
edges |= np.sqrt(sobel(ch, axis=1) ** 2 + sobel(ch, axis=0) ** 2) > tol
|
| 124 |
+
return edges & mask if mask is not None else edges
|
| 125 |
+
_u3d_np.normals_edge = _normals_edge
|
| 126 |
+
print("Injected depth_edge + normals_edge into utils3d.numpy")
|
| 127 |
+
except Exception as e:
|
| 128 |
+
print(f"utils3d patch skipped: {e}")
|
| 129 |
+
|
| 130 |
+
# --- Pre-download checkpoints ---
|
| 131 |
+
print("Downloading SAM3D checkpoints...")
|
| 132 |
+
CKPT_DIR = snapshot_download(repo_id="facebook/sam-3d-objects",
|
| 133 |
+
token=os.environ.get("HF_TOKEN"))
|
| 134 |
+
hf_ckpt = Path(CKPT_DIR) / "checkpoints"
|
| 135 |
+
local_ckpt = SAM3D_PATH / "checkpoints" / "hf"
|
| 136 |
+
if hf_ckpt.exists() and not local_ckpt.exists():
|
| 137 |
+
local_ckpt.parent.mkdir(parents=True, exist_ok=True)
|
| 138 |
+
local_ckpt.symlink_to(hf_ckpt)
|
| 139 |
+
CONFIG_PATH = str(local_ckpt / "pipeline.yaml")
|
| 140 |
+
print(f"Config exists: {Path(CONFIG_PATH).exists()}")
|
| 141 |
+
print("=== Startup complete ===")
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def _gaussians_to_glb(result, out_dir):
|
| 145 |
+
"""Extract gaussians from pipeline result and Poisson-mesh them to GLB."""
|
| 146 |
import torch
|
| 147 |
+
gs = None
|
| 148 |
+
if hasattr(result, "save_ply"):
|
| 149 |
+
gs = result
|
| 150 |
+
elif isinstance(result, dict):
|
| 151 |
+
for k in ("gs", "gaussian", "gaussians", "scene"):
|
| 152 |
+
v = result.get(k)
|
| 153 |
+
if v is not None:
|
| 154 |
+
gs = v[0] if isinstance(v, (list, tuple)) else v
|
| 155 |
+
break
|
| 156 |
+
if gs is None:
|
| 157 |
+
return None
|
| 158 |
+
|
| 159 |
+
import open3d as o3d
|
| 160 |
+
glb = f"{out_dir}/object.glb"
|
| 161 |
+
if hasattr(gs, "save_ply"):
|
| 162 |
+
ply = f"{out_dir}/temp.ply"
|
| 163 |
+
gs.save_ply(ply)
|
| 164 |
+
pcd = o3d.io.read_point_cloud(ply)
|
| 165 |
+
elif hasattr(gs, "_xyz"):
|
| 166 |
+
pcd = o3d.geometry.PointCloud()
|
| 167 |
+
pcd.points = o3d.utility.Vector3dVector(gs._xyz.detach().cpu().numpy())
|
| 168 |
+
else:
|
| 169 |
+
return None
|
| 170 |
+
pcd.estimate_normals()
|
| 171 |
+
mesh, _ = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(pcd, depth=8)
|
| 172 |
+
o3d.io.write_triangle_mesh(glb, mesh)
|
| 173 |
+
return glb
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
@spaces.GPU(duration=60)
|
| 177 |
+
def diagnose():
|
| 178 |
+
"""Check GPU and importability of all critical modules."""
|
| 179 |
+
import torch
|
| 180 |
+
lines = [f"torch={torch.__version__}", f"cuda={torch.cuda.is_available()}"]
|
| 181 |
+
if torch.cuda.is_available():
|
| 182 |
+
lines.append(f"gpu={torch.cuda.get_device_name()}")
|
| 183 |
+
for mod in ["kaolin", "pytorch3d", "spconv", "gsplat", "utils3d", "open3d", "moge"]:
|
| 184 |
+
try:
|
| 185 |
+
m = __import__(mod)
|
| 186 |
+
lines.append(f"{mod}: OK ({getattr(m, '__version__', '-')})")
|
| 187 |
+
except Exception as e:
|
| 188 |
+
lines.append(f"{mod}: FAIL - {e}")
|
| 189 |
+
try:
|
| 190 |
+
from sam2.automatic_mask_generator import SAM2AutomaticMaskGenerator # noqa: F401
|
| 191 |
+
lines.append("sam2: OK")
|
| 192 |
+
except Exception as e:
|
| 193 |
+
lines.append(f"sam2: FAIL - {e}")
|
| 194 |
+
try:
|
| 195 |
+
from inference import Inference # noqa: F401
|
| 196 |
+
lines.append("SAM3D Inference: importable")
|
| 197 |
+
except Exception as e:
|
| 198 |
+
lines.append(f"SAM3D Inference: FAIL - {e}")
|
| 199 |
+
lines.append(f"config: {Path(CONFIG_PATH).exists()}")
|
| 200 |
+
return "\n".join(lines)
|
| 201 |
|
| 202 |
|
| 203 |
+
@spaces.GPU(duration=300)
|
| 204 |
def reconstruct_objects(image: np.ndarray):
|
| 205 |
"""
|
| 206 |
+
Automatically detect the largest object with SAM2 and reconstruct it
|
| 207 |
+
in 3D with SAM 3D Objects.
|
| 208 |
|
| 209 |
Args:
|
| 210 |
image: Input RGB image
|
|
|
|
| 214 |
"""
|
| 215 |
if image is None:
|
| 216 |
return None, None, "❌ No image provided"
|
|
|
|
| 217 |
try:
|
| 218 |
import torch
|
| 219 |
+
import time
|
| 220 |
+
t0 = time.time()
|
| 221 |
+
print(f"GPU: {torch.cuda.get_device_name()}")
|
| 222 |
+
|
| 223 |
+
from sam2.automatic_mask_generator import SAM2AutomaticMaskGenerator
|
| 224 |
+
sam2_gen = SAM2AutomaticMaskGenerator.from_pretrained("facebook/sam2-hiera-small")
|
| 225 |
+
print(f" SAM2 loaded ({time.time()-t0:.0f}s)")
|
| 226 |
+
|
| 227 |
+
image_np = np.array(image) if not isinstance(image, np.ndarray) else image
|
| 228 |
+
masks = sam2_gen.generate(image_np)
|
| 229 |
+
if not masks:
|
| 230 |
+
return None, image_np, "⚠️ No objects detected"
|
| 231 |
+
masks = sorted(masks, key=lambda x: x["area"], reverse=True)
|
| 232 |
+
best_mask = masks[0]["segmentation"]
|
| 233 |
+
|
| 234 |
+
preview = image_np.copy()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
preview[best_mask] = (preview[best_mask] * 0.5 + np.array([0, 255, 0]) * 0.5).astype(np.uint8)
|
| 236 |
+
print(f" {len(masks)} masks ({time.time()-t0:.0f}s)")
|
| 237 |
|
| 238 |
+
# Free SAM2 before loading SAM3D
|
| 239 |
+
del sam2_gen
|
| 240 |
+
torch.cuda.empty_cache()
|
| 241 |
|
| 242 |
+
from inference import Inference
|
| 243 |
+
sam3d = Inference(CONFIG_PATH, compile=False)
|
| 244 |
+
print(f" SAM3D loaded ({time.time()-t0:.0f}s)")
|
| 245 |
|
| 246 |
+
result = sam3d(image=image_np, mask=best_mask, seed=42)
|
| 247 |
+
print(f" Reconstructed ({time.time()-t0:.0f}s)")
|
| 248 |
if result is None:
|
| 249 |
+
return None, preview, "⚠️ Reconstruction returned None"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 250 |
|
| 251 |
+
glb = _gaussians_to_glb(result, tempfile.mkdtemp())
|
| 252 |
+
if glb is None:
|
| 253 |
+
keys = list(result.keys()) if isinstance(result, dict) else dir(result)
|
| 254 |
+
return None, preview, f"⚠️ Cannot extract 3D data. Keys: {keys}"
|
| 255 |
+
|
| 256 |
+
import trimesh
|
| 257 |
+
try:
|
| 258 |
+
n_faces = len(trimesh.load(glb, force="mesh").faces)
|
| 259 |
+
except Exception:
|
| 260 |
+
n_faces = 0
|
| 261 |
+
return glb, preview, f"✓ {len(masks)} objects detected, {n_faces:,} faces ({int(time.time()-t0)}s)"
|
| 262 |
except Exception as e:
|
| 263 |
import traceback
|
| 264 |
+
tb = traceback.format_exc()
|
| 265 |
+
print(tb)
|
| 266 |
+
return None, None, f"❌ Error:\n{tb[-1500:]}"
|
| 267 |
|
| 268 |
|
| 269 |
# Gradio Interface
|
|
|
|
| 275 |
Automatically detects objects and reconstructs the largest one in 3D.
|
| 276 |
""")
|
| 277 |
|
| 278 |
+
with gr.Tab("Reconstruct"):
|
| 279 |
+
with gr.Row():
|
| 280 |
+
with gr.Column():
|
| 281 |
+
input_image = gr.Image(label="Input Image", type="numpy")
|
| 282 |
+
btn = gr.Button("🚀 Detect & Reconstruct", variant="primary", size="lg")
|
| 283 |
+
with gr.Column():
|
| 284 |
+
preview = gr.Image(label="Detected Object", type="numpy", interactive=False)
|
| 285 |
+
status = gr.Textbox(label="Status")
|
| 286 |
+
with gr.Row():
|
|
|
|
|
|
|
| 287 |
output_model = gr.Model3D(label="3D Preview")
|
|
|
|
| 288 |
output_file = gr.File(label="Download GLB")
|
| 289 |
+
btn.click(reconstruct_objects, inputs=[input_image], outputs=[output_model, preview, status])
|
| 290 |
+
output_model.change(lambda x: x, inputs=[output_model], outputs=[output_file])
|
| 291 |
|
| 292 |
+
with gr.Tab("Diagnose"):
|
| 293 |
+
diag_btn = gr.Button("Diagnose GPU & Modules")
|
| 294 |
+
diag_out = gr.Textbox(lines=15, label="Diagnostics")
|
| 295 |
+
diag_btn.click(diagnose, outputs=[diag_out])
|
|
|
|
|
|
|
| 296 |
|
| 297 |
gr.Markdown("""
|
| 298 |
---
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/flash_attn_stub%2Fflash_attn%2F__init__.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fflash_attn_stub%2Fflash_attn%2F__init__.py=&etag=%22e8154a003b387b4a013f13f81e6b1d904ed91395%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/kaolin_stub%2Fkaolin%2F__init__.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fkaolin_stub%2Fkaolin%2F__init__.py=&etag=%22b33d127162fbf6be3b4108393add91478bb521bc%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/kaolin_stub%2Fkaolin%2Fops%2F__init__.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fkaolin_stub%2Fkaolin%2Fops%2F__init__.py=&etag=%227f1d5481770b3ffa39effdbf052adca321305592%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/kaolin_stub%2Fkaolin%2Fops%2Fmesh.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fkaolin_stub%2Fkaolin%2Fops%2Fmesh.py=&etag=%22d225c551eeaf6ff221c55bdc3521cb0a49ba8db8%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/kaolin_stub%2Fkaolin%2Frender%2F__init__.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fkaolin_stub%2Fkaolin%2Frender%2F__init__.py=&etag=%228b7c03b43ec0b7f5d8fa5812a970190761e6ec80%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/kaolin_stub%2Fkaolin%2Frender%2Fcamera.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fkaolin_stub%2Fkaolin%2Frender%2Fcamera.py=&etag=%2297e63314864f4ef5e2d5d3a21971d3a9297ebee3%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/kaolin_stub%2Fkaolin%2Futils%2F__init__.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fkaolin_stub%2Fkaolin%2Futils%2F__init__.py=&etag=%2297df528c28d988591233bdc1ef2ba44e78cddf94%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/kaolin_stub%2Fkaolin%2Futils%2Ftesting.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fkaolin_stub%2Fkaolin%2Futils%2Ftesting.py=&etag=%2270331aa66fe7f80563073a85d5102d6485e7213f%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/kaolin_stub%2Fkaolin%2Fvisualize%2F__init__.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fkaolin_stub%2Fkaolin%2Fvisualize%2F__init__.py=&etag=%228492fef06d88908029b877d623ab3f9ec746a767%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/pytorch3d_stub%2Fpytorch3d%2F__init__.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fpytorch3d_stub%2Fpytorch3d%2F__init__.py=&etag=%220be164bba8a444d77ed028c703c53164efa59be1%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/pytorch3d_stub%2Fpytorch3d%2Frenderer%2F__init__.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fpytorch3d_stub%2Fpytorch3d%2Frenderer%2F__init__.py=&etag=%224ad1da5bd5d463828d7c2da904e7e130b1644f44%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/pytorch3d_stub%2Fpytorch3d%2Frenderer%2Fcamera_utils.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fpytorch3d_stub%2Fpytorch3d%2Frenderer%2Fcamera_utils.py=&etag=%22819d1ca667d03548de636a7079ab25e239735d6c%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/pytorch3d_stub%2Fpytorch3d%2Frenderer%2Fcameras.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fpytorch3d_stub%2Fpytorch3d%2Frenderer%2Fcameras.py=&etag=%2222da58b8bd463e8347effc1530fac0e77313a845%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/pytorch3d_stub%2Fpytorch3d%2Frenderer%2Fmesh%2F__init__.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fpytorch3d_stub%2Fpytorch3d%2Frenderer%2Fmesh%2F__init__.py=&etag=%2254776a5dc4978bdd6adc39d981fe242aaa0327f7%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/pytorch3d_stub%2Fpytorch3d%2Frenderer%2Fmesh%2Ftextures.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fpytorch3d_stub%2Fpytorch3d%2Frenderer%2Fmesh%2Ftextures.py=&etag=%2254776a5dc4978bdd6adc39d981fe242aaa0327f7%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/pytorch3d_stub%2Fpytorch3d%2Fstructures%2F__init__.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fpytorch3d_stub%2Fpytorch3d%2Fstructures%2F__init__.py=&etag=%221d9561d58a998fb884e75446b8a8417d346bfa9a%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/pytorch3d_stub%2Fpytorch3d%2Ftransforms%2F__init__.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fpytorch3d_stub%2Fpytorch3d%2Ftransforms%2F__init__.py=&etag=%22561aad8e8e0e3e6182e3f62283f3cbdd710e4bf8%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/pytorch3d_stub%2Fpytorch3d%2Fvis%2F__init__.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fpytorch3d_stub%2Fpytorch3d%2Fvis%2F__init__.py=&etag=%22c887a2b86e91aa7227f07cf6d66d964ee465d552%22
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Temporary Redirect. Redirecting to /api/resolve-cache/spaces/jboth/sam3d-objects-fixed/7b8ab13b284b67e82dd8c321d957c0080dd66a51/pytorch3d_stub%2Fpytorch3d%2Fvis%2Fplotly_vis.py?%2Fspaces%2Fjboth%2Fsam3d-objects-fixed%2Fresolve%2Fmain%2Fpytorch3d_stub%2Fpytorch3d%2Fvis%2Fplotly_vis.py=&etag=%22623e37cd1073af06175464439f26d85ed2b79241%22
|
|
@@ -1,25 +1,50 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
Pillow>=10.0.0
|
| 12 |
-
trimesh>=4.0.0
|
| 13 |
-
open3d>=0.18.0
|
| 14 |
-
hydra-core>=1.3.0
|
| 15 |
-
omegaconf>=2.3.0
|
| 16 |
-
einops>=0.7.0
|
| 17 |
-
loguru>=0.7.0
|
| 18 |
-
seaborn==0.13.2
|
| 19 |
-
ninja
|
| 20 |
jaxtyping
|
| 21 |
rich
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
numpy
|
| 2 |
+
Pillow
|
| 3 |
+
trimesh
|
| 4 |
+
hydra-core
|
| 5 |
+
hydra-submitit-launcher
|
| 6 |
+
omegaconf
|
| 7 |
+
einops
|
| 8 |
+
einops-exts
|
| 9 |
+
loguru
|
| 10 |
+
seaborn
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
jaxtyping
|
| 12 |
rich
|
| 13 |
+
torchvision
|
| 14 |
+
opencv-python-headless
|
| 15 |
+
ninja
|
| 16 |
+
timm
|
| 17 |
+
optree
|
| 18 |
+
roma
|
| 19 |
+
fvcore
|
| 20 |
+
rootutils
|
| 21 |
+
easydict
|
| 22 |
+
scikit-image
|
| 23 |
+
colorama
|
| 24 |
+
ftfy
|
| 25 |
+
astor
|
| 26 |
+
av
|
| 27 |
+
gdown
|
| 28 |
+
deprecation
|
| 29 |
+
orjson
|
| 30 |
+
randomname
|
| 31 |
+
simplejson
|
| 32 |
+
pydot
|
| 33 |
+
igraph
|
| 34 |
+
plyfile
|
| 35 |
+
point-cloud-utils
|
| 36 |
+
pymeshfix
|
| 37 |
+
xatlas
|
| 38 |
+
pycocotools
|
| 39 |
+
lightning
|
| 40 |
+
pytorch-lightning
|
| 41 |
+
scipy
|
| 42 |
+
pyvista
|
| 43 |
+
plotly
|
| 44 |
+
kaleido
|
| 45 |
+
webdataset
|
| 46 |
+
wandb
|
| 47 |
+
tensorboard
|
| 48 |
+
sentence-transformers
|
| 49 |
+
accelerate
|
| 50 |
+
safetensors
|