Spaces:
Running on Zero
Running on Zero
Deploy DiffusionGS online generation demos
Browse files- .gitattributes +1 -0
- README.md +1 -1
- diffusionGS/configs/diffusionGS_scene_eval.yaml +2 -2
- diffusionGS/configs/diffusionGS_scene_eval_512.yaml +3 -3
- diffusionGS/data/base.py +6 -4
- diffusionGS/data/base_scene.py +5 -4
- diffusionGS/models/denoiser/denoiser.py +4 -26
- diffusionGS/models/denoiser/denoiser_scene.py +4 -25
- diffusionGS/models/denoiser/denoiser_utils.py +3 -3
- diffusionGS/models/gsrenderer/gs_core.py +28 -10
- diffusionGS/pipline_obj.py +16 -3
- diffusionGS/systems/diffusion_gs_system_scene.py +20 -11
- diffusionGS/utils/callbacks.py +3 -3
- diffusionGS/utils/checkpoint.py +13 -1
- diffusionGS/utils/losses.py +3 -3
- diffusionGS/utils/mesh_preview.py +93 -0
- diffusionGS/utils/misc.py +4 -4
- img/demos/object_demo.png +2 -2
- img/demos/online_demos.png +3 -0
- object_demo.py +65 -27
.gitattributes
CHANGED
|
@@ -43,3 +43,4 @@ extra_files/test_cases/popmart.png filter=lfs diff=lfs merge=lfs -text
|
|
| 43 |
wheels/diff_gaussian_rasterization-0.0.0-cp312-cp312-linux_x86_64.whl filter=lfs diff=lfs merge=lfs -text
|
| 44 |
img/demos/object_demo.png filter=lfs diff=lfs merge=lfs -text
|
| 45 |
img/demos/scene_demo.png filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 43 |
wheels/diff_gaussian_rasterization-0.0.0-cp312-cp312-linux_x86_64.whl filter=lfs diff=lfs merge=lfs -text
|
| 44 |
img/demos/object_demo.png filter=lfs diff=lfs merge=lfs -text
|
| 45 |
img/demos/scene_demo.png filter=lfs diff=lfs merge=lfs -text
|
| 46 |
+
img/demos/online_demos.png filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
|
@@ -21,7 +21,7 @@ tags:
|
|
| 21 |
|
| 22 |
# DiffusionGS · Object Generation
|
| 23 |
|
| 24 |
-
Generate a 3D object from one reference image. Upload an image or choose an example, click **Generate object**, then preview and download the rendered views and Gaussian point cloud. **Extract mesh**
|
| 25 |
|
| 26 |
[Scene demo](https://huggingface.co/spaces/CaiYuanhao/DiffusionGS-Scene) · [GitHub](https://github.com/caiyuanhao1998/Open-DiffusionGS) · [Models](https://huggingface.co/CaiYuanhao/DiffusionGS) · [Paper](https://arxiv.org/abs/2411.14384)
|
| 27 |
|
|
|
|
| 21 |
|
| 22 |
# DiffusionGS · Object Generation
|
| 23 |
|
| 24 |
+
Generate a 3D object from one reference image. Upload an image or choose an example, click **Generate object**, then preview and download the rendered views and Gaussian point cloud. **Extract mesh** creates a colored mesh preview with GLB and OBJ downloads.
|
| 25 |
|
| 26 |
[Scene demo](https://huggingface.co/spaces/CaiYuanhao/DiffusionGS-Scene) · [GitHub](https://github.com/caiyuanhao1998/Open-DiffusionGS) · [Models](https://huggingface.co/CaiYuanhao/DiffusionGS) · [Paper](https://arxiv.org/abs/2411.14384)
|
| 27 |
|
diffusionGS/configs/diffusionGS_scene_eval.yaml
CHANGED
|
@@ -14,7 +14,7 @@ data:
|
|
| 14 |
sel_views_train: 4
|
| 15 |
training_res: [256, 256]
|
| 16 |
batch_size: 16
|
| 17 |
-
eval_batch_size:
|
| 18 |
num_workers: 4
|
| 19 |
num_workers_val: 4
|
| 20 |
|
|
@@ -89,4 +89,4 @@ trainer:
|
|
| 89 |
checkpoint:
|
| 90 |
save_last: true
|
| 91 |
save_top_k: -1
|
| 92 |
-
every_n_train_steps: 5000
|
|
|
|
| 14 |
sel_views_train: 4
|
| 15 |
training_res: [256, 256]
|
| 16 |
batch_size: 16
|
| 17 |
+
eval_batch_size: 1
|
| 18 |
num_workers: 4
|
| 19 |
num_workers_val: 4
|
| 20 |
|
|
|
|
| 89 |
checkpoint:
|
| 90 |
save_last: true
|
| 91 |
save_top_k: -1
|
| 92 |
+
every_n_train_steps: 5000
|
diffusionGS/configs/diffusionGS_scene_eval_512.yaml
CHANGED
|
@@ -14,7 +14,7 @@ data:
|
|
| 14 |
sel_views_train: 4
|
| 15 |
training_res: [512, 512]
|
| 16 |
batch_size: 16
|
| 17 |
-
eval_batch_size:
|
| 18 |
num_workers: 4
|
| 19 |
num_workers_val: 4
|
| 20 |
|
|
@@ -22,7 +22,7 @@ system_type: "diffusion-gs-scene-system"
|
|
| 22 |
system:
|
| 23 |
num_inference_steps: 30
|
| 24 |
save_intermediate_video: True
|
| 25 |
-
save_result_for_eval:
|
| 26 |
|
| 27 |
|
| 28 |
shape_model_type: "diffusion-gs-model-scene"
|
|
@@ -93,4 +93,4 @@ trainer:
|
|
| 93 |
checkpoint:
|
| 94 |
save_last: true
|
| 95 |
save_top_k: -1
|
| 96 |
-
every_n_train_steps: 5000
|
|
|
|
| 14 |
sel_views_train: 4
|
| 15 |
training_res: [512, 512]
|
| 16 |
batch_size: 16
|
| 17 |
+
eval_batch_size: 1
|
| 18 |
num_workers: 4
|
| 19 |
num_workers_val: 4
|
| 20 |
|
|
|
|
| 22 |
system:
|
| 23 |
num_inference_steps: 30
|
| 24 |
save_intermediate_video: True
|
| 25 |
+
save_result_for_eval: True
|
| 26 |
|
| 27 |
|
| 28 |
shape_model_type: "diffusion-gs-model-scene"
|
|
|
|
| 93 |
checkpoint:
|
| 94 |
save_last: true
|
| 95 |
save_top_k: -1
|
| 96 |
+
every_n_train_steps: 5000
|
diffusionGS/data/base.py
CHANGED
|
@@ -106,6 +106,8 @@ class BaseDataset(Dataset):
|
|
| 106 |
self.cfg: BaseDataModuleConfig = cfg
|
| 107 |
self.split = split
|
| 108 |
self.uids = json.load(open(f'{cfg.local_dir}/{split}.json'))
|
|
|
|
|
|
|
| 109 |
self.train_idxs = get_remaining_indices(self.cfg.gen_idxs,self.cfg.all_idxs)
|
| 110 |
# Default camera intrinsics [for Gobjaverse]
|
| 111 |
self.fxfycxcy = torch.tensor([self.cfg.default_fxfy, self.cfg.default_fxfy, 0.5, 0.5], dtype=torch.float32)
|
|
@@ -166,7 +168,8 @@ class BaseDataset(Dataset):
|
|
| 166 |
train_idxs = get_remaining_indices(sel_gen_idxs,self.cfg.all_idxs)
|
| 167 |
sel_train_idxs = random.sample(train_idxs, k=self.cfg.sel_views) # hard code for 40 views [GOBJAVERSE]
|
| 168 |
all_sel_idxs = sel_gen_idxs + sel_train_idxs
|
| 169 |
-
image_file_dirs = [self.cfg.image_dir
|
|
|
|
| 170 |
# background color
|
| 171 |
background_color = torch.as_tensor(self.cfg.background_color)
|
| 172 |
rgbs = []
|
|
@@ -246,8 +249,7 @@ class BaseDataset(Dataset):
|
|
| 246 |
try:
|
| 247 |
return self._get_data(index)
|
| 248 |
except Exception as e:
|
| 249 |
-
|
| 250 |
-
return self.__getitem__(np.random.randint(len(self)))
|
| 251 |
|
| 252 |
def collate(self, batch):
|
| 253 |
from torch.utils.data._utils.collate import default_collate_fn_map
|
|
@@ -262,4 +264,4 @@ class BaseDataset(Dataset):
|
|
| 262 |
else:
|
| 263 |
print(key)
|
| 264 |
ret[key] = default_collate_fn_map[type(batch[0][key])](batch)
|
| 265 |
-
return ret
|
|
|
|
| 106 |
self.cfg: BaseDataModuleConfig = cfg
|
| 107 |
self.split = split
|
| 108 |
self.uids = json.load(open(f'{cfg.local_dir}/{split}.json'))
|
| 109 |
+
if not self.uids:
|
| 110 |
+
raise ValueError(f"No {split} objects listed in {cfg.local_dir}")
|
| 111 |
self.train_idxs = get_remaining_indices(self.cfg.gen_idxs,self.cfg.all_idxs)
|
| 112 |
# Default camera intrinsics [for Gobjaverse]
|
| 113 |
self.fxfycxcy = torch.tensor([self.cfg.default_fxfy, self.cfg.default_fxfy, 0.5, 0.5], dtype=torch.float32)
|
|
|
|
| 168 |
train_idxs = get_remaining_indices(sel_gen_idxs,self.cfg.all_idxs)
|
| 169 |
sel_train_idxs = random.sample(train_idxs, k=self.cfg.sel_views) # hard code for 40 views [GOBJAVERSE]
|
| 170 |
all_sel_idxs = sel_gen_idxs + sel_train_idxs
|
| 171 |
+
image_file_dirs = [os.path.join(self.cfg.image_dir, str(uid), "campos_512_v4", f"{sel_idx:05d}")
|
| 172 |
+
for sel_idx in all_sel_idxs]
|
| 173 |
# background color
|
| 174 |
background_color = torch.as_tensor(self.cfg.background_color)
|
| 175 |
rgbs = []
|
|
|
|
| 249 |
try:
|
| 250 |
return self._get_data(index)
|
| 251 |
except Exception as e:
|
| 252 |
+
raise RuntimeError(f"Could not load object {self.uids[index]}: {e}") from e
|
|
|
|
| 253 |
|
| 254 |
def collate(self, batch):
|
| 255 |
from torch.utils.data._utils.collate import default_collate_fn_map
|
|
|
|
| 264 |
else:
|
| 265 |
print(key)
|
| 266 |
ret[key] = default_collate_fn_map[type(batch[0][key])](batch)
|
| 267 |
+
return ret
|
diffusionGS/data/base_scene.py
CHANGED
|
@@ -71,6 +71,8 @@ class BaseDataset(Dataset):
|
|
| 71 |
all_scene_paths = all_scene_paths[:self.cfg.eval_subset]
|
| 72 |
self.uids = all_scene_paths
|
| 73 |
# load anything
|
|
|
|
|
|
|
| 74 |
print(f"Loaded {len(self.uids)} {split} uids")
|
| 75 |
# breakpoint()
|
| 76 |
def __len__(self):
|
|
@@ -149,7 +151,7 @@ class BaseDataset(Dataset):
|
|
| 149 |
|
| 150 |
# Rescale the whole scene to a fixed scale
|
| 151 |
scene_scale = torch.max(torch.abs(in_c2ws[:, :3, 3]))
|
| 152 |
-
scene_scale = scene_scale_factor * scene_scale
|
| 153 |
|
| 154 |
in_c2ws[:, :3, 3] /= scene_scale
|
| 155 |
|
|
@@ -205,8 +207,7 @@ class BaseDataset(Dataset):
|
|
| 205 |
try:
|
| 206 |
return self._get_data(index)
|
| 207 |
except Exception as e:
|
| 208 |
-
|
| 209 |
-
return self.__getitem__(np.random.randint(len(self)))
|
| 210 |
|
| 211 |
def collate(self, batch):
|
| 212 |
from torch.utils.data._utils.collate import default_collate_fn_map
|
|
@@ -221,4 +222,4 @@ class BaseDataset(Dataset):
|
|
| 221 |
else:
|
| 222 |
print(key)
|
| 223 |
ret[key] = default_collate_fn_map[type(batch[0][key])](batch)
|
| 224 |
-
return ret
|
|
|
|
| 71 |
all_scene_paths = all_scene_paths[:self.cfg.eval_subset]
|
| 72 |
self.uids = all_scene_paths
|
| 73 |
# load anything
|
| 74 |
+
if not self.uids:
|
| 75 |
+
raise ValueError(f"No {split} scenes found; check the scene list and evaluation index")
|
| 76 |
print(f"Loaded {len(self.uids)} {split} uids")
|
| 77 |
# breakpoint()
|
| 78 |
def __len__(self):
|
|
|
|
| 151 |
|
| 152 |
# Rescale the whole scene to a fixed scale
|
| 153 |
scene_scale = torch.max(torch.abs(in_c2ws[:, :3, 3]))
|
| 154 |
+
scene_scale = (scene_scale_factor * scene_scale).clamp_min(1e-8)
|
| 155 |
|
| 156 |
in_c2ws[:, :3, 3] /= scene_scale
|
| 157 |
|
|
|
|
| 207 |
try:
|
| 208 |
return self._get_data(index)
|
| 209 |
except Exception as e:
|
| 210 |
+
raise RuntimeError(f"Could not load scene {self.uids[index]}: {e}") from e
|
|
|
|
| 211 |
|
| 212 |
def collate(self, batch):
|
| 213 |
from torch.utils.data._utils.collate import default_collate_fn_map
|
|
|
|
| 222 |
else:
|
| 223 |
print(key)
|
| 224 |
ret[key] = default_collate_fn_map[type(batch[0][key])](batch)
|
| 225 |
+
return ret
|
diffusionGS/models/denoiser/denoiser.py
CHANGED
|
@@ -9,7 +9,7 @@ from easydict import EasyDict as edict
|
|
| 9 |
import diffusionGS
|
| 10 |
from diffusionGS.models.transformers.utils_transformer import DiTBlock,_init_weights
|
| 11 |
from diffusionGS.models.gsrenderer.renderer import Renderer
|
| 12 |
-
from diffusionGS.utils.checkpoint import checkpoint
|
| 13 |
from diffusionGS.utils.base import BaseModule
|
| 14 |
from diffusionGS.utils.typing import *
|
| 15 |
from diffusionGS.utils.ops import generate_dense_grid_points
|
|
@@ -255,31 +255,9 @@ class DGSDenoiser(BaseModule):
|
|
| 255 |
# renderer
|
| 256 |
if self.cfg.pretrained_model_name_or_path != "":
|
| 257 |
print(f"Loading pretrained shape model from {self.cfg.pretrained_model_name_or_path}")
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
_pretrained_ckpt = {}
|
| 262 |
-
# breakpoint()
|
| 263 |
-
for k, v in pret_weights.items():
|
| 264 |
-
if k.startswith('denoiser.') and not k.startswith('denoiser.loss_computer'):
|
| 265 |
-
_pretrained_ckpt[k.replace('denoiser.', '')] = v
|
| 266 |
-
#
|
| 267 |
-
pretrained_ckpt = _pretrained_ckpt
|
| 268 |
-
|
| 269 |
-
# if 'state_dict' in pretrained_ckpt:
|
| 270 |
-
# _pretrained_ckpt = {}
|
| 271 |
-
# for k, v in pretrained_ckpt['state_dict'].items():
|
| 272 |
-
# if k.startswith('shape_model.'):
|
| 273 |
-
# _pretrained_ckpt[k.replace('shape_model.', '')] = v
|
| 274 |
-
# pretrained_ckpt = _pretrained_ckpt
|
| 275 |
-
# else:
|
| 276 |
-
# _pretrained_ckpt = {}
|
| 277 |
-
# for k, v in pretrained_ckpt.items():
|
| 278 |
-
# if k.startswith('shape_model.'):
|
| 279 |
-
# _pretrained_ckpt[k.replace('shape_model.', '')] = v
|
| 280 |
-
# pretrained_ckpt = _pretrained_ckpt
|
| 281 |
-
# breakpoint()
|
| 282 |
-
self.load_state_dict(pretrained_ckpt, strict=True)
|
| 283 |
|
| 284 |
def forward(self, input_batch, timesteps):
|
| 285 |
guassians_parameters,_ = self.image_to_gaussians(input_batch['image'], input_batch['ray_o'], input_batch['ray_d'], timesteps)
|
|
|
|
| 9 |
import diffusionGS
|
| 10 |
from diffusionGS.models.transformers.utils_transformer import DiTBlock,_init_weights
|
| 11 |
from diffusionGS.models.gsrenderer.renderer import Renderer
|
| 12 |
+
from diffusionGS.utils.checkpoint import checkpoint, load_denoiser_checkpoint
|
| 13 |
from diffusionGS.utils.base import BaseModule
|
| 14 |
from diffusionGS.utils.typing import *
|
| 15 |
from diffusionGS.utils.ops import generate_dense_grid_points
|
|
|
|
| 255 |
# renderer
|
| 256 |
if self.cfg.pretrained_model_name_or_path != "":
|
| 257 |
print(f"Loading pretrained shape model from {self.cfg.pretrained_model_name_or_path}")
|
| 258 |
+
self.load_state_dict(
|
| 259 |
+
load_denoiser_checkpoint(self.cfg.pretrained_model_name_or_path), strict=True,
|
| 260 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 261 |
|
| 262 |
def forward(self, input_batch, timesteps):
|
| 263 |
guassians_parameters,_ = self.image_to_gaussians(input_batch['image'], input_batch['ray_o'], input_batch['ray_d'], timesteps)
|
diffusionGS/models/denoiser/denoiser_scene.py
CHANGED
|
@@ -9,7 +9,7 @@ from easydict import EasyDict as edict
|
|
| 9 |
import diffusionGS
|
| 10 |
from diffusionGS.models.transformers.utils_transformer import DiTBlock,_init_weights
|
| 11 |
from diffusionGS.models.gsrenderer.renderer import Renderer,SceneRenderer
|
| 12 |
-
from diffusionGS.utils.checkpoint import checkpoint
|
| 13 |
from diffusionGS.utils.base import BaseModule
|
| 14 |
from diffusionGS.utils.typing import *
|
| 15 |
from diffusionGS.utils.ops import generate_dense_grid_points
|
|
@@ -264,30 +264,9 @@ class DGSDenoiser(BaseModule):
|
|
| 264 |
# renderer
|
| 265 |
if self.cfg.pretrained_model_name_or_path != "":
|
| 266 |
print(f"Loading pretrained shape model from {self.cfg.pretrained_model_name_or_path}")
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
_pretrained_ckpt = {}
|
| 271 |
-
# breakpoint()
|
| 272 |
-
for k, v in pret_weights.items():
|
| 273 |
-
if k.startswith('denoiser.') and not k.startswith('denoiser.loss_computer'):
|
| 274 |
-
_pretrained_ckpt[k.replace('denoiser.', '')] = v
|
| 275 |
-
#
|
| 276 |
-
pretrained_ckpt = _pretrained_ckpt
|
| 277 |
-
|
| 278 |
-
if 'state_dict' in pretrained_ckpt:
|
| 279 |
-
_pretrained_ckpt = {}
|
| 280 |
-
for k, v in pretrained_ckpt['state_dict'].items():
|
| 281 |
-
if k.startswith('shape_model.'):
|
| 282 |
-
_pretrained_ckpt[k.replace('shape_model.', '')] = v
|
| 283 |
-
pretrained_ckpt = _pretrained_ckpt
|
| 284 |
-
else:
|
| 285 |
-
_pretrained_ckpt = {}
|
| 286 |
-
for k, v in pretrained_ckpt.items():
|
| 287 |
-
if k.startswith('shape_model.'):
|
| 288 |
-
_pretrained_ckpt[k.replace('shape_model.', '')] = v
|
| 289 |
-
pretrained_ckpt = _pretrained_ckpt
|
| 290 |
-
self.load_state_dict(pretrained_ckpt, strict=True)
|
| 291 |
|
| 292 |
def forward(self, input_batch, timesteps):
|
| 293 |
guassians_parameters,_ = self.image_to_gaussians(input_batch['image'], input_batch['ray_o'], input_batch['ray_d'], timesteps)
|
|
|
|
| 9 |
import diffusionGS
|
| 10 |
from diffusionGS.models.transformers.utils_transformer import DiTBlock,_init_weights
|
| 11 |
from diffusionGS.models.gsrenderer.renderer import Renderer,SceneRenderer
|
| 12 |
+
from diffusionGS.utils.checkpoint import checkpoint, load_denoiser_checkpoint
|
| 13 |
from diffusionGS.utils.base import BaseModule
|
| 14 |
from diffusionGS.utils.typing import *
|
| 15 |
from diffusionGS.utils.ops import generate_dense_grid_points
|
|
|
|
| 264 |
# renderer
|
| 265 |
if self.cfg.pretrained_model_name_or_path != "":
|
| 266 |
print(f"Loading pretrained shape model from {self.cfg.pretrained_model_name_or_path}")
|
| 267 |
+
self.load_state_dict(
|
| 268 |
+
load_denoiser_checkpoint(self.cfg.pretrained_model_name_or_path), strict=True,
|
| 269 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 270 |
|
| 271 |
def forward(self, input_batch, timesteps):
|
| 272 |
guassians_parameters,_ = self.image_to_gaussians(input_batch['image'], input_batch['ray_o'], input_batch['ray_d'], timesteps)
|
diffusionGS/models/denoiser/denoiser_utils.py
CHANGED
|
@@ -39,8 +39,8 @@ class SplitData(nn.Module):
|
|
| 39 |
for key, value in data_batch.items():
|
| 40 |
try:
|
| 41 |
input[key] = value[:, : self.config.training.num_input_views, ...] # 取前 num_input_views 个视图, 在应用中 num_input_views = 4
|
| 42 |
-
except:
|
| 43 |
-
|
| 44 |
if self.config.training.num_target_views >= value.size(1): # 如果 target 的数量大于等于输入的数量, 4 < 8, 这一步不执行
|
| 45 |
target[key] = value
|
| 46 |
else:
|
|
@@ -689,4 +689,4 @@ class TransformMeshTarget(nn.Module):
|
|
| 689 |
ray_o=ray_o,
|
| 690 |
ray_d=ray_d,
|
| 691 |
)
|
| 692 |
-
return input
|
|
|
|
| 39 |
for key, value in data_batch.items():
|
| 40 |
try:
|
| 41 |
input[key] = value[:, : self.config.training.num_input_views, ...] # 取前 num_input_views 个视图, 在应用中 num_input_views = 4
|
| 42 |
+
except Exception as exc:
|
| 43 |
+
raise ValueError(f"Batch field {key!r} must have batch and view dimensions") from exc
|
| 44 |
if self.config.training.num_target_views >= value.size(1): # 如果 target 的数量大于等于输入的数量, 4 < 8, 这一步不执行
|
| 45 |
target[key] = value
|
| 46 |
else:
|
|
|
|
| 689 |
ray_o=ray_o,
|
| 690 |
ray_d=ray_d,
|
| 691 |
)
|
| 692 |
+
return input
|
diffusionGS/models/gsrenderer/gs_core.py
CHANGED
|
@@ -784,9 +784,12 @@ class GaussianModel:
|
|
| 784 |
self._rotation = torch.from_numpy(rots.astype(np.float32)).contiguous()
|
| 785 |
|
| 786 |
def extract_fields(self, resolution=128, num_blocks=16, relax_ratio=1.5):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 787 |
# resolution: resolution of field
|
| 788 |
block_size = 2 / num_blocks
|
| 789 |
-
assert resolution % block_size == 0
|
| 790 |
split_size = resolution // num_blocks
|
| 791 |
## recenter
|
| 792 |
xyzs = self.get_xyz
|
|
@@ -810,22 +813,34 @@ class GaussianModel:
|
|
| 810 |
Y = torch.linspace(-1, 1, resolution).split(split_size)
|
| 811 |
Z = torch.linspace(-1, 1, resolution).split(split_size)
|
| 812 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 813 |
|
| 814 |
# loop blocks (assume max size of gaussian is small than relax_ratio * block_size !!!)
|
| 815 |
for xi, xs in enumerate(X):
|
| 816 |
for yi, ys in enumerate(Y):
|
| 817 |
for zi, zs in enumerate(Z):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 818 |
xx, yy, zz = torch.meshgrid(xs, ys, zs)
|
| 819 |
# sample points [M, 3]
|
| 820 |
pts = torch.cat([xx.reshape(-1, 1), yy.reshape(-1, 1), zz.reshape(-1, 1)], dim=-1).to(device)
|
| 821 |
-
# in-tile gaussians mask
|
| 822 |
-
vmin, vmax = pts.amin(0), pts.amax(0)
|
| 823 |
-
vmin -= block_size * relax_ratio
|
| 824 |
-
vmax += block_size * relax_ratio
|
| 825 |
-
mask = (xyzs < vmax).all(-1) & (xyzs > vmin).all(-1)
|
| 826 |
-
# if hit no gaussian, continue to next block
|
| 827 |
-
if not mask.any():
|
| 828 |
-
continue
|
| 829 |
mask_xyzs = xyzs[mask] # [L, 3]
|
| 830 |
mask_covs = covs[mask] # [L, 6]
|
| 831 |
mask_opas = opacities[mask].view(1, -1) # [L, 1] --> [1, L]
|
|
@@ -861,6 +876,9 @@ class GaussianModel:
|
|
| 861 |
vertices, triangles = clean_mesh(vertices, triangles, remesh=True, remesh_size=0.015)
|
| 862 |
if decimate_target > 0 and triangles.shape[0] > decimate_target:
|
| 863 |
vertices, triangles = decimate_mesh(vertices, triangles, decimate_target)
|
|
|
|
|
|
|
|
|
|
| 864 |
# print(
|
| 865 |
# f"[INFO] marching cubes result: {v.shape} ({v.min().item()}-{v.max().item()}), {f.shape}"
|
| 866 |
# )
|
|
@@ -1379,4 +1397,4 @@ if __name__ == "__main__":
|
|
| 1379 |
Image.fromarray(im).save(f"{out_dir}/render_{i:08d}.png")
|
| 1380 |
|
| 1381 |
create_video(out_dir, f"{out_dir}/render.mp4", framerate=30)
|
| 1382 |
-
print(f"Saved {out_dir}/render.mp4")
|
|
|
|
| 784 |
self._rotation = torch.from_numpy(rots.astype(np.float32)).contiguous()
|
| 785 |
|
| 786 |
def extract_fields(self, resolution=128, num_blocks=16, relax_ratio=1.5):
|
| 787 |
+
from scipy.spatial import cKDTree
|
| 788 |
+
|
| 789 |
+
if num_blocks < 1 or resolution < num_blocks or resolution % num_blocks:
|
| 790 |
+
raise ValueError("Field resolution must be a positive multiple of num_blocks.")
|
| 791 |
# resolution: resolution of field
|
| 792 |
block_size = 2 / num_blocks
|
|
|
|
| 793 |
split_size = resolution // num_blocks
|
| 794 |
## recenter
|
| 795 |
xyzs = self.get_xyz
|
|
|
|
| 813 |
Y = torch.linspace(-1, 1, resolution).split(split_size)
|
| 814 |
Z = torch.linspace(-1, 1, resolution).split(split_size)
|
| 815 |
|
| 816 |
+
# Find each tile's Gaussian centers on the CPU. Scanning the full cloud
|
| 817 |
+
# and synchronizing mask.any() for every tile is especially slow for
|
| 818 |
+
# sparse clouds and shared GPUs (64**3 tiles in the default mesh).
|
| 819 |
+
centers = xyzs.detach().cpu().numpy()
|
| 820 |
+
tree = cKDTree(centers)
|
| 821 |
|
| 822 |
# loop blocks (assume max size of gaussian is small than relax_ratio * block_size !!!)
|
| 823 |
for xi, xs in enumerate(X):
|
| 824 |
for yi, ys in enumerate(Y):
|
| 825 |
for zi, zs in enumerate(Z):
|
| 826 |
+
lower = np.array([xs[0], ys[0], zs[0]], dtype=np.float32) - block_size * relax_ratio
|
| 827 |
+
upper = np.array([xs[-1], ys[-1], zs[-1]], dtype=np.float32) + block_size * relax_ratio
|
| 828 |
+
midpoint = (lower.astype(np.float64) + upper) / 2
|
| 829 |
+
radius = np.max((upper.astype(np.float64) - lower) / 2)
|
| 830 |
+
candidates = tree.query_ball_point(midpoint, np.nextafter(radius, np.inf), p=np.inf)
|
| 831 |
+
if not candidates:
|
| 832 |
+
continue
|
| 833 |
+
candidates = np.asarray(candidates, dtype=np.int64)
|
| 834 |
+
in_tile = ((centers[candidates] > lower) & (centers[candidates] < upper)).all(-1)
|
| 835 |
+
candidates = np.sort(candidates[in_tile])
|
| 836 |
+
if len(candidates) == 0:
|
| 837 |
+
continue
|
| 838 |
+
# Preserve the original accumulation order and strict tile
|
| 839 |
+
# bounds so the density field stays numerically equivalent.
|
| 840 |
+
mask = torch.as_tensor(candidates, device=device)
|
| 841 |
xx, yy, zz = torch.meshgrid(xs, ys, zs)
|
| 842 |
# sample points [M, 3]
|
| 843 |
pts = torch.cat([xx.reshape(-1, 1), yy.reshape(-1, 1), zz.reshape(-1, 1)], dim=-1).to(device)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 844 |
mask_xyzs = xyzs[mask] # [L, 3]
|
| 845 |
mask_covs = covs[mask] # [L, 6]
|
| 846 |
mask_opas = opacities[mask].view(1, -1) # [L, 1] --> [1, L]
|
|
|
|
| 876 |
vertices, triangles = clean_mesh(vertices, triangles, remesh=True, remesh_size=0.015)
|
| 877 |
if decimate_target > 0 and triangles.shape[0] > decimate_target:
|
| 878 |
vertices, triangles = decimate_mesh(vertices, triangles, decimate_target)
|
| 879 |
+
# Field queries use centered/scaled coordinates; export in the same
|
| 880 |
+
# world coordinates as the original Gaussian cloud.
|
| 881 |
+
vertices = vertices / self.mesh_scale + self.mesh_center.detach().cpu().numpy()
|
| 882 |
# print(
|
| 883 |
# f"[INFO] marching cubes result: {v.shape} ({v.min().item()}-{v.max().item()}), {f.shape}"
|
| 884 |
# )
|
|
|
|
| 1397 |
Image.fromarray(im).save(f"{out_dir}/render_{i:08d}.png")
|
| 1398 |
|
| 1399 |
create_video(out_dir, f"{out_dir}/render.mp4", framerate=30)
|
| 1400 |
+
print(f"Saved {out_dir}/render.mp4")
|
diffusionGS/pipline_obj.py
CHANGED
|
@@ -2,12 +2,14 @@ import os
|
|
| 2 |
import warnings
|
| 3 |
from typing import Callable, List, Optional, Union, Dict, Any
|
| 4 |
import PIL.Image
|
|
|
|
| 5 |
import trimesh
|
| 6 |
import torch
|
| 7 |
import numpy as np
|
| 8 |
from huggingface_hub import hf_hub_download
|
| 9 |
from diffusers.utils import BaseOutput
|
| 10 |
import torch.nn.functional as F
|
|
|
|
| 11 |
import diffusionGS
|
| 12 |
from diffusionGS.utils.config import ExperimentConfig, load_config
|
| 13 |
from diffusionGS.systems.utils import *
|
|
@@ -90,6 +92,10 @@ class DiffusionGSPipeline():
|
|
| 90 |
"""
|
| 91 |
if isinstance(image, str):
|
| 92 |
assert os.path.isfile(image) or image.startswith("http"), "Input image must be a valid URL or a file path."
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
elif not isinstance(image, (torch.Tensor, PIL.Image.Image)):
|
| 94 |
raise ValueError("Input image must be a `torch.Tensor` or `PIL.Image.Image`.")
|
| 95 |
|
|
@@ -110,9 +116,11 @@ class DiffusionGSPipeline():
|
|
| 110 |
Returns:
|
| 111 |
`List[PIL.Image.Image]`: List of `PIL.Image.Image` objects representing the preprocessed image.
|
| 112 |
"""
|
|
|
|
|
|
|
| 113 |
preprocessed_images = []
|
| 114 |
for i in range(len(images_pil)):
|
| 115 |
-
image = images_pil[i]
|
| 116 |
width, height, size = image.width, image.height, image.size
|
| 117 |
do_remove = True
|
| 118 |
if image.mode == "RGBA" and image.getextrema()[3][0] < 255:
|
|
@@ -127,10 +135,12 @@ class DiffusionGSPipeline():
|
|
| 127 |
# calculate the min bbox of the image
|
| 128 |
alpha = image.split()[-1]
|
| 129 |
bboxs = alpha.getbbox()
|
|
|
|
|
|
|
| 130 |
x1, y1, x2, y2 = bboxs
|
| 131 |
dy, dx = y2 - y1, x2 - x1
|
| 132 |
s = min(height * foreground_ratio / dy, width * foreground_ratio / dx)
|
| 133 |
-
Ht, Wt = int(dy * s), int(dx * s)
|
| 134 |
|
| 135 |
background = PIL.Image.new("RGBA", image.size, (*background_color, 255))
|
| 136 |
image = PIL.Image.alpha_composite(background, image)
|
|
@@ -227,7 +237,8 @@ class DiffusionGSPipeline():
|
|
| 227 |
[`~MeshPipelineOutput`] or `tuple`: [`~MeshPipelineOutput`] if `return_dict` is True, otherwise a `tuple`.
|
| 228 |
When returning a tuple, the first element is a list with the generated meshes.
|
| 229 |
"""
|
| 230 |
-
|
|
|
|
| 231 |
# 0. Check inputs. Raise error if not correct
|
| 232 |
self.check_inputs(
|
| 233 |
image=image,
|
|
@@ -235,6 +246,8 @@ class DiffusionGSPipeline():
|
|
| 235 |
|
| 236 |
# 1. Define call parameters
|
| 237 |
if isinstance(image, torch.Tensor):
|
|
|
|
|
|
|
| 238 |
batch_size = image.shape[0]
|
| 239 |
elif isinstance(image, PIL.Image.Image) or isinstance(image, str):
|
| 240 |
batch_size = 1
|
|
|
|
| 2 |
import warnings
|
| 3 |
from typing import Callable, List, Optional, Union, Dict, Any
|
| 4 |
import PIL.Image
|
| 5 |
+
from PIL import ImageOps
|
| 6 |
import trimesh
|
| 7 |
import torch
|
| 8 |
import numpy as np
|
| 9 |
from huggingface_hub import hf_hub_download
|
| 10 |
from diffusers.utils import BaseOutput
|
| 11 |
import torch.nn.functional as F
|
| 12 |
+
import torchvision.transforms.functional as TF
|
| 13 |
import diffusionGS
|
| 14 |
from diffusionGS.utils.config import ExperimentConfig, load_config
|
| 15 |
from diffusionGS.systems.utils import *
|
|
|
|
| 92 |
"""
|
| 93 |
if isinstance(image, str):
|
| 94 |
assert os.path.isfile(image) or image.startswith("http"), "Input image must be a valid URL or a file path."
|
| 95 |
+
elif isinstance(image, torch.Tensor):
|
| 96 |
+
if (image.ndim not in (3, 4) or image.shape[-3] not in (3, 4)
|
| 97 |
+
or (image.ndim == 4 and image.shape[0] != 1)):
|
| 98 |
+
raise ValueError("Provide one RGB/RGBA tensor with shape [C, H, W] or [1, C, H, W].")
|
| 99 |
elif not isinstance(image, (torch.Tensor, PIL.Image.Image)):
|
| 100 |
raise ValueError("Input image must be a `torch.Tensor` or `PIL.Image.Image`.")
|
| 101 |
|
|
|
|
| 116 |
Returns:
|
| 117 |
`List[PIL.Image.Image]`: List of `PIL.Image.Image` objects representing the preprocessed image.
|
| 118 |
"""
|
| 119 |
+
if not np.isfinite(foreground_ratio) or not 0 < foreground_ratio <= 1:
|
| 120 |
+
raise ValueError("Foreground ratio must be finite and in (0, 1].")
|
| 121 |
preprocessed_images = []
|
| 122 |
for i in range(len(images_pil)):
|
| 123 |
+
image = ImageOps.exif_transpose(images_pil[i]).convert("RGBA")
|
| 124 |
width, height, size = image.width, image.height, image.size
|
| 125 |
do_remove = True
|
| 126 |
if image.mode == "RGBA" and image.getextrema()[3][0] < 255:
|
|
|
|
| 135 |
# calculate the min bbox of the image
|
| 136 |
alpha = image.split()[-1]
|
| 137 |
bboxs = alpha.getbbox()
|
| 138 |
+
if bboxs is None:
|
| 139 |
+
raise ValueError("No foreground remains in the image; use an image with a visible object.")
|
| 140 |
x1, y1, x2, y2 = bboxs
|
| 141 |
dy, dx = y2 - y1, x2 - x1
|
| 142 |
s = min(height * foreground_ratio / dy, width * foreground_ratio / dx)
|
| 143 |
+
Ht, Wt = max(1, int(dy * s)), max(1, int(dx * s))
|
| 144 |
|
| 145 |
background = PIL.Image.new("RGBA", image.size, (*background_color, 255))
|
| 146 |
image = PIL.Image.alpha_composite(background, image)
|
|
|
|
| 237 |
[`~MeshPipelineOutput`] or `tuple`: [`~MeshPipelineOutput`] if `return_dict` is True, otherwise a `tuple`.
|
| 238 |
When returning a tuple, the first element is a list with the generated meshes.
|
| 239 |
"""
|
| 240 |
+
if seed is not None:
|
| 241 |
+
torch.manual_seed(seed)
|
| 242 |
# 0. Check inputs. Raise error if not correct
|
| 243 |
self.check_inputs(
|
| 244 |
image=image,
|
|
|
|
| 246 |
|
| 247 |
# 1. Define call parameters
|
| 248 |
if isinstance(image, torch.Tensor):
|
| 249 |
+
if image.ndim == 3:
|
| 250 |
+
image = image.unsqueeze(0)
|
| 251 |
batch_size = image.shape[0]
|
| 252 |
elif isinstance(image, PIL.Image.Image) or isinstance(image, str):
|
| 253 |
batch_size = 1
|
diffusionGS/systems/diffusion_gs_system_scene.py
CHANGED
|
@@ -18,6 +18,7 @@ from einops import repeat, rearrange
|
|
| 18 |
from diffusionGS.utils.misc import get_rank
|
| 19 |
from diffusionGS.systems.base import BaseSystem
|
| 20 |
from diffusionGS.models.diffusion import create_diffusion
|
|
|
|
| 21 |
from diffusionGS.utils.typing import *
|
| 22 |
from diffusionGS.utils.losses import LossComputer,MetricComputer
|
| 23 |
from diffusionGS.systems.utils import *
|
|
@@ -220,20 +221,28 @@ class PointDiffusionSystem(BaseSystem):
|
|
| 220 |
|
| 221 |
if self.cfg.save_result_for_eval:
|
| 222 |
for i in range(len(batch['uid'])):
|
| 223 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 224 |
result_pkg = {
|
| 225 |
-
'render_images':
|
| 226 |
-
'image':
|
|
|
|
| 227 |
}
|
| 228 |
self.save_torch(f"it{self.true_global_step}/{batch['uid'][i]}.pt", result_pkg)
|
| 229 |
-
psnr = 0.0
|
| 230 |
-
ssim = 0.0
|
| 231 |
-
lpips = 0.0
|
| 232 |
-
self.log("val/psnr", psnr, sync_dist=True)
|
| 233 |
-
self.log("val/ssim", ssim, sync_dist=True)
|
| 234 |
-
self.log("val/lpips", lpips, sync_dist=True)
|
| 235 |
torch.cuda.empty_cache()
|
| 236 |
-
|
|
|
|
| 237 |
|
| 238 |
def on_validation_epoch_end(self):
|
| 239 |
-
pass
|
|
|
|
| 18 |
from diffusionGS.utils.misc import get_rank
|
| 19 |
from diffusionGS.systems.base import BaseSystem
|
| 20 |
from diffusionGS.models.diffusion import create_diffusion
|
| 21 |
+
from diffusionGS.models.gsrenderer.gs_core import render_opencv_cam
|
| 22 |
from diffusionGS.utils.typing import *
|
| 23 |
from diffusionGS.utils.losses import LossComputer,MetricComputer
|
| 24 |
from diffusionGS.systems.utils import *
|
|
|
|
| 221 |
|
| 222 |
if self.cfg.save_result_for_eval:
|
| 223 |
for i in range(len(batch['uid'])):
|
| 224 |
+
# Score every specified target view, excluding the conditioning image.
|
| 225 |
+
targets = batch['rgbs'][i, 1:]
|
| 226 |
+
cameras = batch['c2ws'][i, 1:]
|
| 227 |
+
intrinsics = batch['fxfycxcys'][i, 1:]
|
| 228 |
+
gaussians = final_out['denoiser_output_dict']['pred_gaussians'][i]
|
| 229 |
+
# The native rasterizer requires float32 camera matrices, even
|
| 230 |
+
# while Lightning runs the denoiser with mixed precision.
|
| 231 |
+
with torch.autocast(device_type="cuda", enabled=False):
|
| 232 |
+
predictions = torch.stack([
|
| 233 |
+
render_opencv_cam(gaussians, targets.shape[-2], targets.shape[-1],
|
| 234 |
+
camera.float(), intrinsic.float())['render']
|
| 235 |
+
for camera, intrinsic in zip(cameras, intrinsics)
|
| 236 |
+
])
|
| 237 |
result_pkg = {
|
| 238 |
+
'render_images': predictions.detach().cpu(),
|
| 239 |
+
'image': targets.detach().cpu(),
|
| 240 |
+
'target_indices': batch['image_indices'][i, 1:].detach().cpu(),
|
| 241 |
}
|
| 242 |
self.save_torch(f"it{self.true_global_step}/{batch['uid'][i]}.pt", result_pkg)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
torch.cuda.empty_cache()
|
| 244 |
+
# Metrics are computed from the saved target views by eval_scene_result.py.
|
| 245 |
+
return {}
|
| 246 |
|
| 247 |
def on_validation_epoch_end(self):
|
| 248 |
+
pass
|
diffusionGS/utils/callbacks.py
CHANGED
|
@@ -29,13 +29,13 @@ class EarlyEnvironmentSetter(Callback):
|
|
| 29 |
|
| 30 |
def setup(self, trainer, pl_module, stage):
|
| 31 |
if not self.rank_set:
|
| 32 |
-
world_size = trainer.
|
| 33 |
local_rank = trainer.strategy.local_rank
|
| 34 |
|
| 35 |
os.environ['WORLD_SIZE'] = str(world_size)
|
| 36 |
-
os.environ['LOCAL_WORLD_SIZE'] = str(
|
| 37 |
os.environ['LOCAL_RANK'] = str(local_rank)
|
| 38 |
-
os.environ['RANK'] = str(
|
| 39 |
|
| 40 |
self.rank_set = True
|
| 41 |
|
|
|
|
| 29 |
|
| 30 |
def setup(self, trainer, pl_module, stage):
|
| 31 |
if not self.rank_set:
|
| 32 |
+
world_size = trainer.world_size
|
| 33 |
local_rank = trainer.strategy.local_rank
|
| 34 |
|
| 35 |
os.environ['WORLD_SIZE'] = str(world_size)
|
| 36 |
+
os.environ['LOCAL_WORLD_SIZE'] = str(trainer.num_devices)
|
| 37 |
os.environ['LOCAL_RANK'] = str(local_rank)
|
| 38 |
+
os.environ['RANK'] = str(trainer.global_rank)
|
| 39 |
|
| 40 |
self.rank_set = True
|
| 41 |
|
diffusionGS/utils/checkpoint.py
CHANGED
|
@@ -6,6 +6,18 @@ Adapted from: https://github.com/openai/guided-diffusion/blob/22e0df8183507e13a7
|
|
| 6 |
import torch
|
| 7 |
from diffusionGS.utils.typing import *
|
| 8 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
def checkpoint(
|
| 10 |
func: Callable[..., Union[torch.Tensor, Sequence[torch.Tensor]]],
|
| 11 |
inputs: Sequence[torch.Tensor],
|
|
@@ -65,4 +77,4 @@ class CheckpointFunction(torch.autograd.Function):
|
|
| 65 |
del ctx.input_tensors
|
| 66 |
del ctx.input_params
|
| 67 |
del output_tensors
|
| 68 |
-
return (None, None) + input_grads
|
|
|
|
| 6 |
import torch
|
| 7 |
from diffusionGS.utils.typing import *
|
| 8 |
|
| 9 |
+
|
| 10 |
+
def load_denoiser_checkpoint(path):
|
| 11 |
+
"""Load denoiser weights from raw, Lightning, or legacy training checkpoints."""
|
| 12 |
+
checkpoint = torch.load(path, map_location="cpu", weights_only=False)
|
| 13 |
+
state = checkpoint.get("state_dict", checkpoint.get("model", checkpoint))
|
| 14 |
+
state = {key.removeprefix("module."): value for key, value in state.items()}
|
| 15 |
+
for prefix in ("shape_model.", "denoiser."):
|
| 16 |
+
if any(key.startswith(prefix) for key in state):
|
| 17 |
+
return {key.removeprefix(prefix): value for key, value in state.items()
|
| 18 |
+
if key.startswith(prefix) and not key.startswith(prefix + "loss_computer.")}
|
| 19 |
+
return state
|
| 20 |
+
|
| 21 |
def checkpoint(
|
| 22 |
func: Callable[..., Union[torch.Tensor, Sequence[torch.Tensor]]],
|
| 23 |
inputs: Sequence[torch.Tensor],
|
|
|
|
| 77 |
del ctx.input_tensors
|
| 78 |
del ctx.input_params
|
| 79 |
del output_tensors
|
| 80 |
+
return (None, None) + input_grads
|
diffusionGS/utils/losses.py
CHANGED
|
@@ -288,7 +288,7 @@ class LossComputer(nn.Module):
|
|
| 288 |
if img_aligned_xyz is not None and gt_img_aligned_xyz is not None:
|
| 289 |
l2_loss_xyz = torch.ones(b, device=rendering.device) * 1e-8
|
| 290 |
per_element_loss_xyz = F.mse_loss(img_aligned_xyz * masks, gt_img_aligned_xyz * masks, reduction='sum')
|
| 291 |
-
l2_loss_xyz = per_element_loss_xyz/masks.sum()
|
| 292 |
else:
|
| 293 |
l2_loss_xyz = torch.zeros_like(l2_loss)
|
| 294 |
## depth loss
|
|
@@ -421,7 +421,7 @@ class MetricComputer(nn.Module):
|
|
| 421 |
lpips_loss_per_element = self.lpips_loss_module(
|
| 422 |
F.interpolate(predicted, size=[256,256], mode='bilinear') * 2.0 - 1.0, F.interpolate(ground_truth, size=[256,256], mode='bilinear') * 2.0 - 1.0
|
| 423 |
)
|
| 424 |
-
return lpips_loss_per_element.
|
| 425 |
|
| 426 |
|
| 427 |
|
|
@@ -470,4 +470,4 @@ class MetricComputer(nn.Module):
|
|
| 470 |
psnr = self.compute_psnr(target, rendering)
|
| 471 |
ssim = self.compute_ssim(target, rendering)
|
| 472 |
lpips = self.compute_lpips(target, rendering)
|
| 473 |
-
return psnr, ssim, lpips
|
|
|
|
| 288 |
if img_aligned_xyz is not None and gt_img_aligned_xyz is not None:
|
| 289 |
l2_loss_xyz = torch.ones(b, device=rendering.device) * 1e-8
|
| 290 |
per_element_loss_xyz = F.mse_loss(img_aligned_xyz * masks, gt_img_aligned_xyz * masks, reduction='sum')
|
| 291 |
+
l2_loss_xyz = per_element_loss_xyz / masks.sum(dtype=torch.float32).clamp_min(1e-8)
|
| 292 |
else:
|
| 293 |
l2_loss_xyz = torch.zeros_like(l2_loss)
|
| 294 |
## depth loss
|
|
|
|
| 421 |
lpips_loss_per_element = self.lpips_loss_module(
|
| 422 |
F.interpolate(predicted, size=[256,256], mode='bilinear') * 2.0 - 1.0, F.interpolate(ground_truth, size=[256,256], mode='bilinear') * 2.0 - 1.0
|
| 423 |
)
|
| 424 |
+
return lpips_loss_per_element.reshape(-1)
|
| 425 |
|
| 426 |
|
| 427 |
|
|
|
|
| 470 |
psnr = self.compute_psnr(target, rendering)
|
| 471 |
ssim = self.compute_ssim(target, rendering)
|
| 472 |
lpips = self.compute_lpips(target, rendering)
|
| 473 |
+
return psnr, ssim, lpips
|
diffusionGS/utils/mesh_preview.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Transfer Gaussian colors to extracted meshes and export portable previews."""
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
|
| 4 |
+
import numpy as np
|
| 5 |
+
from plyfile import PlyData
|
| 6 |
+
from scipy.spatial import cKDTree
|
| 7 |
+
import trimesh
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
PREVIEW_VERSION = 1
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def read_gaussian_colors(point_cloud_path):
|
| 14 |
+
"""Read world positions, display RGB colors and Gaussian opacities."""
|
| 15 |
+
points = PlyData.read(point_cloud_path)["vertex"].data
|
| 16 |
+
if not len(points):
|
| 17 |
+
raise ValueError("A nonempty Gaussian cloud is required.")
|
| 18 |
+
xyz = np.column_stack([points[axis] for axis in ("x", "y", "z")])
|
| 19 |
+
colors = np.column_stack([points[f"f_dc_{axis}"] for axis in range(3)])
|
| 20 |
+
colors = np.clip(0.5 + 0.28209479177387814 * colors, 0, 1)
|
| 21 |
+
if not np.isfinite(xyz).all() or not np.isfinite(colors).all():
|
| 22 |
+
raise ValueError("Gaussian positions and colors must be finite.")
|
| 23 |
+
opacity = np.ones(len(points), dtype=np.float64)
|
| 24 |
+
if "opacity" in points.dtype.names:
|
| 25 |
+
opacity = 1 / (1 + np.exp(-np.clip(points["opacity"].astype(np.float64), -80, 80)))
|
| 26 |
+
if not np.isfinite(opacity).all():
|
| 27 |
+
raise ValueError("Gaussian opacities must be finite.")
|
| 28 |
+
return xyz, colors, opacity
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def color_mesh_from_gaussians(mesh, point_cloud_path, neighbors=8):
|
| 32 |
+
"""Interpolate visible Gaussian DC colors at the mesh's world-space vertices."""
|
| 33 |
+
if not len(mesh.vertices):
|
| 34 |
+
raise ValueError("A nonempty mesh is required.")
|
| 35 |
+
xyz, colors, opacity = read_gaussian_colors(point_cloud_path)
|
| 36 |
+
count = min(max(1, int(neighbors)), len(xyz))
|
| 37 |
+
distances, indices = cKDTree(xyz).query(np.asarray(mesh.vertices), k=count)
|
| 38 |
+
distances = distances.reshape(len(mesh.vertices), count)
|
| 39 |
+
indices = indices.reshape(len(mesh.vertices), count)
|
| 40 |
+
weights = 1 / np.maximum(distances, 1e-6) ** 2
|
| 41 |
+
weights *= opacity[indices]
|
| 42 |
+
weights /= weights.sum(axis=1, keepdims=True)
|
| 43 |
+
interpolated = np.sum(colors[indices] * weights[..., None], axis=1)
|
| 44 |
+
rgba = np.column_stack([np.rint(interpolated * 255).astype(np.uint8),
|
| 45 |
+
np.full(len(mesh.vertices), 255, dtype=np.uint8)])
|
| 46 |
+
colored = mesh.copy()
|
| 47 |
+
colored.visual.vertex_colors = rgba
|
| 48 |
+
return colored
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _baked_color_material(tree):
|
| 52 |
+
# Gaussian colors already contain the appearance predicted by the model.
|
| 53 |
+
# Unlit glTF materials display those colors consistently without depending
|
| 54 |
+
# on the viewer having an environment map or additional scene lights.
|
| 55 |
+
tree["materials"] = [{
|
| 56 |
+
"name": "Gaussian colors",
|
| 57 |
+
"pbrMetallicRoughness": {"baseColorFactor": [1, 1, 1, 1],
|
| 58 |
+
"metallicFactor": 0, "roughnessFactor": 1},
|
| 59 |
+
"doubleSided": True,
|
| 60 |
+
"extensions": {"KHR_materials_unlit": {}},
|
| 61 |
+
}]
|
| 62 |
+
extensions = tree.setdefault("extensionsUsed", [])
|
| 63 |
+
if "KHR_materials_unlit" not in extensions:
|
| 64 |
+
extensions.append("KHR_materials_unlit")
|
| 65 |
+
for item in tree["meshes"]:
|
| 66 |
+
for primitive in item["primitives"]:
|
| 67 |
+
primitive["material"] = 0
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def export_colored_mesh(mesh, point_cloud_path, obj_path, glb_path):
|
| 71 |
+
"""Save a colored Z-up OBJ and a self-contained, Y-up GLB for the viewer."""
|
| 72 |
+
colored = color_mesh_from_gaussians(mesh, point_cloud_path)
|
| 73 |
+
obj_path, glb_path = Path(obj_path), Path(glb_path)
|
| 74 |
+
obj_data = trimesh.exchange.obj.export_obj(colored, include_normals=True, include_color=True)
|
| 75 |
+
preview = colored.copy()
|
| 76 |
+
preview.apply_transform(trimesh.transformations.rotation_matrix(-np.pi / 2, [1, 0, 0]))
|
| 77 |
+
# glTF vertex colors are linear; the model's display RGB values are sRGB.
|
| 78 |
+
rgb = preview.visual.vertex_colors[:, :3].astype(np.float64) / 255
|
| 79 |
+
linear = np.where(rgb <= 0.04045, rgb / 12.92, ((rgb + 0.055) / 1.055) ** 2.4)
|
| 80 |
+
preview.visual.vertex_colors = np.column_stack([
|
| 81 |
+
np.rint(linear * 255).astype(np.uint8), np.full(len(rgb), 255, dtype=np.uint8),
|
| 82 |
+
])
|
| 83 |
+
glb_data = trimesh.exchange.gltf.export_glb(
|
| 84 |
+
trimesh.Scene(preview), include_normals=True, tree_postprocessor=_baked_color_material,
|
| 85 |
+
)
|
| 86 |
+
# Publish complete files, including when recoloring previously extracted meshes.
|
| 87 |
+
obj_temp = obj_path.with_name(obj_path.name + ".tmp")
|
| 88 |
+
glb_temp = glb_path.with_name(glb_path.name + ".tmp")
|
| 89 |
+
obj_temp.write_text(obj_data)
|
| 90 |
+
glb_temp.write_bytes(glb_data)
|
| 91 |
+
obj_temp.replace(obj_path)
|
| 92 |
+
glb_temp.replace(glb_path)
|
| 93 |
+
return colored
|
diffusionGS/utils/misc.py
CHANGED
|
@@ -16,8 +16,7 @@ def parse_version(ver: str):
|
|
| 16 |
|
| 17 |
|
| 18 |
def get_rank():
|
| 19 |
-
#
|
| 20 |
-
# therefore LOCAL_RANK needs to be checked first
|
| 21 |
rank_keys = ("RANK", "LOCAL_RANK", "SLURM_PROCID", "JSM_NAMESPACE_RANK")
|
| 22 |
for key in rank_keys:
|
| 23 |
rank = os.environ.get(key)
|
|
@@ -34,7 +33,8 @@ def get_world_size():
|
|
| 34 |
return 1
|
| 35 |
|
| 36 |
def get_device():
|
| 37 |
-
|
|
|
|
| 38 |
|
| 39 |
|
| 40 |
def load_module_weights(
|
|
@@ -165,4 +165,4 @@ def all_gather_batch(tensors):
|
|
| 165 |
output_tensor.append(torch.cat(tensor_all, dim=0))
|
| 166 |
if not is_list:
|
| 167 |
return output_tensor[0]
|
| 168 |
-
return output_tensor
|
|
|
|
| 16 |
|
| 17 |
|
| 18 |
def get_rank():
|
| 19 |
+
# Prefer torchrun's global rank; local device selection is handled separately.
|
|
|
|
| 20 |
rank_keys = ("RANK", "LOCAL_RANK", "SLURM_PROCID", "JSM_NAMESPACE_RANK")
|
| 21 |
for key in rank_keys:
|
| 22 |
rank = os.environ.get(key)
|
|
|
|
| 33 |
return 1
|
| 34 |
|
| 35 |
def get_device():
|
| 36 |
+
local_rank = int(os.environ.get("LOCAL_RANK", get_rank()))
|
| 37 |
+
return torch.device(f"cuda:{local_rank}")
|
| 38 |
|
| 39 |
|
| 40 |
def load_module_weights(
|
|
|
|
| 165 |
output_tensor.append(torch.cat(tensor_all, dim=0))
|
| 166 |
if not is_list:
|
| 167 |
return output_tensor[0]
|
| 168 |
+
return output_tensor
|
img/demos/object_demo.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
img/demos/online_demos.png
ADDED
|
Git LFS Details
|
object_demo.py
CHANGED
|
@@ -104,7 +104,7 @@ class ObjectDemo:
|
|
| 104 |
}, indent=2))
|
| 105 |
progress(1, desc="Object ready")
|
| 106 |
return (views, point_cloud_figure(xyz, colors), str(directory / "test.ply"),
|
| 107 |
-
str(directory / "test.png"), None, None, str(directory),
|
| 108 |
f"**Done.** {len(xyz):,} Gaussians · {elapsed:.1f}s. Mesh export is available below.",
|
| 109 |
"Mesh has not been extracted for this object.")
|
| 110 |
|
|
@@ -134,27 +134,50 @@ class ObjectDemo:
|
|
| 134 |
report.update(mesh_exported=True, mesh_seconds=time.perf_counter() - start,
|
| 135 |
mesh_vertices=len(mesh.vertices), mesh_faces=len(mesh.faces))
|
| 136 |
(directory / "result.json").write_text(json.dumps(report, indent=2))
|
| 137 |
-
|
| 138 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 139 |
import trimesh
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
progress(1, desc="Mesh ready")
|
| 146 |
-
return str(preview_path), str(destination),
|
|
|
|
| 147 |
|
| 148 |
|
| 149 |
-
def build_demo(model_dir="CaiYuanhao/DiffusionGS", output_dir="outputs/object_demo", device="cuda:0", *,
|
|
|
|
| 150 |
service = service if service is not None else ObjectDemo(model_dir, output_dir, device)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
with gr.Blocks(title="DiffusionGS · Object Generation", theme=gr.themes.Soft()) as demo:
|
| 152 |
-
result_directory = gr.State()
|
| 153 |
gr.Markdown("# DiffusionGS · Object Generation\n"
|
| 154 |
"Generate a 3D object from one image, then optionally extract its mesh.")
|
| 155 |
with gr.Row():
|
| 156 |
with gr.Column(scale=1):
|
| 157 |
-
image = gr.Image(value=str(OBJECT_EXAMPLES[0]),
|
|
|
|
| 158 |
label="Object image", height=300, elem_id="object-input")
|
| 159 |
gr.Examples([[str(path)] for path in OBJECT_EXAMPLES], inputs=[image],
|
| 160 |
examples_per_page=7, label="Object examples · click to select",
|
|
@@ -166,33 +189,47 @@ def build_demo(model_dir="CaiYuanhao/DiffusionGS", output_dir="outputs/object_de
|
|
| 166 |
gr.Markdown("Background removal runs automatically for opaque images. "
|
| 167 |
"Transparent images can use their existing alpha mask.")
|
| 168 |
with gr.Column(scale=2):
|
| 169 |
-
views = gr.Gallery(label="Rendered object views", columns=4, height=270)
|
| 170 |
-
cloud = gr.Plot(label="3D point cloud · drag to rotate / scroll to zoom")
|
| 171 |
gr.Markdown("The preview shows up to 5,000 points; the PLY download contains the full object.")
|
| 172 |
-
status = gr.Markdown("
|
|
|
|
| 173 |
with gr.Row():
|
| 174 |
-
ply = gr.DownloadButton("Download point cloud (.ply)",
|
| 175 |
-
|
| 176 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 177 |
gr.Markdown("Mesh extraction is optional and can take several minutes. "
|
| 178 |
"It reuses the generated object without another diffusion pass.")
|
| 179 |
-
mesh_status = gr.Markdown()
|
| 180 |
-
mesh_preview = gr.Model3D(
|
| 181 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 182 |
generation = generate.click(
|
| 183 |
service.generate, [image, seed, ratio, force],
|
| 184 |
-
[views, cloud, ply, png, mesh_preview, obj, result_directory, status, mesh_status],
|
| 185 |
api_name="generate_object", concurrency_id="object_gpu", concurrency_limit=1,
|
| 186 |
)
|
| 187 |
generation.success(
|
| 188 |
lambda: (gr.update(interactive=True), gr.update(interactive=True),
|
| 189 |
-
gr.update(interactive=True), gr.update(interactive=False)
|
| 190 |
-
|
|
|
|
| 191 |
)
|
| 192 |
extraction = mesh_button.click(service.extract_mesh, [result_directory],
|
| 193 |
-
[mesh_preview, obj, mesh_status], api_name="extract_mesh",
|
| 194 |
concurrency_id="object_gpu", concurrency_limit=1)
|
| 195 |
-
extraction.success(lambda: gr.update(interactive=True),
|
|
|
|
| 196 |
return demo.queue(max_size=8, default_concurrency_limit=1)
|
| 197 |
|
| 198 |
|
|
@@ -204,8 +241,9 @@ def main():
|
|
| 204 |
parser.add_argument("--server-name", default="127.0.0.1")
|
| 205 |
parser.add_argument("--server-port", type=int, default=7863)
|
| 206 |
parser.add_argument("--share", action="store_true")
|
|
|
|
| 207 |
args = parser.parse_args()
|
| 208 |
-
demo = build_demo(args.model_dir, args.output_dir, args.device)
|
| 209 |
demo.launch(server_name=args.server_name, server_port=args.server_port,
|
| 210 |
share=args.share, show_error=True, allowed_paths=[str(Path(args.output_dir).resolve())])
|
| 211 |
|
|
|
|
| 104 |
}, indent=2))
|
| 105 |
progress(1, desc="Object ready")
|
| 106 |
return (views, point_cloud_figure(xyz, colors), str(directory / "test.ply"),
|
| 107 |
+
str(directory / "test.png"), None, None, None, str(directory),
|
| 108 |
f"**Done.** {len(xyz):,} Gaussians · {elapsed:.1f}s. Mesh export is available below.",
|
| 109 |
"Mesh has not been extracted for this object.")
|
| 110 |
|
|
|
|
| 134 |
report.update(mesh_exported=True, mesh_seconds=time.perf_counter() - start,
|
| 135 |
mesh_vertices=len(mesh.vertices), mesh_faces=len(mesh.faces))
|
| 136 |
(directory / "result.json").write_text(json.dumps(report, indent=2))
|
| 137 |
+
from diffusionGS.utils.mesh_preview import export_colored_mesh, PREVIEW_VERSION
|
| 138 |
+
preview_path = directory / "mesh_preview.glb"
|
| 139 |
+
report = json.loads((directory / "result.json").read_text())
|
| 140 |
+
if (not preview_path.is_file()
|
| 141 |
+
or report.get("mesh_preview_version") != PREVIEW_VERSION
|
| 142 |
+
or preview_path.stat().st_mtime < max(destination.stat().st_mtime,
|
| 143 |
+
(directory / "test.ply").stat().st_mtime)):
|
| 144 |
import trimesh
|
| 145 |
+
progress(0.95, desc="Adding mesh colors")
|
| 146 |
+
mesh = trimesh.load(destination, force="mesh", process=False)
|
| 147 |
+
export_colored_mesh(mesh, directory / "test.ply", destination, preview_path)
|
| 148 |
+
report.update(mesh_preview_version=PREVIEW_VERSION, colored_mesh=True)
|
| 149 |
+
(directory / "result.json").write_text(json.dumps(report, indent=2))
|
| 150 |
progress(1, desc="Mesh ready")
|
| 151 |
+
return (str(preview_path), str(destination), str(preview_path),
|
| 152 |
+
"**Mesh ready.** Rotate the color preview or download the GLB / OBJ below.")
|
| 153 |
|
| 154 |
|
| 155 |
+
def build_demo(model_dir="CaiYuanhao/DiffusionGS", output_dir="outputs/object_demo", device="cuda:0", *,
|
| 156 |
+
service=None, resume_result=None):
|
| 157 |
service = service if service is not None else ObjectDemo(model_dir, output_dir, device)
|
| 158 |
+
restored = None
|
| 159 |
+
gallery_value, cloud_value = None, None
|
| 160 |
+
if resume_result is not None:
|
| 161 |
+
directory = Path(resume_result).resolve()
|
| 162 |
+
# Restoring is explicit for a local launch, never shared between Space visitors.
|
| 163 |
+
if not directory.is_relative_to(service.output_dir) or not (directory / "input.png").is_file():
|
| 164 |
+
raise ValueError("The restored result must be inside the demo output directory.")
|
| 165 |
+
restored = service.extract_mesh(str(directory))
|
| 166 |
+
from diffusionGS.utils.mesh_preview import read_gaussian_colors
|
| 167 |
+
xyz, rgb, _ = read_gaussian_colors(directory / "test.ply")
|
| 168 |
+
cloud_value = point_cloud_figure(xyz, np.rint(rgb * 255).astype(np.uint8))
|
| 169 |
+
with Image.open(directory / "test.png") as grid:
|
| 170 |
+
size = grid.height - 4
|
| 171 |
+
gallery_value = [(grid.crop((2 + i * (size + 2), 2, 2 + i * (size + 2) + size, 2 + size)),
|
| 172 |
+
f"View {i + 1}") for i in range(4)]
|
| 173 |
with gr.Blocks(title="DiffusionGS · Object Generation", theme=gr.themes.Soft()) as demo:
|
| 174 |
+
result_directory = gr.State(str(directory) if restored else None)
|
| 175 |
gr.Markdown("# DiffusionGS · Object Generation\n"
|
| 176 |
"Generate a 3D object from one image, then optionally extract its mesh.")
|
| 177 |
with gr.Row():
|
| 178 |
with gr.Column(scale=1):
|
| 179 |
+
image = gr.Image(value=str(directory / "input.png") if restored else str(OBJECT_EXAMPLES[0]),
|
| 180 |
+
type="pil", image_mode="RGBA",
|
| 181 |
label="Object image", height=300, elem_id="object-input")
|
| 182 |
gr.Examples([[str(path)] for path in OBJECT_EXAMPLES], inputs=[image],
|
| 183 |
examples_per_page=7, label="Object examples · click to select",
|
|
|
|
| 189 |
gr.Markdown("Background removal runs automatically for opaque images. "
|
| 190 |
"Transparent images can use their existing alpha mask.")
|
| 191 |
with gr.Column(scale=2):
|
| 192 |
+
views = gr.Gallery(value=gallery_value, label="Rendered object views", columns=4, height=270)
|
| 193 |
+
cloud = gr.Plot(value=cloud_value, label="3D point cloud · drag to rotate / scroll to zoom")
|
| 194 |
gr.Markdown("The preview shows up to 5,000 points; the PLY download contains the full object.")
|
| 195 |
+
status = gr.Markdown("Previous object restored. Preview or download the result below."
|
| 196 |
+
if restored else "Choose an image and generate an object.")
|
| 197 |
with gr.Row():
|
| 198 |
+
ply = gr.DownloadButton("Download point cloud (.ply)",
|
| 199 |
+
value=str(directory / "test.ply") if restored else None,
|
| 200 |
+
interactive=bool(restored))
|
| 201 |
+
png = gr.DownloadButton("Download rendered views (.png)",
|
| 202 |
+
value=str(directory / "test.png") if restored else None,
|
| 203 |
+
interactive=bool(restored))
|
| 204 |
+
mesh_button = gr.Button("Extract mesh", interactive=bool(restored))
|
| 205 |
gr.Markdown("Mesh extraction is optional and can take several minutes. "
|
| 206 |
"It reuses the generated object without another diffusion pass.")
|
| 207 |
+
mesh_status = gr.Markdown(restored[3] if restored else None)
|
| 208 |
+
mesh_preview = gr.Model3D(value=restored[0] if restored else None,
|
| 209 |
+
label="Color mesh preview", height=450, interactive=False,
|
| 210 |
+
display_mode="solid", clear_color=(1, 1, 1, 1),
|
| 211 |
+
camera_position=(75, 75, None))
|
| 212 |
+
with gr.Row():
|
| 213 |
+
glb = gr.DownloadButton("Download color mesh (.glb)",
|
| 214 |
+
value=restored[2] if restored else None, interactive=bool(restored))
|
| 215 |
+
obj = gr.DownloadButton("Download mesh (.obj)",
|
| 216 |
+
value=restored[1] if restored else None, interactive=bool(restored))
|
| 217 |
generation = generate.click(
|
| 218 |
service.generate, [image, seed, ratio, force],
|
| 219 |
+
[views, cloud, ply, png, mesh_preview, obj, glb, result_directory, status, mesh_status],
|
| 220 |
api_name="generate_object", concurrency_id="object_gpu", concurrency_limit=1,
|
| 221 |
)
|
| 222 |
generation.success(
|
| 223 |
lambda: (gr.update(interactive=True), gr.update(interactive=True),
|
| 224 |
+
gr.update(interactive=True), gr.update(interactive=False),
|
| 225 |
+
gr.update(interactive=False)),
|
| 226 |
+
outputs=[ply, png, mesh_button, obj, glb], queue=False, api_name=False,
|
| 227 |
)
|
| 228 |
extraction = mesh_button.click(service.extract_mesh, [result_directory],
|
| 229 |
+
[mesh_preview, obj, glb, mesh_status], api_name="extract_mesh",
|
| 230 |
concurrency_id="object_gpu", concurrency_limit=1)
|
| 231 |
+
extraction.success(lambda: (gr.update(interactive=True), gr.update(interactive=True)),
|
| 232 |
+
outputs=[obj, glb], queue=False, api_name=False)
|
| 233 |
return demo.queue(max_size=8, default_concurrency_limit=1)
|
| 234 |
|
| 235 |
|
|
|
|
| 241 |
parser.add_argument("--server-name", default="127.0.0.1")
|
| 242 |
parser.add_argument("--server-port", type=int, default=7863)
|
| 243 |
parser.add_argument("--share", action="store_true")
|
| 244 |
+
parser.add_argument("--resume-result", help="Restore an existing local demo result without regenerating it")
|
| 245 |
args = parser.parse_args()
|
| 246 |
+
demo = build_demo(args.model_dir, args.output_dir, args.device, resume_result=args.resume_result)
|
| 247 |
demo.launch(server_name=args.server_name, server_port=args.server_port,
|
| 248 |
share=args.share, show_error=True, allowed_paths=[str(Path(args.output_dir).resolve())])
|
| 249 |
|