CaiYuanhao commited on
Commit
98e7453
·
verified ·
1 Parent(s): 871867d

Deploy DiffusionGS online generation demos

Browse files
.gitattributes CHANGED
@@ -43,3 +43,4 @@ extra_files/test_cases/popmart.png filter=lfs diff=lfs merge=lfs -text
43
  wheels/diff_gaussian_rasterization-0.0.0-cp312-cp312-linux_x86_64.whl filter=lfs diff=lfs merge=lfs -text
44
  img/demos/object_demo.png filter=lfs diff=lfs merge=lfs -text
45
  img/demos/scene_demo.png filter=lfs diff=lfs merge=lfs -text
 
 
43
  wheels/diff_gaussian_rasterization-0.0.0-cp312-cp312-linux_x86_64.whl filter=lfs diff=lfs merge=lfs -text
44
  img/demos/object_demo.png filter=lfs diff=lfs merge=lfs -text
45
  img/demos/scene_demo.png filter=lfs diff=lfs merge=lfs -text
46
+ img/demos/online_demos.png filter=lfs diff=lfs merge=lfs -text
README.md CHANGED
@@ -21,7 +21,7 @@ tags:
21
 
22
  # DiffusionGS · Object Generation
23
 
24
- Generate a 3D object from one reference image. Upload an image or choose an example, click **Generate object**, then preview and download the rendered views and Gaussian point cloud. **Extract mesh** exports an OBJ from the generated object.
25
 
26
  [Scene demo](https://huggingface.co/spaces/CaiYuanhao/DiffusionGS-Scene) · [GitHub](https://github.com/caiyuanhao1998/Open-DiffusionGS) · [Models](https://huggingface.co/CaiYuanhao/DiffusionGS) · [Paper](https://arxiv.org/abs/2411.14384)
27
 
 
21
 
22
  # DiffusionGS · Object Generation
23
 
24
+ Generate a 3D object from one reference image. Upload an image or choose an example, click **Generate object**, then preview and download the rendered views and Gaussian point cloud. **Extract mesh** creates a colored mesh preview with GLB and OBJ downloads.
25
 
26
  [Scene demo](https://huggingface.co/spaces/CaiYuanhao/DiffusionGS-Scene) · [GitHub](https://github.com/caiyuanhao1998/Open-DiffusionGS) · [Models](https://huggingface.co/CaiYuanhao/DiffusionGS) · [Paper](https://arxiv.org/abs/2411.14384)
27
 
diffusionGS/configs/diffusionGS_scene_eval.yaml CHANGED
@@ -14,7 +14,7 @@ data:
14
  sel_views_train: 4
15
  training_res: [256, 256]
16
  batch_size: 16
17
- eval_batch_size: 16
18
  num_workers: 4
19
  num_workers_val: 4
20
 
@@ -89,4 +89,4 @@ trainer:
89
  checkpoint:
90
  save_last: true
91
  save_top_k: -1
92
- every_n_train_steps: 5000
 
14
  sel_views_train: 4
15
  training_res: [256, 256]
16
  batch_size: 16
17
+ eval_batch_size: 1
18
  num_workers: 4
19
  num_workers_val: 4
20
 
 
89
  checkpoint:
90
  save_last: true
91
  save_top_k: -1
92
+ every_n_train_steps: 5000
diffusionGS/configs/diffusionGS_scene_eval_512.yaml CHANGED
@@ -14,7 +14,7 @@ data:
14
  sel_views_train: 4
15
  training_res: [512, 512]
16
  batch_size: 16
17
- eval_batch_size: 32
18
  num_workers: 4
19
  num_workers_val: 4
20
 
@@ -22,7 +22,7 @@ system_type: "diffusion-gs-scene-system"
22
  system:
23
  num_inference_steps: 30
24
  save_intermediate_video: True
25
- save_result_for_eval: False
26
 
27
 
28
  shape_model_type: "diffusion-gs-model-scene"
@@ -93,4 +93,4 @@ trainer:
93
  checkpoint:
94
  save_last: true
95
  save_top_k: -1
96
- every_n_train_steps: 5000
 
14
  sel_views_train: 4
15
  training_res: [512, 512]
16
  batch_size: 16
17
+ eval_batch_size: 1
18
  num_workers: 4
19
  num_workers_val: 4
20
 
 
22
  system:
23
  num_inference_steps: 30
24
  save_intermediate_video: True
25
+ save_result_for_eval: True
26
 
27
 
28
  shape_model_type: "diffusion-gs-model-scene"
 
93
  checkpoint:
94
  save_last: true
95
  save_top_k: -1
96
+ every_n_train_steps: 5000
diffusionGS/data/base.py CHANGED
@@ -106,6 +106,8 @@ class BaseDataset(Dataset):
106
  self.cfg: BaseDataModuleConfig = cfg
107
  self.split = split
108
  self.uids = json.load(open(f'{cfg.local_dir}/{split}.json'))
 
 
109
  self.train_idxs = get_remaining_indices(self.cfg.gen_idxs,self.cfg.all_idxs)
110
  # Default camera intrinsics [for Gobjaverse]
111
  self.fxfycxcy = torch.tensor([self.cfg.default_fxfy, self.cfg.default_fxfy, 0.5, 0.5], dtype=torch.float32)
@@ -166,7 +168,8 @@ class BaseDataset(Dataset):
166
  train_idxs = get_remaining_indices(sel_gen_idxs,self.cfg.all_idxs)
167
  sel_train_idxs = random.sample(train_idxs, k=self.cfg.sel_views) # hard code for 40 views [GOBJAVERSE]
168
  all_sel_idxs = sel_gen_idxs + sel_train_idxs
169
- image_file_dirs = [self.cfg.image_dir + f"{uid}/campos_512_v4/{str(sel_idx).zfill(5)}" for sel_idx in all_sel_idxs]
 
170
  # background color
171
  background_color = torch.as_tensor(self.cfg.background_color)
172
  rgbs = []
@@ -246,8 +249,7 @@ class BaseDataset(Dataset):
246
  try:
247
  return self._get_data(index)
248
  except Exception as e:
249
- print(f"Error in {self.uids[index]}: {e}")
250
- return self.__getitem__(np.random.randint(len(self)))
251
 
252
  def collate(self, batch):
253
  from torch.utils.data._utils.collate import default_collate_fn_map
@@ -262,4 +264,4 @@ class BaseDataset(Dataset):
262
  else:
263
  print(key)
264
  ret[key] = default_collate_fn_map[type(batch[0][key])](batch)
265
- return ret
 
106
  self.cfg: BaseDataModuleConfig = cfg
107
  self.split = split
108
  self.uids = json.load(open(f'{cfg.local_dir}/{split}.json'))
109
+ if not self.uids:
110
+ raise ValueError(f"No {split} objects listed in {cfg.local_dir}")
111
  self.train_idxs = get_remaining_indices(self.cfg.gen_idxs,self.cfg.all_idxs)
112
  # Default camera intrinsics [for Gobjaverse]
113
  self.fxfycxcy = torch.tensor([self.cfg.default_fxfy, self.cfg.default_fxfy, 0.5, 0.5], dtype=torch.float32)
 
168
  train_idxs = get_remaining_indices(sel_gen_idxs,self.cfg.all_idxs)
169
  sel_train_idxs = random.sample(train_idxs, k=self.cfg.sel_views) # hard code for 40 views [GOBJAVERSE]
170
  all_sel_idxs = sel_gen_idxs + sel_train_idxs
171
+ image_file_dirs = [os.path.join(self.cfg.image_dir, str(uid), "campos_512_v4", f"{sel_idx:05d}")
172
+ for sel_idx in all_sel_idxs]
173
  # background color
174
  background_color = torch.as_tensor(self.cfg.background_color)
175
  rgbs = []
 
249
  try:
250
  return self._get_data(index)
251
  except Exception as e:
252
+ raise RuntimeError(f"Could not load object {self.uids[index]}: {e}") from e
 
253
 
254
  def collate(self, batch):
255
  from torch.utils.data._utils.collate import default_collate_fn_map
 
264
  else:
265
  print(key)
266
  ret[key] = default_collate_fn_map[type(batch[0][key])](batch)
267
+ return ret
diffusionGS/data/base_scene.py CHANGED
@@ -71,6 +71,8 @@ class BaseDataset(Dataset):
71
  all_scene_paths = all_scene_paths[:self.cfg.eval_subset]
72
  self.uids = all_scene_paths
73
  # load anything
 
 
74
  print(f"Loaded {len(self.uids)} {split} uids")
75
  # breakpoint()
76
  def __len__(self):
@@ -149,7 +151,7 @@ class BaseDataset(Dataset):
149
 
150
  # Rescale the whole scene to a fixed scale
151
  scene_scale = torch.max(torch.abs(in_c2ws[:, :3, 3]))
152
- scene_scale = scene_scale_factor * scene_scale
153
 
154
  in_c2ws[:, :3, 3] /= scene_scale
155
 
@@ -205,8 +207,7 @@ class BaseDataset(Dataset):
205
  try:
206
  return self._get_data(index)
207
  except Exception as e:
208
- print(f"Error in {self.uids[index]}: {e}")
209
- return self.__getitem__(np.random.randint(len(self)))
210
 
211
  def collate(self, batch):
212
  from torch.utils.data._utils.collate import default_collate_fn_map
@@ -221,4 +222,4 @@ class BaseDataset(Dataset):
221
  else:
222
  print(key)
223
  ret[key] = default_collate_fn_map[type(batch[0][key])](batch)
224
- return ret
 
71
  all_scene_paths = all_scene_paths[:self.cfg.eval_subset]
72
  self.uids = all_scene_paths
73
  # load anything
74
+ if not self.uids:
75
+ raise ValueError(f"No {split} scenes found; check the scene list and evaluation index")
76
  print(f"Loaded {len(self.uids)} {split} uids")
77
  # breakpoint()
78
  def __len__(self):
 
151
 
152
  # Rescale the whole scene to a fixed scale
153
  scene_scale = torch.max(torch.abs(in_c2ws[:, :3, 3]))
154
+ scene_scale = (scene_scale_factor * scene_scale).clamp_min(1e-8)
155
 
156
  in_c2ws[:, :3, 3] /= scene_scale
157
 
 
207
  try:
208
  return self._get_data(index)
209
  except Exception as e:
210
+ raise RuntimeError(f"Could not load scene {self.uids[index]}: {e}") from e
 
211
 
212
  def collate(self, batch):
213
  from torch.utils.data._utils.collate import default_collate_fn_map
 
222
  else:
223
  print(key)
224
  ret[key] = default_collate_fn_map[type(batch[0][key])](batch)
225
+ return ret
diffusionGS/models/denoiser/denoiser.py CHANGED
@@ -9,7 +9,7 @@ from easydict import EasyDict as edict
9
  import diffusionGS
10
  from diffusionGS.models.transformers.utils_transformer import DiTBlock,_init_weights
11
  from diffusionGS.models.gsrenderer.renderer import Renderer
12
- from diffusionGS.utils.checkpoint import checkpoint
13
  from diffusionGS.utils.base import BaseModule
14
  from diffusionGS.utils.typing import *
15
  from diffusionGS.utils.ops import generate_dense_grid_points
@@ -255,31 +255,9 @@ class DGSDenoiser(BaseModule):
255
  # renderer
256
  if self.cfg.pretrained_model_name_or_path != "":
257
  print(f"Loading pretrained shape model from {self.cfg.pretrained_model_name_or_path}")
258
- pretrained_ckpt = torch.load(self.cfg.pretrained_model_name_or_path, map_location="cpu")
259
- if 'model' in pretrained_ckpt.keys():
260
- pret_weights = pretrained_ckpt['model']
261
- _pretrained_ckpt = {}
262
- # breakpoint()
263
- for k, v in pret_weights.items():
264
- if k.startswith('denoiser.') and not k.startswith('denoiser.loss_computer'):
265
- _pretrained_ckpt[k.replace('denoiser.', '')] = v
266
- #
267
- pretrained_ckpt = _pretrained_ckpt
268
-
269
- # if 'state_dict' in pretrained_ckpt:
270
- # _pretrained_ckpt = {}
271
- # for k, v in pretrained_ckpt['state_dict'].items():
272
- # if k.startswith('shape_model.'):
273
- # _pretrained_ckpt[k.replace('shape_model.', '')] = v
274
- # pretrained_ckpt = _pretrained_ckpt
275
- # else:
276
- # _pretrained_ckpt = {}
277
- # for k, v in pretrained_ckpt.items():
278
- # if k.startswith('shape_model.'):
279
- # _pretrained_ckpt[k.replace('shape_model.', '')] = v
280
- # pretrained_ckpt = _pretrained_ckpt
281
- # breakpoint()
282
- self.load_state_dict(pretrained_ckpt, strict=True)
283
 
284
  def forward(self, input_batch, timesteps):
285
  guassians_parameters,_ = self.image_to_gaussians(input_batch['image'], input_batch['ray_o'], input_batch['ray_d'], timesteps)
 
9
  import diffusionGS
10
  from diffusionGS.models.transformers.utils_transformer import DiTBlock,_init_weights
11
  from diffusionGS.models.gsrenderer.renderer import Renderer
12
+ from diffusionGS.utils.checkpoint import checkpoint, load_denoiser_checkpoint
13
  from diffusionGS.utils.base import BaseModule
14
  from diffusionGS.utils.typing import *
15
  from diffusionGS.utils.ops import generate_dense_grid_points
 
255
  # renderer
256
  if self.cfg.pretrained_model_name_or_path != "":
257
  print(f"Loading pretrained shape model from {self.cfg.pretrained_model_name_or_path}")
258
+ self.load_state_dict(
259
+ load_denoiser_checkpoint(self.cfg.pretrained_model_name_or_path), strict=True,
260
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
261
 
262
  def forward(self, input_batch, timesteps):
263
  guassians_parameters,_ = self.image_to_gaussians(input_batch['image'], input_batch['ray_o'], input_batch['ray_d'], timesteps)
diffusionGS/models/denoiser/denoiser_scene.py CHANGED
@@ -9,7 +9,7 @@ from easydict import EasyDict as edict
9
  import diffusionGS
10
  from diffusionGS.models.transformers.utils_transformer import DiTBlock,_init_weights
11
  from diffusionGS.models.gsrenderer.renderer import Renderer,SceneRenderer
12
- from diffusionGS.utils.checkpoint import checkpoint
13
  from diffusionGS.utils.base import BaseModule
14
  from diffusionGS.utils.typing import *
15
  from diffusionGS.utils.ops import generate_dense_grid_points
@@ -264,30 +264,9 @@ class DGSDenoiser(BaseModule):
264
  # renderer
265
  if self.cfg.pretrained_model_name_or_path != "":
266
  print(f"Loading pretrained shape model from {self.cfg.pretrained_model_name_or_path}")
267
- pretrained_ckpt = torch.load(self.cfg.pretrained_model_name_or_path, map_location="cpu")
268
- if 'model' in pretrained_ckpt.keys():
269
- pret_weights = pretrained_ckpt['model']
270
- _pretrained_ckpt = {}
271
- # breakpoint()
272
- for k, v in pret_weights.items():
273
- if k.startswith('denoiser.') and not k.startswith('denoiser.loss_computer'):
274
- _pretrained_ckpt[k.replace('denoiser.', '')] = v
275
- #
276
- pretrained_ckpt = _pretrained_ckpt
277
-
278
- if 'state_dict' in pretrained_ckpt:
279
- _pretrained_ckpt = {}
280
- for k, v in pretrained_ckpt['state_dict'].items():
281
- if k.startswith('shape_model.'):
282
- _pretrained_ckpt[k.replace('shape_model.', '')] = v
283
- pretrained_ckpt = _pretrained_ckpt
284
- else:
285
- _pretrained_ckpt = {}
286
- for k, v in pretrained_ckpt.items():
287
- if k.startswith('shape_model.'):
288
- _pretrained_ckpt[k.replace('shape_model.', '')] = v
289
- pretrained_ckpt = _pretrained_ckpt
290
- self.load_state_dict(pretrained_ckpt, strict=True)
291
 
292
  def forward(self, input_batch, timesteps):
293
  guassians_parameters,_ = self.image_to_gaussians(input_batch['image'], input_batch['ray_o'], input_batch['ray_d'], timesteps)
 
9
  import diffusionGS
10
  from diffusionGS.models.transformers.utils_transformer import DiTBlock,_init_weights
11
  from diffusionGS.models.gsrenderer.renderer import Renderer,SceneRenderer
12
+ from diffusionGS.utils.checkpoint import checkpoint, load_denoiser_checkpoint
13
  from diffusionGS.utils.base import BaseModule
14
  from diffusionGS.utils.typing import *
15
  from diffusionGS.utils.ops import generate_dense_grid_points
 
264
  # renderer
265
  if self.cfg.pretrained_model_name_or_path != "":
266
  print(f"Loading pretrained shape model from {self.cfg.pretrained_model_name_or_path}")
267
+ self.load_state_dict(
268
+ load_denoiser_checkpoint(self.cfg.pretrained_model_name_or_path), strict=True,
269
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
270
 
271
  def forward(self, input_batch, timesteps):
272
  guassians_parameters,_ = self.image_to_gaussians(input_batch['image'], input_batch['ray_o'], input_batch['ray_d'], timesteps)
diffusionGS/models/denoiser/denoiser_utils.py CHANGED
@@ -39,8 +39,8 @@ class SplitData(nn.Module):
39
  for key, value in data_batch.items():
40
  try:
41
  input[key] = value[:, : self.config.training.num_input_views, ...] # 取前 num_input_views 个视图, 在应用中 num_input_views = 4
42
- except:
43
- breakpoint()
44
  if self.config.training.num_target_views >= value.size(1): # 如果 target 的数量大于等于输入的数量, 4 < 8, 这一步不执行
45
  target[key] = value
46
  else:
@@ -689,4 +689,4 @@ class TransformMeshTarget(nn.Module):
689
  ray_o=ray_o,
690
  ray_d=ray_d,
691
  )
692
- return input
 
39
  for key, value in data_batch.items():
40
  try:
41
  input[key] = value[:, : self.config.training.num_input_views, ...] # 取前 num_input_views 个视图, 在应用中 num_input_views = 4
42
+ except Exception as exc:
43
+ raise ValueError(f"Batch field {key!r} must have batch and view dimensions") from exc
44
  if self.config.training.num_target_views >= value.size(1): # 如果 target 的数量大于等于输入的数量, 4 < 8, 这一步不执行
45
  target[key] = value
46
  else:
 
689
  ray_o=ray_o,
690
  ray_d=ray_d,
691
  )
692
+ return input
diffusionGS/models/gsrenderer/gs_core.py CHANGED
@@ -784,9 +784,12 @@ class GaussianModel:
784
  self._rotation = torch.from_numpy(rots.astype(np.float32)).contiguous()
785
 
786
  def extract_fields(self, resolution=128, num_blocks=16, relax_ratio=1.5):
 
 
 
 
787
  # resolution: resolution of field
788
  block_size = 2 / num_blocks
789
- assert resolution % block_size == 0
790
  split_size = resolution // num_blocks
791
  ## recenter
792
  xyzs = self.get_xyz
@@ -810,22 +813,34 @@ class GaussianModel:
810
  Y = torch.linspace(-1, 1, resolution).split(split_size)
811
  Z = torch.linspace(-1, 1, resolution).split(split_size)
812
 
 
 
 
 
 
813
 
814
  # loop blocks (assume max size of gaussian is small than relax_ratio * block_size !!!)
815
  for xi, xs in enumerate(X):
816
  for yi, ys in enumerate(Y):
817
  for zi, zs in enumerate(Z):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
818
  xx, yy, zz = torch.meshgrid(xs, ys, zs)
819
  # sample points [M, 3]
820
  pts = torch.cat([xx.reshape(-1, 1), yy.reshape(-1, 1), zz.reshape(-1, 1)], dim=-1).to(device)
821
- # in-tile gaussians mask
822
- vmin, vmax = pts.amin(0), pts.amax(0)
823
- vmin -= block_size * relax_ratio
824
- vmax += block_size * relax_ratio
825
- mask = (xyzs < vmax).all(-1) & (xyzs > vmin).all(-1)
826
- # if hit no gaussian, continue to next block
827
- if not mask.any():
828
- continue
829
  mask_xyzs = xyzs[mask] # [L, 3]
830
  mask_covs = covs[mask] # [L, 6]
831
  mask_opas = opacities[mask].view(1, -1) # [L, 1] --> [1, L]
@@ -861,6 +876,9 @@ class GaussianModel:
861
  vertices, triangles = clean_mesh(vertices, triangles, remesh=True, remesh_size=0.015)
862
  if decimate_target > 0 and triangles.shape[0] > decimate_target:
863
  vertices, triangles = decimate_mesh(vertices, triangles, decimate_target)
 
 
 
864
  # print(
865
  # f"[INFO] marching cubes result: {v.shape} ({v.min().item()}-{v.max().item()}), {f.shape}"
866
  # )
@@ -1379,4 +1397,4 @@ if __name__ == "__main__":
1379
  Image.fromarray(im).save(f"{out_dir}/render_{i:08d}.png")
1380
 
1381
  create_video(out_dir, f"{out_dir}/render.mp4", framerate=30)
1382
- print(f"Saved {out_dir}/render.mp4")
 
784
  self._rotation = torch.from_numpy(rots.astype(np.float32)).contiguous()
785
 
786
  def extract_fields(self, resolution=128, num_blocks=16, relax_ratio=1.5):
787
+ from scipy.spatial import cKDTree
788
+
789
+ if num_blocks < 1 or resolution < num_blocks or resolution % num_blocks:
790
+ raise ValueError("Field resolution must be a positive multiple of num_blocks.")
791
  # resolution: resolution of field
792
  block_size = 2 / num_blocks
 
793
  split_size = resolution // num_blocks
794
  ## recenter
795
  xyzs = self.get_xyz
 
813
  Y = torch.linspace(-1, 1, resolution).split(split_size)
814
  Z = torch.linspace(-1, 1, resolution).split(split_size)
815
 
816
+ # Find each tile's Gaussian centers on the CPU. Scanning the full cloud
817
+ # and synchronizing mask.any() for every tile is especially slow for
818
+ # sparse clouds and shared GPUs (64**3 tiles in the default mesh).
819
+ centers = xyzs.detach().cpu().numpy()
820
+ tree = cKDTree(centers)
821
 
822
  # loop blocks (assume max size of gaussian is small than relax_ratio * block_size !!!)
823
  for xi, xs in enumerate(X):
824
  for yi, ys in enumerate(Y):
825
  for zi, zs in enumerate(Z):
826
+ lower = np.array([xs[0], ys[0], zs[0]], dtype=np.float32) - block_size * relax_ratio
827
+ upper = np.array([xs[-1], ys[-1], zs[-1]], dtype=np.float32) + block_size * relax_ratio
828
+ midpoint = (lower.astype(np.float64) + upper) / 2
829
+ radius = np.max((upper.astype(np.float64) - lower) / 2)
830
+ candidates = tree.query_ball_point(midpoint, np.nextafter(radius, np.inf), p=np.inf)
831
+ if not candidates:
832
+ continue
833
+ candidates = np.asarray(candidates, dtype=np.int64)
834
+ in_tile = ((centers[candidates] > lower) & (centers[candidates] < upper)).all(-1)
835
+ candidates = np.sort(candidates[in_tile])
836
+ if len(candidates) == 0:
837
+ continue
838
+ # Preserve the original accumulation order and strict tile
839
+ # bounds so the density field stays numerically equivalent.
840
+ mask = torch.as_tensor(candidates, device=device)
841
  xx, yy, zz = torch.meshgrid(xs, ys, zs)
842
  # sample points [M, 3]
843
  pts = torch.cat([xx.reshape(-1, 1), yy.reshape(-1, 1), zz.reshape(-1, 1)], dim=-1).to(device)
 
 
 
 
 
 
 
 
844
  mask_xyzs = xyzs[mask] # [L, 3]
845
  mask_covs = covs[mask] # [L, 6]
846
  mask_opas = opacities[mask].view(1, -1) # [L, 1] --> [1, L]
 
876
  vertices, triangles = clean_mesh(vertices, triangles, remesh=True, remesh_size=0.015)
877
  if decimate_target > 0 and triangles.shape[0] > decimate_target:
878
  vertices, triangles = decimate_mesh(vertices, triangles, decimate_target)
879
+ # Field queries use centered/scaled coordinates; export in the same
880
+ # world coordinates as the original Gaussian cloud.
881
+ vertices = vertices / self.mesh_scale + self.mesh_center.detach().cpu().numpy()
882
  # print(
883
  # f"[INFO] marching cubes result: {v.shape} ({v.min().item()}-{v.max().item()}), {f.shape}"
884
  # )
 
1397
  Image.fromarray(im).save(f"{out_dir}/render_{i:08d}.png")
1398
 
1399
  create_video(out_dir, f"{out_dir}/render.mp4", framerate=30)
1400
+ print(f"Saved {out_dir}/render.mp4")
diffusionGS/pipline_obj.py CHANGED
@@ -2,12 +2,14 @@ import os
2
  import warnings
3
  from typing import Callable, List, Optional, Union, Dict, Any
4
  import PIL.Image
 
5
  import trimesh
6
  import torch
7
  import numpy as np
8
  from huggingface_hub import hf_hub_download
9
  from diffusers.utils import BaseOutput
10
  import torch.nn.functional as F
 
11
  import diffusionGS
12
  from diffusionGS.utils.config import ExperimentConfig, load_config
13
  from diffusionGS.systems.utils import *
@@ -90,6 +92,10 @@ class DiffusionGSPipeline():
90
  """
91
  if isinstance(image, str):
92
  assert os.path.isfile(image) or image.startswith("http"), "Input image must be a valid URL or a file path."
 
 
 
 
93
  elif not isinstance(image, (torch.Tensor, PIL.Image.Image)):
94
  raise ValueError("Input image must be a `torch.Tensor` or `PIL.Image.Image`.")
95
 
@@ -110,9 +116,11 @@ class DiffusionGSPipeline():
110
  Returns:
111
  `List[PIL.Image.Image]`: List of `PIL.Image.Image` objects representing the preprocessed image.
112
  """
 
 
113
  preprocessed_images = []
114
  for i in range(len(images_pil)):
115
- image = images_pil[i]
116
  width, height, size = image.width, image.height, image.size
117
  do_remove = True
118
  if image.mode == "RGBA" and image.getextrema()[3][0] < 255:
@@ -127,10 +135,12 @@ class DiffusionGSPipeline():
127
  # calculate the min bbox of the image
128
  alpha = image.split()[-1]
129
  bboxs = alpha.getbbox()
 
 
130
  x1, y1, x2, y2 = bboxs
131
  dy, dx = y2 - y1, x2 - x1
132
  s = min(height * foreground_ratio / dy, width * foreground_ratio / dx)
133
- Ht, Wt = int(dy * s), int(dx * s)
134
 
135
  background = PIL.Image.new("RGBA", image.size, (*background_color, 255))
136
  image = PIL.Image.alpha_composite(background, image)
@@ -227,7 +237,8 @@ class DiffusionGSPipeline():
227
  [`~MeshPipelineOutput`] or `tuple`: [`~MeshPipelineOutput`] if `return_dict` is True, otherwise a `tuple`.
228
  When returning a tuple, the first element is a list with the generated meshes.
229
  """
230
- torch.manual_seed(seed)
 
231
  # 0. Check inputs. Raise error if not correct
232
  self.check_inputs(
233
  image=image,
@@ -235,6 +246,8 @@ class DiffusionGSPipeline():
235
 
236
  # 1. Define call parameters
237
  if isinstance(image, torch.Tensor):
 
 
238
  batch_size = image.shape[0]
239
  elif isinstance(image, PIL.Image.Image) or isinstance(image, str):
240
  batch_size = 1
 
2
  import warnings
3
  from typing import Callable, List, Optional, Union, Dict, Any
4
  import PIL.Image
5
+ from PIL import ImageOps
6
  import trimesh
7
  import torch
8
  import numpy as np
9
  from huggingface_hub import hf_hub_download
10
  from diffusers.utils import BaseOutput
11
  import torch.nn.functional as F
12
+ import torchvision.transforms.functional as TF
13
  import diffusionGS
14
  from diffusionGS.utils.config import ExperimentConfig, load_config
15
  from diffusionGS.systems.utils import *
 
92
  """
93
  if isinstance(image, str):
94
  assert os.path.isfile(image) or image.startswith("http"), "Input image must be a valid URL or a file path."
95
+ elif isinstance(image, torch.Tensor):
96
+ if (image.ndim not in (3, 4) or image.shape[-3] not in (3, 4)
97
+ or (image.ndim == 4 and image.shape[0] != 1)):
98
+ raise ValueError("Provide one RGB/RGBA tensor with shape [C, H, W] or [1, C, H, W].")
99
  elif not isinstance(image, (torch.Tensor, PIL.Image.Image)):
100
  raise ValueError("Input image must be a `torch.Tensor` or `PIL.Image.Image`.")
101
 
 
116
  Returns:
117
  `List[PIL.Image.Image]`: List of `PIL.Image.Image` objects representing the preprocessed image.
118
  """
119
+ if not np.isfinite(foreground_ratio) or not 0 < foreground_ratio <= 1:
120
+ raise ValueError("Foreground ratio must be finite and in (0, 1].")
121
  preprocessed_images = []
122
  for i in range(len(images_pil)):
123
+ image = ImageOps.exif_transpose(images_pil[i]).convert("RGBA")
124
  width, height, size = image.width, image.height, image.size
125
  do_remove = True
126
  if image.mode == "RGBA" and image.getextrema()[3][0] < 255:
 
135
  # calculate the min bbox of the image
136
  alpha = image.split()[-1]
137
  bboxs = alpha.getbbox()
138
+ if bboxs is None:
139
+ raise ValueError("No foreground remains in the image; use an image with a visible object.")
140
  x1, y1, x2, y2 = bboxs
141
  dy, dx = y2 - y1, x2 - x1
142
  s = min(height * foreground_ratio / dy, width * foreground_ratio / dx)
143
+ Ht, Wt = max(1, int(dy * s)), max(1, int(dx * s))
144
 
145
  background = PIL.Image.new("RGBA", image.size, (*background_color, 255))
146
  image = PIL.Image.alpha_composite(background, image)
 
237
  [`~MeshPipelineOutput`] or `tuple`: [`~MeshPipelineOutput`] if `return_dict` is True, otherwise a `tuple`.
238
  When returning a tuple, the first element is a list with the generated meshes.
239
  """
240
+ if seed is not None:
241
+ torch.manual_seed(seed)
242
  # 0. Check inputs. Raise error if not correct
243
  self.check_inputs(
244
  image=image,
 
246
 
247
  # 1. Define call parameters
248
  if isinstance(image, torch.Tensor):
249
+ if image.ndim == 3:
250
+ image = image.unsqueeze(0)
251
  batch_size = image.shape[0]
252
  elif isinstance(image, PIL.Image.Image) or isinstance(image, str):
253
  batch_size = 1
diffusionGS/systems/diffusion_gs_system_scene.py CHANGED
@@ -18,6 +18,7 @@ from einops import repeat, rearrange
18
  from diffusionGS.utils.misc import get_rank
19
  from diffusionGS.systems.base import BaseSystem
20
  from diffusionGS.models.diffusion import create_diffusion
 
21
  from diffusionGS.utils.typing import *
22
  from diffusionGS.utils.losses import LossComputer,MetricComputer
23
  from diffusionGS.systems.utils import *
@@ -220,20 +221,28 @@ class PointDiffusionSystem(BaseSystem):
220
 
221
  if self.cfg.save_result_for_eval:
222
  for i in range(len(batch['uid'])):
223
- # breakpoint()
 
 
 
 
 
 
 
 
 
 
 
 
224
  result_pkg = {
225
- 'render_images': final_out['denoiser_output_dict']['render_images'][i].detach().cpu(),
226
- 'image': batch['rgbs_input'][i].detach().cpu(),
 
227
  }
228
  self.save_torch(f"it{self.true_global_step}/{batch['uid'][i]}.pt", result_pkg)
229
- psnr = 0.0
230
- ssim = 0.0
231
- lpips = 0.0
232
- self.log("val/psnr", psnr, sync_dist=True)
233
- self.log("val/ssim", ssim, sync_dist=True)
234
- self.log("val/lpips", lpips, sync_dist=True)
235
  torch.cuda.empty_cache()
236
- return {"val/psnr": psnr, "val/ssim": ssim, "val/lpips": lpips}
 
237
 
238
  def on_validation_epoch_end(self):
239
- pass
 
18
  from diffusionGS.utils.misc import get_rank
19
  from diffusionGS.systems.base import BaseSystem
20
  from diffusionGS.models.diffusion import create_diffusion
21
+ from diffusionGS.models.gsrenderer.gs_core import render_opencv_cam
22
  from diffusionGS.utils.typing import *
23
  from diffusionGS.utils.losses import LossComputer,MetricComputer
24
  from diffusionGS.systems.utils import *
 
221
 
222
  if self.cfg.save_result_for_eval:
223
  for i in range(len(batch['uid'])):
224
+ # Score every specified target view, excluding the conditioning image.
225
+ targets = batch['rgbs'][i, 1:]
226
+ cameras = batch['c2ws'][i, 1:]
227
+ intrinsics = batch['fxfycxcys'][i, 1:]
228
+ gaussians = final_out['denoiser_output_dict']['pred_gaussians'][i]
229
+ # The native rasterizer requires float32 camera matrices, even
230
+ # while Lightning runs the denoiser with mixed precision.
231
+ with torch.autocast(device_type="cuda", enabled=False):
232
+ predictions = torch.stack([
233
+ render_opencv_cam(gaussians, targets.shape[-2], targets.shape[-1],
234
+ camera.float(), intrinsic.float())['render']
235
+ for camera, intrinsic in zip(cameras, intrinsics)
236
+ ])
237
  result_pkg = {
238
+ 'render_images': predictions.detach().cpu(),
239
+ 'image': targets.detach().cpu(),
240
+ 'target_indices': batch['image_indices'][i, 1:].detach().cpu(),
241
  }
242
  self.save_torch(f"it{self.true_global_step}/{batch['uid'][i]}.pt", result_pkg)
 
 
 
 
 
 
243
  torch.cuda.empty_cache()
244
+ # Metrics are computed from the saved target views by eval_scene_result.py.
245
+ return {}
246
 
247
  def on_validation_epoch_end(self):
248
+ pass
diffusionGS/utils/callbacks.py CHANGED
@@ -29,13 +29,13 @@ class EarlyEnvironmentSetter(Callback):
29
 
30
  def setup(self, trainer, pl_module, stage):
31
  if not self.rank_set:
32
- world_size = trainer.num_devices
33
  local_rank = trainer.strategy.local_rank
34
 
35
  os.environ['WORLD_SIZE'] = str(world_size)
36
- os.environ['LOCAL_WORLD_SIZE'] = str(world_size)
37
  os.environ['LOCAL_RANK'] = str(local_rank)
38
- os.environ['RANK'] = str(local_rank)
39
 
40
  self.rank_set = True
41
 
 
29
 
30
  def setup(self, trainer, pl_module, stage):
31
  if not self.rank_set:
32
+ world_size = trainer.world_size
33
  local_rank = trainer.strategy.local_rank
34
 
35
  os.environ['WORLD_SIZE'] = str(world_size)
36
+ os.environ['LOCAL_WORLD_SIZE'] = str(trainer.num_devices)
37
  os.environ['LOCAL_RANK'] = str(local_rank)
38
+ os.environ['RANK'] = str(trainer.global_rank)
39
 
40
  self.rank_set = True
41
 
diffusionGS/utils/checkpoint.py CHANGED
@@ -6,6 +6,18 @@ Adapted from: https://github.com/openai/guided-diffusion/blob/22e0df8183507e13a7
6
  import torch
7
  from diffusionGS.utils.typing import *
8
 
 
 
 
 
 
 
 
 
 
 
 
 
9
  def checkpoint(
10
  func: Callable[..., Union[torch.Tensor, Sequence[torch.Tensor]]],
11
  inputs: Sequence[torch.Tensor],
@@ -65,4 +77,4 @@ class CheckpointFunction(torch.autograd.Function):
65
  del ctx.input_tensors
66
  del ctx.input_params
67
  del output_tensors
68
- return (None, None) + input_grads
 
6
  import torch
7
  from diffusionGS.utils.typing import *
8
 
9
+
10
+ def load_denoiser_checkpoint(path):
11
+ """Load denoiser weights from raw, Lightning, or legacy training checkpoints."""
12
+ checkpoint = torch.load(path, map_location="cpu", weights_only=False)
13
+ state = checkpoint.get("state_dict", checkpoint.get("model", checkpoint))
14
+ state = {key.removeprefix("module."): value for key, value in state.items()}
15
+ for prefix in ("shape_model.", "denoiser."):
16
+ if any(key.startswith(prefix) for key in state):
17
+ return {key.removeprefix(prefix): value for key, value in state.items()
18
+ if key.startswith(prefix) and not key.startswith(prefix + "loss_computer.")}
19
+ return state
20
+
21
  def checkpoint(
22
  func: Callable[..., Union[torch.Tensor, Sequence[torch.Tensor]]],
23
  inputs: Sequence[torch.Tensor],
 
77
  del ctx.input_tensors
78
  del ctx.input_params
79
  del output_tensors
80
+ return (None, None) + input_grads
diffusionGS/utils/losses.py CHANGED
@@ -288,7 +288,7 @@ class LossComputer(nn.Module):
288
  if img_aligned_xyz is not None and gt_img_aligned_xyz is not None:
289
  l2_loss_xyz = torch.ones(b, device=rendering.device) * 1e-8
290
  per_element_loss_xyz = F.mse_loss(img_aligned_xyz * masks, gt_img_aligned_xyz * masks, reduction='sum')
291
- l2_loss_xyz = per_element_loss_xyz/masks.sum()
292
  else:
293
  l2_loss_xyz = torch.zeros_like(l2_loss)
294
  ## depth loss
@@ -421,7 +421,7 @@ class MetricComputer(nn.Module):
421
  lpips_loss_per_element = self.lpips_loss_module(
422
  F.interpolate(predicted, size=[256,256], mode='bilinear') * 2.0 - 1.0, F.interpolate(ground_truth, size=[256,256], mode='bilinear') * 2.0 - 1.0
423
  )
424
- return lpips_loss_per_element.squeeze()
425
 
426
 
427
 
@@ -470,4 +470,4 @@ class MetricComputer(nn.Module):
470
  psnr = self.compute_psnr(target, rendering)
471
  ssim = self.compute_ssim(target, rendering)
472
  lpips = self.compute_lpips(target, rendering)
473
- return psnr, ssim, lpips
 
288
  if img_aligned_xyz is not None and gt_img_aligned_xyz is not None:
289
  l2_loss_xyz = torch.ones(b, device=rendering.device) * 1e-8
290
  per_element_loss_xyz = F.mse_loss(img_aligned_xyz * masks, gt_img_aligned_xyz * masks, reduction='sum')
291
+ l2_loss_xyz = per_element_loss_xyz / masks.sum(dtype=torch.float32).clamp_min(1e-8)
292
  else:
293
  l2_loss_xyz = torch.zeros_like(l2_loss)
294
  ## depth loss
 
421
  lpips_loss_per_element = self.lpips_loss_module(
422
  F.interpolate(predicted, size=[256,256], mode='bilinear') * 2.0 - 1.0, F.interpolate(ground_truth, size=[256,256], mode='bilinear') * 2.0 - 1.0
423
  )
424
+ return lpips_loss_per_element.reshape(-1)
425
 
426
 
427
 
 
470
  psnr = self.compute_psnr(target, rendering)
471
  ssim = self.compute_ssim(target, rendering)
472
  lpips = self.compute_lpips(target, rendering)
473
+ return psnr, ssim, lpips
diffusionGS/utils/mesh_preview.py ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Transfer Gaussian colors to extracted meshes and export portable previews."""
2
+ from pathlib import Path
3
+
4
+ import numpy as np
5
+ from plyfile import PlyData
6
+ from scipy.spatial import cKDTree
7
+ import trimesh
8
+
9
+
10
+ PREVIEW_VERSION = 1
11
+
12
+
13
+ def read_gaussian_colors(point_cloud_path):
14
+ """Read world positions, display RGB colors and Gaussian opacities."""
15
+ points = PlyData.read(point_cloud_path)["vertex"].data
16
+ if not len(points):
17
+ raise ValueError("A nonempty Gaussian cloud is required.")
18
+ xyz = np.column_stack([points[axis] for axis in ("x", "y", "z")])
19
+ colors = np.column_stack([points[f"f_dc_{axis}"] for axis in range(3)])
20
+ colors = np.clip(0.5 + 0.28209479177387814 * colors, 0, 1)
21
+ if not np.isfinite(xyz).all() or not np.isfinite(colors).all():
22
+ raise ValueError("Gaussian positions and colors must be finite.")
23
+ opacity = np.ones(len(points), dtype=np.float64)
24
+ if "opacity" in points.dtype.names:
25
+ opacity = 1 / (1 + np.exp(-np.clip(points["opacity"].astype(np.float64), -80, 80)))
26
+ if not np.isfinite(opacity).all():
27
+ raise ValueError("Gaussian opacities must be finite.")
28
+ return xyz, colors, opacity
29
+
30
+
31
+ def color_mesh_from_gaussians(mesh, point_cloud_path, neighbors=8):
32
+ """Interpolate visible Gaussian DC colors at the mesh's world-space vertices."""
33
+ if not len(mesh.vertices):
34
+ raise ValueError("A nonempty mesh is required.")
35
+ xyz, colors, opacity = read_gaussian_colors(point_cloud_path)
36
+ count = min(max(1, int(neighbors)), len(xyz))
37
+ distances, indices = cKDTree(xyz).query(np.asarray(mesh.vertices), k=count)
38
+ distances = distances.reshape(len(mesh.vertices), count)
39
+ indices = indices.reshape(len(mesh.vertices), count)
40
+ weights = 1 / np.maximum(distances, 1e-6) ** 2
41
+ weights *= opacity[indices]
42
+ weights /= weights.sum(axis=1, keepdims=True)
43
+ interpolated = np.sum(colors[indices] * weights[..., None], axis=1)
44
+ rgba = np.column_stack([np.rint(interpolated * 255).astype(np.uint8),
45
+ np.full(len(mesh.vertices), 255, dtype=np.uint8)])
46
+ colored = mesh.copy()
47
+ colored.visual.vertex_colors = rgba
48
+ return colored
49
+
50
+
51
+ def _baked_color_material(tree):
52
+ # Gaussian colors already contain the appearance predicted by the model.
53
+ # Unlit glTF materials display those colors consistently without depending
54
+ # on the viewer having an environment map or additional scene lights.
55
+ tree["materials"] = [{
56
+ "name": "Gaussian colors",
57
+ "pbrMetallicRoughness": {"baseColorFactor": [1, 1, 1, 1],
58
+ "metallicFactor": 0, "roughnessFactor": 1},
59
+ "doubleSided": True,
60
+ "extensions": {"KHR_materials_unlit": {}},
61
+ }]
62
+ extensions = tree.setdefault("extensionsUsed", [])
63
+ if "KHR_materials_unlit" not in extensions:
64
+ extensions.append("KHR_materials_unlit")
65
+ for item in tree["meshes"]:
66
+ for primitive in item["primitives"]:
67
+ primitive["material"] = 0
68
+
69
+
70
+ def export_colored_mesh(mesh, point_cloud_path, obj_path, glb_path):
71
+ """Save a colored Z-up OBJ and a self-contained, Y-up GLB for the viewer."""
72
+ colored = color_mesh_from_gaussians(mesh, point_cloud_path)
73
+ obj_path, glb_path = Path(obj_path), Path(glb_path)
74
+ obj_data = trimesh.exchange.obj.export_obj(colored, include_normals=True, include_color=True)
75
+ preview = colored.copy()
76
+ preview.apply_transform(trimesh.transformations.rotation_matrix(-np.pi / 2, [1, 0, 0]))
77
+ # glTF vertex colors are linear; the model's display RGB values are sRGB.
78
+ rgb = preview.visual.vertex_colors[:, :3].astype(np.float64) / 255
79
+ linear = np.where(rgb <= 0.04045, rgb / 12.92, ((rgb + 0.055) / 1.055) ** 2.4)
80
+ preview.visual.vertex_colors = np.column_stack([
81
+ np.rint(linear * 255).astype(np.uint8), np.full(len(rgb), 255, dtype=np.uint8),
82
+ ])
83
+ glb_data = trimesh.exchange.gltf.export_glb(
84
+ trimesh.Scene(preview), include_normals=True, tree_postprocessor=_baked_color_material,
85
+ )
86
+ # Publish complete files, including when recoloring previously extracted meshes.
87
+ obj_temp = obj_path.with_name(obj_path.name + ".tmp")
88
+ glb_temp = glb_path.with_name(glb_path.name + ".tmp")
89
+ obj_temp.write_text(obj_data)
90
+ glb_temp.write_bytes(glb_data)
91
+ obj_temp.replace(obj_path)
92
+ glb_temp.replace(glb_path)
93
+ return colored
diffusionGS/utils/misc.py CHANGED
@@ -16,8 +16,7 @@ def parse_version(ver: str):
16
 
17
 
18
  def get_rank():
19
- # SLURM_PROCID can be set even if SLURM is not managing the multiprocessing,
20
- # therefore LOCAL_RANK needs to be checked first
21
  rank_keys = ("RANK", "LOCAL_RANK", "SLURM_PROCID", "JSM_NAMESPACE_RANK")
22
  for key in rank_keys:
23
  rank = os.environ.get(key)
@@ -34,7 +33,8 @@ def get_world_size():
34
  return 1
35
 
36
  def get_device():
37
- return torch.device(f"cuda:{get_rank()}")
 
38
 
39
 
40
  def load_module_weights(
@@ -165,4 +165,4 @@ def all_gather_batch(tensors):
165
  output_tensor.append(torch.cat(tensor_all, dim=0))
166
  if not is_list:
167
  return output_tensor[0]
168
- return output_tensor
 
16
 
17
 
18
  def get_rank():
19
+ # Prefer torchrun's global rank; local device selection is handled separately.
 
20
  rank_keys = ("RANK", "LOCAL_RANK", "SLURM_PROCID", "JSM_NAMESPACE_RANK")
21
  for key in rank_keys:
22
  rank = os.environ.get(key)
 
33
  return 1
34
 
35
  def get_device():
36
+ local_rank = int(os.environ.get("LOCAL_RANK", get_rank()))
37
+ return torch.device(f"cuda:{local_rank}")
38
 
39
 
40
  def load_module_weights(
 
165
  output_tensor.append(torch.cat(tensor_all, dim=0))
166
  if not is_list:
167
  return output_tensor[0]
168
+ return output_tensor
img/demos/object_demo.png CHANGED

Git LFS Details

  • SHA256: 0143cd8a3315c75e23a9f1303b8517655f4d53eb29b26b9eefc369d18008685c
  • Pointer size: 131 Bytes
  • Size of remote file: 317 kB

Git LFS Details

  • SHA256: 36e21736ab8576e22d9f704132c348a6cd7f9ff1ae5ed98597c71c382a5d13ba
  • Pointer size: 131 Bytes
  • Size of remote file: 596 kB
img/demos/online_demos.png ADDED

Git LFS Details

  • SHA256: 1e438619a2ba1ffbdeb8da92f4d34c07b5b6dddb445f1c41969edd8c9d6a1ed2
  • Pointer size: 131 Bytes
  • Size of remote file: 546 kB
object_demo.py CHANGED
@@ -104,7 +104,7 @@ class ObjectDemo:
104
  }, indent=2))
105
  progress(1, desc="Object ready")
106
  return (views, point_cloud_figure(xyz, colors), str(directory / "test.ply"),
107
- str(directory / "test.png"), None, None, str(directory),
108
  f"**Done.** {len(xyz):,} Gaussians · {elapsed:.1f}s. Mesh export is available below.",
109
  "Mesh has not been extracted for this object.")
110
 
@@ -134,27 +134,50 @@ class ObjectDemo:
134
  report.update(mesh_exported=True, mesh_seconds=time.perf_counter() - start,
135
  mesh_vertices=len(mesh.vertices), mesh_faces=len(mesh.faces))
136
  (directory / "result.json").write_text(json.dumps(report, indent=2))
137
- preview_path = directory / "mesh_preview.obj"
138
- if not preview_path.is_file() or preview_path.stat().st_mtime < destination.stat().st_mtime:
 
 
 
 
 
139
  import trimesh
140
- preview_mesh = trimesh.load(destination, force="mesh", process=False)
141
- # The web viewer is Y-up; retain the model's original Z-up OBJ
142
- # for download and rotate only the separate preview asset.
143
- preview_mesh.apply_transform(trimesh.transformations.rotation_matrix(-np.pi / 2, [1, 0, 0]))
144
- preview_mesh.export(preview_path)
145
  progress(1, desc="Mesh ready")
146
- return str(preview_path), str(destination), "**Mesh ready.** Preview or download the OBJ below."
 
147
 
148
 
149
- def build_demo(model_dir="CaiYuanhao/DiffusionGS", output_dir="outputs/object_demo", device="cuda:0", *, service=None):
 
150
  service = service if service is not None else ObjectDemo(model_dir, output_dir, device)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
151
  with gr.Blocks(title="DiffusionGS · Object Generation", theme=gr.themes.Soft()) as demo:
152
- result_directory = gr.State()
153
  gr.Markdown("# DiffusionGS · Object Generation\n"
154
  "Generate a 3D object from one image, then optionally extract its mesh.")
155
  with gr.Row():
156
  with gr.Column(scale=1):
157
- image = gr.Image(value=str(OBJECT_EXAMPLES[0]), type="pil", image_mode="RGBA",
 
158
  label="Object image", height=300, elem_id="object-input")
159
  gr.Examples([[str(path)] for path in OBJECT_EXAMPLES], inputs=[image],
160
  examples_per_page=7, label="Object examples · click to select",
@@ -166,33 +189,47 @@ def build_demo(model_dir="CaiYuanhao/DiffusionGS", output_dir="outputs/object_de
166
  gr.Markdown("Background removal runs automatically for opaque images. "
167
  "Transparent images can use their existing alpha mask.")
168
  with gr.Column(scale=2):
169
- views = gr.Gallery(label="Rendered object views", columns=4, height=270)
170
- cloud = gr.Plot(label="3D point cloud · drag to rotate / scroll to zoom")
171
  gr.Markdown("The preview shows up to 5,000 points; the PLY download contains the full object.")
172
- status = gr.Markdown("Choose an image and generate an object.")
 
173
  with gr.Row():
174
- ply = gr.DownloadButton("Download point cloud (.ply)", interactive=False)
175
- png = gr.DownloadButton("Download rendered views (.png)", interactive=False)
176
- mesh_button = gr.Button("Extract mesh", interactive=False)
 
 
 
 
177
  gr.Markdown("Mesh extraction is optional and can take several minutes. "
178
  "It reuses the generated object without another diffusion pass.")
179
- mesh_status = gr.Markdown()
180
- mesh_preview = gr.Model3D(label="Mesh preview", height=350, interactive=False)
181
- obj = gr.DownloadButton("Download mesh (.obj)", interactive=False)
 
 
 
 
 
 
 
182
  generation = generate.click(
183
  service.generate, [image, seed, ratio, force],
184
- [views, cloud, ply, png, mesh_preview, obj, result_directory, status, mesh_status],
185
  api_name="generate_object", concurrency_id="object_gpu", concurrency_limit=1,
186
  )
187
  generation.success(
188
  lambda: (gr.update(interactive=True), gr.update(interactive=True),
189
- gr.update(interactive=True), gr.update(interactive=False)),
190
- outputs=[ply, png, mesh_button, obj], queue=False, api_name=False,
 
191
  )
192
  extraction = mesh_button.click(service.extract_mesh, [result_directory],
193
- [mesh_preview, obj, mesh_status], api_name="extract_mesh",
194
  concurrency_id="object_gpu", concurrency_limit=1)
195
- extraction.success(lambda: gr.update(interactive=True), outputs=obj, queue=False, api_name=False)
 
196
  return demo.queue(max_size=8, default_concurrency_limit=1)
197
 
198
 
@@ -204,8 +241,9 @@ def main():
204
  parser.add_argument("--server-name", default="127.0.0.1")
205
  parser.add_argument("--server-port", type=int, default=7863)
206
  parser.add_argument("--share", action="store_true")
 
207
  args = parser.parse_args()
208
- demo = build_demo(args.model_dir, args.output_dir, args.device)
209
  demo.launch(server_name=args.server_name, server_port=args.server_port,
210
  share=args.share, show_error=True, allowed_paths=[str(Path(args.output_dir).resolve())])
211
 
 
104
  }, indent=2))
105
  progress(1, desc="Object ready")
106
  return (views, point_cloud_figure(xyz, colors), str(directory / "test.ply"),
107
+ str(directory / "test.png"), None, None, None, str(directory),
108
  f"**Done.** {len(xyz):,} Gaussians · {elapsed:.1f}s. Mesh export is available below.",
109
  "Mesh has not been extracted for this object.")
110
 
 
134
  report.update(mesh_exported=True, mesh_seconds=time.perf_counter() - start,
135
  mesh_vertices=len(mesh.vertices), mesh_faces=len(mesh.faces))
136
  (directory / "result.json").write_text(json.dumps(report, indent=2))
137
+ from diffusionGS.utils.mesh_preview import export_colored_mesh, PREVIEW_VERSION
138
+ preview_path = directory / "mesh_preview.glb"
139
+ report = json.loads((directory / "result.json").read_text())
140
+ if (not preview_path.is_file()
141
+ or report.get("mesh_preview_version") != PREVIEW_VERSION
142
+ or preview_path.stat().st_mtime < max(destination.stat().st_mtime,
143
+ (directory / "test.ply").stat().st_mtime)):
144
  import trimesh
145
+ progress(0.95, desc="Adding mesh colors")
146
+ mesh = trimesh.load(destination, force="mesh", process=False)
147
+ export_colored_mesh(mesh, directory / "test.ply", destination, preview_path)
148
+ report.update(mesh_preview_version=PREVIEW_VERSION, colored_mesh=True)
149
+ (directory / "result.json").write_text(json.dumps(report, indent=2))
150
  progress(1, desc="Mesh ready")
151
+ return (str(preview_path), str(destination), str(preview_path),
152
+ "**Mesh ready.** Rotate the color preview or download the GLB / OBJ below.")
153
 
154
 
155
+ def build_demo(model_dir="CaiYuanhao/DiffusionGS", output_dir="outputs/object_demo", device="cuda:0", *,
156
+ service=None, resume_result=None):
157
  service = service if service is not None else ObjectDemo(model_dir, output_dir, device)
158
+ restored = None
159
+ gallery_value, cloud_value = None, None
160
+ if resume_result is not None:
161
+ directory = Path(resume_result).resolve()
162
+ # Restoring is explicit for a local launch, never shared between Space visitors.
163
+ if not directory.is_relative_to(service.output_dir) or not (directory / "input.png").is_file():
164
+ raise ValueError("The restored result must be inside the demo output directory.")
165
+ restored = service.extract_mesh(str(directory))
166
+ from diffusionGS.utils.mesh_preview import read_gaussian_colors
167
+ xyz, rgb, _ = read_gaussian_colors(directory / "test.ply")
168
+ cloud_value = point_cloud_figure(xyz, np.rint(rgb * 255).astype(np.uint8))
169
+ with Image.open(directory / "test.png") as grid:
170
+ size = grid.height - 4
171
+ gallery_value = [(grid.crop((2 + i * (size + 2), 2, 2 + i * (size + 2) + size, 2 + size)),
172
+ f"View {i + 1}") for i in range(4)]
173
  with gr.Blocks(title="DiffusionGS · Object Generation", theme=gr.themes.Soft()) as demo:
174
+ result_directory = gr.State(str(directory) if restored else None)
175
  gr.Markdown("# DiffusionGS · Object Generation\n"
176
  "Generate a 3D object from one image, then optionally extract its mesh.")
177
  with gr.Row():
178
  with gr.Column(scale=1):
179
+ image = gr.Image(value=str(directory / "input.png") if restored else str(OBJECT_EXAMPLES[0]),
180
+ type="pil", image_mode="RGBA",
181
  label="Object image", height=300, elem_id="object-input")
182
  gr.Examples([[str(path)] for path in OBJECT_EXAMPLES], inputs=[image],
183
  examples_per_page=7, label="Object examples · click to select",
 
189
  gr.Markdown("Background removal runs automatically for opaque images. "
190
  "Transparent images can use their existing alpha mask.")
191
  with gr.Column(scale=2):
192
+ views = gr.Gallery(value=gallery_value, label="Rendered object views", columns=4, height=270)
193
+ cloud = gr.Plot(value=cloud_value, label="3D point cloud · drag to rotate / scroll to zoom")
194
  gr.Markdown("The preview shows up to 5,000 points; the PLY download contains the full object.")
195
+ status = gr.Markdown("Previous object restored. Preview or download the result below."
196
+ if restored else "Choose an image and generate an object.")
197
  with gr.Row():
198
+ ply = gr.DownloadButton("Download point cloud (.ply)",
199
+ value=str(directory / "test.ply") if restored else None,
200
+ interactive=bool(restored))
201
+ png = gr.DownloadButton("Download rendered views (.png)",
202
+ value=str(directory / "test.png") if restored else None,
203
+ interactive=bool(restored))
204
+ mesh_button = gr.Button("Extract mesh", interactive=bool(restored))
205
  gr.Markdown("Mesh extraction is optional and can take several minutes. "
206
  "It reuses the generated object without another diffusion pass.")
207
+ mesh_status = gr.Markdown(restored[3] if restored else None)
208
+ mesh_preview = gr.Model3D(value=restored[0] if restored else None,
209
+ label="Color mesh preview", height=450, interactive=False,
210
+ display_mode="solid", clear_color=(1, 1, 1, 1),
211
+ camera_position=(75, 75, None))
212
+ with gr.Row():
213
+ glb = gr.DownloadButton("Download color mesh (.glb)",
214
+ value=restored[2] if restored else None, interactive=bool(restored))
215
+ obj = gr.DownloadButton("Download mesh (.obj)",
216
+ value=restored[1] if restored else None, interactive=bool(restored))
217
  generation = generate.click(
218
  service.generate, [image, seed, ratio, force],
219
+ [views, cloud, ply, png, mesh_preview, obj, glb, result_directory, status, mesh_status],
220
  api_name="generate_object", concurrency_id="object_gpu", concurrency_limit=1,
221
  )
222
  generation.success(
223
  lambda: (gr.update(interactive=True), gr.update(interactive=True),
224
+ gr.update(interactive=True), gr.update(interactive=False),
225
+ gr.update(interactive=False)),
226
+ outputs=[ply, png, mesh_button, obj, glb], queue=False, api_name=False,
227
  )
228
  extraction = mesh_button.click(service.extract_mesh, [result_directory],
229
+ [mesh_preview, obj, glb, mesh_status], api_name="extract_mesh",
230
  concurrency_id="object_gpu", concurrency_limit=1)
231
+ extraction.success(lambda: (gr.update(interactive=True), gr.update(interactive=True)),
232
+ outputs=[obj, glb], queue=False, api_name=False)
233
  return demo.queue(max_size=8, default_concurrency_limit=1)
234
 
235
 
 
241
  parser.add_argument("--server-name", default="127.0.0.1")
242
  parser.add_argument("--server-port", type=int, default=7863)
243
  parser.add_argument("--share", action="store_true")
244
+ parser.add_argument("--resume-result", help="Restore an existing local demo result without regenerating it")
245
  args = parser.parse_args()
246
+ demo = build_demo(args.model_dir, args.output_dir, args.device, resume_result=args.resume_result)
247
  demo.launch(server_name=args.server_name, server_port=args.server_port,
248
  share=args.share, show_error=True, allowed_paths=[str(Path(args.output_dir).resolve())])
249