+
+
+
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+)
+
+[//]: # ( 
)
+
+[//]: # ( 
)
+
| 
| 
| 
|
+
+
+
+## ☯️ **Hunyuan3D 2.1**
+
+### Architecture
+
+Tencent Hunyuan3D-2.1 is a scalable 3D asset creation system that advances state-of-the-art 3D generation through two pivotal innovations: Fully Open-Source Framework and Physically-Based Rendering (PBR) Texture Synthesis. For the first time, the system releases full model weights and training code, enabling community developers to directly fine-tune and extend the model for diverse downstream applications. This transparency accelerates academic research and industrial deployment. Moreover, replacing the prior RGB-based texture model, the upgraded PBR pipeline leverages physics-grounded material simulation to generate textures with photorealistic light interaction (e.g., metallic reflections, subsurface scattering).
+
+
+ 
+
Welcome to Hunyuan3D!
+No mesh here.
+| {image_tag} | +{mesh_tag} | +
| {image_tag} | +{views_tag} | +{mesh_tag} | +
| {content} | +
+ """
+
+ return image_tag
diff --git a/Hunyuan3D-2.1/hy3dshape/hy3dshape/utils/visualizers/pythreejs_viewer.py b/Hunyuan3D-2.1/hy3dshape/hy3dshape/utils/visualizers/pythreejs_viewer.py
new file mode 100644
index 0000000000000000000000000000000000000000..817d2efaf74f376a419eae893996540fa643a38d
--- /dev/null
+++ b/Hunyuan3D-2.1/hy3dshape/hy3dshape/utils/visualizers/pythreejs_viewer.py
@@ -0,0 +1,549 @@
+# Hunyuan 3D is licensed under the TENCENT HUNYUAN NON-COMMERCIAL LICENSE AGREEMENT
+# except for the third-party components listed below.
+# Hunyuan 3D does not impose any additional limitations beyond what is outlined
+# in the repsective licenses of these third-party components.
+# Users must comply with all terms and conditions of original licenses of these third-party
+# components and must ensure that the usage of the third party components adheres to
+# all relevant laws and regulations.
+
+# For avoidance of doubts, Hunyuan 3D means the large language models and
+# their software and algorithms, including trained model weights, parameters (including
+# optimizer states), machine-learning model code, inference-enabling code, training-enabling code,
+# fine-tuning enabling code and other elements of the foregoing made publicly available
+# by Tencent in accordance with TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT.
+
+
+import numpy as np
+from ipywidgets import embed
+import pythreejs as p3s
+import uuid
+
+from .color_util import get_colors, gen_circle, gen_checkers
+
+
+EMBED_URL = "https://cdn.jsdelivr.net/npm/@jupyter-widgets/html-manager@1.0.1/dist/embed-amd.js"
+
+
+class PyThreeJSViewer(object):
+
+ def __init__(self, settings, render_mode="WEBSITE"):
+ self.render_mode = render_mode
+ self.__update_settings(settings)
+ self._light = p3s.DirectionalLight(color='white', position=[0, 0, 1], intensity=0.6)
+ self._light2 = p3s.AmbientLight(intensity=0.5)
+ self._cam = p3s.PerspectiveCamera(position=[0, 0, 1], lookAt=[0, 0, 0], fov=self.__s["fov"],
+ aspect=self.__s["width"] / self.__s["height"], children=[self._light])
+ self._orbit = p3s.OrbitControls(controlling=self._cam)
+ self._scene = p3s.Scene(children=[self._cam, self._light2], background=self.__s["background"]) # "#4c4c80"
+ self._renderer = p3s.Renderer(camera=self._cam, scene=self._scene, controls=[self._orbit],
+ width=self.__s["width"], height=self.__s["height"],
+ antialias=self.__s["antialias"])
+
+ self.__objects = {}
+ self.__cnt = 0
+
+ def jupyter_mode(self):
+ self.render_mode = "JUPYTER"
+
+ def offline(self):
+ self.render_mode = "OFFLINE"
+
+ def website(self):
+ self.render_mode = "WEBSITE"
+
+ def __get_shading(self, shading):
+ shad = {"flat": True, "wireframe": False, "wire_width": 0.03, "wire_color": "black",
+ "side": 'DoubleSide', "colormap": "viridis", "normalize": [None, None],
+ "bbox": False, "roughness": 0.5, "metalness": 0.25, "reflectivity": 1.0,
+ "line_width": 1.0, "line_color": "black",
+ "point_color": "red", "point_size": 0.01, "point_shape": "circle",
+ "text_color": "red"
+ }
+ for k in shading:
+ shad[k] = shading[k]
+ return shad
+
+ def __update_settings(self, settings={}):
+ sett = {"width": 1600, "height": 800, "antialias": True, "scale": 1.5, "background": "#ffffff",
+ "fov": 30}
+ for k in settings:
+ sett[k] = settings[k]
+ self.__s = sett
+
+ def __add_object(self, obj, parent=None):
+ if not parent: # Object is added to global scene and objects dict
+ self.__objects[self.__cnt] = obj
+ self.__cnt += 1
+ self._scene.add(obj["mesh"])
+ else: # Object is added to parent object and NOT to objects dict
+ parent.add(obj["mesh"])
+
+ self.__update_view()
+
+ if self.render_mode == "JUPYTER":
+ return self.__cnt - 1
+ elif self.render_mode == "WEBSITE":
+ return self
+
+ def __add_line_geometry(self, lines, shading, obj=None):
+ lines = lines.astype("float32", copy=False)
+ mi = np.min(lines, axis=0)
+ ma = np.max(lines, axis=0)
+
+ geometry = p3s.LineSegmentsGeometry(positions=lines.reshape((-1, 2, 3)))
+ material = p3s.LineMaterial(linewidth=shading["line_width"], color=shading["line_color"])
+ # , vertexColors='VertexColors'),
+ lines = p3s.LineSegments2(geometry=geometry, material=material) # type='LinePieces')
+ line_obj = {"geometry": geometry, "mesh": lines, "material": material,
+ "max": ma, "min": mi, "type": "Lines", "wireframe": None}
+
+ if obj:
+ return self.__add_object(line_obj, obj), line_obj
+ else:
+ return self.__add_object(line_obj)
+
+ def __update_view(self):
+ if len(self.__objects) == 0:
+ return
+ ma = np.zeros((len(self.__objects), 3))
+ mi = np.zeros((len(self.__objects), 3))
+ for r, obj in enumerate(self.__objects):
+ ma[r] = self.__objects[obj]["max"]
+ mi[r] = self.__objects[obj]["min"]
+ ma = np.max(ma, axis=0)
+ mi = np.min(mi, axis=0)
+ diag = np.linalg.norm(ma - mi)
+ mean = ((ma - mi) / 2 + mi).tolist()
+ scale = self.__s["scale"] * (diag)
+ self._orbit.target = mean
+ self._cam.lookAt(mean)
+ self._cam.position = [mean[0], mean[1], mean[2] + scale]
+ self._light.position = [mean[0], mean[1], mean[2] + scale]
+
+ self._orbit.exec_three_obj_method('update')
+ self._cam.exec_three_obj_method('updateProjectionMatrix')
+
+ def __get_bbox(self, v):
+ m = np.min(v, axis=0)
+ M = np.max(v, axis=0)
+
+ # Corners of the bounding box
+ v_box = np.array([[m[0], m[1], m[2]], [M[0], m[1], m[2]], [M[0], M[1], m[2]], [m[0], M[1], m[2]],
+ [m[0], m[1], M[2]], [M[0], m[1], M[2]], [M[0], M[1], M[2]], [m[0], M[1], M[2]]])
+
+ f_box = np.array([[0, 1], [1, 2], [2, 3], [3, 0], [4, 5], [5, 6], [6, 7], [7, 4],
+ [0, 4], [1, 5], [2, 6], [7, 3]], dtype=np.uint32)
+ return v_box, f_box
+
+ def __get_colors(self, v, f, c, sh):
+ coloring = "VertexColors"
+ if type(c) == np.ndarray and c.size == 3: # Single color
+ colors = np.ones_like(v)
+ colors[:, 0] = c[0]
+ colors[:, 1] = c[1]
+ colors[:, 2] = c[2]
+ # print("Single colors")
+ elif type(c) == np.ndarray and len(c.shape) == 2 and c.shape[1] == 3: # Color values for
+ if c.shape[0] == f.shape[0]: # faces
+ colors = np.hstack([c, c, c]).reshape((-1, 3))
+ coloring = "FaceColors"
+ # print("Face color values")
+ elif c.shape[0] == v.shape[0]: # vertices
+ colors = c
+ # print("Vertex color values")
+ else: # Wrong size, fallback
+ print("Invalid color array given! Supported are numpy arrays.", type(c))
+ colors = np.ones_like(v)
+ colors[:, 0] = 1.0
+ colors[:, 1] = 0.874
+ colors[:, 2] = 0.0
+ elif type(c) == np.ndarray and c.size == f.shape[0]: # Function values for faces
+ normalize = sh["normalize"][0] != None and sh["normalize"][1] != None
+ cc = get_colors(c, sh["colormap"], normalize=normalize,
+ vmin=sh["normalize"][0], vmax=sh["normalize"][1])
+ # print(cc.shape)
+ colors = np.hstack([cc, cc, cc]).reshape((-1, 3))
+ coloring = "FaceColors"
+ # print("Face function values")
+ elif type(c) == np.ndarray and c.size == v.shape[0]: # Function values for vertices
+ normalize = sh["normalize"][0] != None and sh["normalize"][1] != None
+ colors = get_colors(c, sh["colormap"], normalize=normalize,
+ vmin=sh["normalize"][0], vmax=sh["normalize"][1])
+ # print("Vertex function values")
+
+ else:
+ colors = np.ones_like(v)
+ colors[:, 0] = 1.0
+ colors[:, 1] = 0.874
+ colors[:, 2] = 0.0
+
+ # No color
+ if c is not None:
+ print("Invalid color array given! Supported are numpy arrays.", type(c))
+
+ return colors, coloring
+
+ def __get_point_colors(self, v, c, sh):
+ v_color = True
+ if c is None: # No color given, use global color
+ # conv = mpl.colors.ColorConverter()
+ colors = sh["point_color"] # np.array(conv.to_rgb(sh["point_color"]))
+ v_color = False
+ elif isinstance(c, str): # No color given, use global color
+ # conv = mpl.colors.ColorConverter()
+ colors = c # np.array(conv.to_rgb(c))
+ v_color = False
+ elif type(c) == np.ndarray and len(c.shape) == 2 and c.shape[0] == v.shape[0] and c.shape[1] == 3:
+ # Point color
+ colors = c.astype("float32", copy=False)
+
+ elif isinstance(c, np.ndarray) and len(c.shape) == 2 and c.shape[0] == v.shape[0] and c.shape[1] != 3:
+ # Function values for vertices, but the colors are features
+ c_norm = np.linalg.norm(c, ord=2, axis=-1)
+ normalize = sh["normalize"][0] != None and sh["normalize"][1] != None
+ colors = get_colors(c_norm, sh["colormap"], normalize=normalize,
+ vmin=sh["normalize"][0], vmax=sh["normalize"][1])
+ colors = colors.astype("float32", copy=False)
+
+ elif type(c) == np.ndarray and c.size == v.shape[0]: # Function color
+ normalize = sh["normalize"][0] != None and sh["normalize"][1] != None
+ colors = get_colors(c, sh["colormap"], normalize=normalize,
+ vmin=sh["normalize"][0], vmax=sh["normalize"][1])
+ colors = colors.astype("float32", copy=False)
+ # print("Vertex function values")
+
+ else:
+ print("Invalid color array given! Supported are numpy arrays.", type(c))
+ colors = sh["point_color"]
+ v_color = False
+
+ return colors, v_color
+
+ def add_mesh(self, v, f, c=None, uv=None, n=None, shading={}, texture_data=None, **kwargs):
+ shading.update(kwargs)
+ sh = self.__get_shading(shading)
+ mesh_obj = {}
+
+ # it is a tet
+ if v.shape[1] == 3 and f.shape[1] == 4:
+ f_tmp = np.ndarray([f.shape[0] * 4, 3], dtype=f.dtype)
+ for i in range(f.shape[0]):
+ f_tmp[i * 4 + 0] = np.array([f[i][1], f[i][0], f[i][2]])
+ f_tmp[i * 4 + 1] = np.array([f[i][0], f[i][1], f[i][3]])
+ f_tmp[i * 4 + 2] = np.array([f[i][1], f[i][2], f[i][3]])
+ f_tmp[i * 4 + 3] = np.array([f[i][2], f[i][0], f[i][3]])
+ f = f_tmp
+
+ if v.shape[1] == 2:
+ v = np.append(v, np.zeros([v.shape[0], 1]), 1)
+
+ # Type adjustment vertices
+ v = v.astype("float32", copy=False)
+
+ # Color setup
+ colors, coloring = self.__get_colors(v, f, c, sh)
+
+ # Type adjustment faces and colors
+ c = colors.astype("float32", copy=False)
+
+ # Material and geometry setup
+ ba_dict = {"color": p3s.BufferAttribute(c)}
+ if coloring == "FaceColors":
+ verts = np.zeros((f.shape[0] * 3, 3), dtype="float32")
+ for ii in range(f.shape[0]):
+ # print(ii*3, f[ii])
+ verts[ii * 3] = v[f[ii, 0]]
+ verts[ii * 3 + 1] = v[f[ii, 1]]
+ verts[ii * 3 + 2] = v[f[ii, 2]]
+ v = verts
+ else:
+ f = f.astype("uint32", copy=False).ravel()
+ ba_dict["index"] = p3s.BufferAttribute(f, normalized=False)
+
+ ba_dict["position"] = p3s.BufferAttribute(v, normalized=False)
+
+ if uv is not None:
+ uv = (uv - np.min(uv)) / (np.max(uv) - np.min(uv))
+ if texture_data is None:
+ texture_data = gen_checkers(20, 20)
+ tex = p3s.DataTexture(data=texture_data, format="RGBFormat", type="FloatType")
+ material = p3s.MeshStandardMaterial(map=tex, reflectivity=sh["reflectivity"], side=sh["side"],
+ roughness=sh["roughness"], metalness=sh["metalness"],
+ flatShading=sh["flat"],
+ polygonOffset=True, polygonOffsetFactor=1, polygonOffsetUnits=5)
+ ba_dict["uv"] = p3s.BufferAttribute(uv.astype("float32", copy=False))
+ else:
+ material = p3s.MeshStandardMaterial(vertexColors=coloring, reflectivity=sh["reflectivity"],
+ side=sh["side"], roughness=sh["roughness"], metalness=sh["metalness"],
+ flatShading=sh["flat"],
+ polygonOffset=True, polygonOffsetFactor=1, polygonOffsetUnits=5)
+
+ if type(n) != type(None) and coloring == "VertexColors": # TODO: properly handle normals for FaceColors as well
+ ba_dict["normal"] = p3s.BufferAttribute(n.astype("float32", copy=False), normalized=True)
+
+ geometry = p3s.BufferGeometry(attributes=ba_dict)
+
+ if coloring == "VertexColors" and type(n) == type(None):
+ geometry.exec_three_obj_method('computeVertexNormals')
+ elif coloring == "FaceColors" and type(n) == type(None):
+ geometry.exec_three_obj_method('computeFaceNormals')
+
+ # Mesh setup
+ mesh = p3s.Mesh(geometry=geometry, material=material)
+
+ # Wireframe setup
+ mesh_obj["wireframe"] = None
+ if sh["wireframe"]:
+ wf_geometry = p3s.WireframeGeometry(mesh.geometry) # WireframeGeometry
+ wf_material = p3s.LineBasicMaterial(color=sh["wire_color"], linewidth=sh["wire_width"])
+ wireframe = p3s.LineSegments(wf_geometry, wf_material)
+ mesh.add(wireframe)
+ mesh_obj["wireframe"] = wireframe
+
+ # Bounding box setup
+ if sh["bbox"]:
+ v_box, f_box = self.__get_bbox(v)
+ _, bbox = self.add_edges(v_box, f_box, sh, mesh)
+ mesh_obj["bbox"] = [bbox, v_box, f_box]
+
+ # Object setup
+ mesh_obj["max"] = np.max(v, axis=0)
+ mesh_obj["min"] = np.min(v, axis=0)
+ mesh_obj["geometry"] = geometry
+ mesh_obj["mesh"] = mesh
+ mesh_obj["material"] = material
+ mesh_obj["type"] = "Mesh"
+ mesh_obj["shading"] = sh
+ mesh_obj["coloring"] = coloring
+ mesh_obj["arrays"] = [v, f, c] # TODO replays with proper storage or remove if not needed
+
+ return self.__add_object(mesh_obj)
+
+ def add_lines(self, beginning, ending, shading={}, obj=None, **kwargs):
+ shading.update(kwargs)
+ if len(beginning.shape) == 1:
+ if len(beginning) == 2:
+ beginning = np.array([[beginning[0], beginning[1], 0]])
+ else:
+ if beginning.shape[1] == 2:
+ beginning = np.append(
+ beginning, np.zeros([beginning.shape[0], 1]), 1)
+ if len(ending.shape) == 1:
+ if len(ending) == 2:
+ ending = np.array([[ending[0], ending[1], 0]])
+ else:
+ if ending.shape[1] == 2:
+ ending = np.append(
+ ending, np.zeros([ending.shape[0], 1]), 1)
+
+ sh = self.__get_shading(shading)
+ lines = np.hstack([beginning, ending])
+ lines = lines.reshape((-1, 3))
+ return self.__add_line_geometry(lines, sh, obj)
+
+ def add_edges(self, vertices, edges, shading={}, obj=None, **kwargs):
+ shading.update(kwargs)
+ if vertices.shape[1] == 2:
+ vertices = np.append(
+ vertices, np.zeros([vertices.shape[0], 1]), 1)
+ sh = self.__get_shading(shading)
+ lines = np.zeros((edges.size, 3))
+ cnt = 0
+ for e in edges:
+ lines[cnt, :] = vertices[e[0]]
+ lines[cnt + 1, :] = vertices[e[1]]
+ cnt += 2
+ return self.__add_line_geometry(lines, sh, obj)
+
+ def add_points(self, points, c=None, shading={}, obj=None, **kwargs):
+ shading.update(kwargs)
+ if len(points.shape) == 1:
+ if len(points) == 2:
+ points = np.array([[points[0], points[1], 0]])
+ else:
+ if points.shape[1] == 2:
+ points = np.append(
+ points, np.zeros([points.shape[0], 1]), 1)
+ sh = self.__get_shading(shading)
+ points = points.astype("float32", copy=False)
+ mi = np.min(points, axis=0)
+ ma = np.max(points, axis=0)
+
+ g_attributes = {"position": p3s.BufferAttribute(points, normalized=False)}
+ m_attributes = {"size": sh["point_size"]}
+
+ if sh["point_shape"] == "circle": # Plot circles
+ tex = p3s.DataTexture(data=gen_circle(16, 16), format="RGBAFormat", type="FloatType")
+ m_attributes["map"] = tex
+ m_attributes["alphaTest"] = 0.5
+ m_attributes["transparency"] = True
+ else: # Plot squares
+ pass
+
+ colors, v_colors = self.__get_point_colors(points, c, sh)
+ if v_colors: # Colors per point
+ m_attributes["vertexColors"] = 'VertexColors'
+ g_attributes["color"] = p3s.BufferAttribute(colors, normalized=False)
+
+ else: # Colors for all points
+ m_attributes["color"] = colors
+
+ material = p3s.PointsMaterial(**m_attributes)
+ geometry = p3s.BufferGeometry(attributes=g_attributes)
+ points = p3s.Points(geometry=geometry, material=material)
+ point_obj = {"geometry": geometry, "mesh": points, "material": material,
+ "max": ma, "min": mi, "type": "Points", "wireframe": None}
+
+ if obj:
+ return self.__add_object(point_obj, obj), point_obj
+ else:
+ return self.__add_object(point_obj)
+
+ def remove_object(self, obj_id):
+ if obj_id not in self.__objects:
+ print("Invalid object id. Valid ids are: ", list(self.__objects.keys()))
+ return
+ self._scene.remove(self.__objects[obj_id]["mesh"])
+ del self.__objects[obj_id]
+ self.__update_view()
+
+ def reset(self):
+ for obj_id in list(self.__objects.keys()).copy():
+ self._scene.remove(self.__objects[obj_id]["mesh"])
+ del self.__objects[obj_id]
+ self.__update_view()
+
+ def update_object(self, oid=0, vertices=None, colors=None, faces=None):
+ obj = self.__objects[oid]
+ if type(vertices) != type(None):
+ if obj["coloring"] == "FaceColors":
+ f = obj["arrays"][1]
+ verts = np.zeros((f.shape[0] * 3, 3), dtype="float32")
+ for ii in range(f.shape[0]):
+ # print(ii*3, f[ii])
+ verts[ii * 3] = vertices[f[ii, 0]]
+ verts[ii * 3 + 1] = vertices[f[ii, 1]]
+ verts[ii * 3 + 2] = vertices[f[ii, 2]]
+ v = verts
+
+ else:
+ v = vertices.astype("float32", copy=False)
+ obj["geometry"].attributes["position"].array = v
+ # self.wireframe.attributes["position"].array = v # Wireframe updates?
+ obj["geometry"].attributes["position"].needsUpdate = True
+ # obj["geometry"].exec_three_obj_method('computeVertexNormals')
+ if type(colors) != type(None):
+ colors, coloring = self.__get_colors(obj["arrays"][0], obj["arrays"][1], colors, obj["shading"])
+ colors = colors.astype("float32", copy=False)
+ obj["geometry"].attributes["color"].array = colors
+ obj["geometry"].attributes["color"].needsUpdate = True
+ if type(faces) != type(None):
+ if obj["coloring"] == "FaceColors":
+ print("Face updates are currently only possible in vertex color mode.")
+ return
+ f = faces.astype("uint32", copy=False).ravel()
+ print(obj["geometry"].attributes)
+ obj["geometry"].attributes["index"].array = f
+ # self.wireframe.attributes["position"].array = v # Wireframe updates?
+ obj["geometry"].attributes["index"].needsUpdate = True
+ # obj["geometry"].exec_three_obj_method('computeVertexNormals')
+ # self.mesh.geometry.verticesNeedUpdate = True
+ # self.mesh.geometry.elementsNeedUpdate = True
+ # self.update()
+ if self.render_mode == "WEBSITE":
+ return self
+
+ # def update(self):
+ # self.mesh.exec_three_obj_method('update')
+ # self.orbit.exec_three_obj_method('update')
+ # self.cam.exec_three_obj_method('updateProjectionMatrix')
+ # self.scene.exec_three_obj_method('update')
+
+ def add_text(self, text, shading={}, **kwargs):
+ shading.update(kwargs)
+ sh = self.__get_shading(shading)
+ tt = p3s.TextTexture(string=text, color=sh["text_color"])
+ sm = p3s.SpriteMaterial(map=tt)
+ text = p3s.Sprite(material=sm, scaleToTexture=True)
+ self._scene.add(text)
+
+ # def add_widget(self, widget, callback):
+ # self.widgets.append(widget)
+ # widget.observe(callback, names='value')
+
+ # def add_dropdown(self, options, default, desc, cb):
+ # widget = widgets.Dropdown(options=options, value=default, description=desc)
+ # self.__widgets.append(widget)
+ # widget.observe(cb, names="value")
+ # display(widget)
+
+ # def add_button(self, text, cb):
+ # button = widgets.Button(description=text)
+ # self.__widgets.append(button)
+ # button.on_click(cb)
+ # display(button)
+
+ def to_html(self, imports=True, html_frame=True):
+ # Bake positions (fixes centering bug in offline rendering)
+ if len(self.__objects) == 0:
+ return
+ ma = np.zeros((len(self.__objects), 3))
+ mi = np.zeros((len(self.__objects), 3))
+ for r, obj in enumerate(self.__objects):
+ ma[r] = self.__objects[obj]["max"]
+ mi[r] = self.__objects[obj]["min"]
+ ma = np.max(ma, axis=0)
+ mi = np.min(mi, axis=0)
+ diag = np.linalg.norm(ma - mi)
+ mean = (ma - mi) / 2 + mi
+ for r, obj in enumerate(self.__objects):
+ v = self.__objects[obj]["geometry"].attributes["position"].array
+ v -= mean
+ # v += np.array([0.0, .9, 0.0]) #! to move the obj to the center of window
+
+ scale = self.__s["scale"] * (diag)
+ self._orbit.target = [0.0, 0.0, 0.0]
+ self._cam.lookAt([0.0, 0.0, 0.0])
+ # self._cam.position = [0.0, 0.0, scale]
+ self._cam.position = [0.0, 0.5, scale * 1.3] #! show four complete meshes in the window
+ self._light.position = [0.0, 0.0, scale]
+
+ state = embed.dependency_state(self._renderer)
+
+ # Somehow these entries are missing when the state is exported in python.
+ # Exporting from the GUI works, so we are inserting the missing entries.
+ for k in state:
+ if state[k]["model_name"] == "OrbitControlsModel":
+ state[k]["state"]["maxAzimuthAngle"] = "inf"
+ state[k]["state"]["maxDistance"] = "inf"
+ state[k]["state"]["maxZoom"] = "inf"
+ state[k]["state"]["minAzimuthAngle"] = "-inf"
+
+ tpl = embed.load_requirejs_template
+ if not imports:
+ embed.load_requirejs_template = ""
+
+ s = embed.embed_snippet(self._renderer, state=state, embed_url=EMBED_URL)
+ # s = embed.embed_snippet(self.__w, state=state)
+ embed.load_requirejs_template = tpl
+
+ if html_frame:
+ s = "\n\n" + s + "\n\n"
+
+ # Revert changes
+ for r, obj in enumerate(self.__objects):
+ v = self.__objects[obj]["geometry"].attributes["position"].array
+ v += mean
+ self.__update_view()
+
+ return s
+
+ def save(self, filename=""):
+ if filename == "":
+ uid = str(uuid.uuid4()) + ".html"
+ else:
+ filename = filename.replace(".html", "")
+ uid = filename + '.html'
+ with open(uid, "w") as f:
+ f.write(self.to_html())
+ print("Plot saved to file %s." % uid)
diff --git a/Hunyuan3D-2.1/hy3dshape/main.py b/Hunyuan3D-2.1/hy3dshape/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..4c1ae51f61e2578dad8ee93e99c2df150968cc73
--- /dev/null
+++ b/Hunyuan3D-2.1/hy3dshape/main.py
@@ -0,0 +1,209 @@
+# Hunyuan 3D is licensed under the TENCENT HUNYUAN NON-COMMERCIAL LICENSE AGREEMENT
+# except for the third-party components listed below.
+# Hunyuan 3D does not impose any additional limitations beyond what is outlined
+# in the repsective licenses of these third-party components.
+# Users must comply with all terms and conditions of original licenses of these third-party
+# components and must ensure that the usage of the third party components adheres to
+# all relevant laws and regulations.
+
+# For avoidance of doubts, Hunyuan 3D means the large language models and
+# their software and algorithms, including trained model weights, parameters (including
+# optimizer states), machine-learning model code, inference-enabling code, training-enabling code,
+# fine-tuning enabling code and other elements of the foregoing made publicly available
+# by Tencent in accordance with TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT.
+
+from typing import Tuple, List
+import warnings
+warnings.filterwarnings("ignore")
+
+import os
+import torch
+import argparse
+from pathlib import Path
+from omegaconf import OmegaConf, DictConfig
+from einops._torch_specific import allow_ops_in_compiled_graph # requires einops>=0.6.1
+allow_ops_in_compiled_graph()
+
+import pytorch_lightning as pl
+from pytorch_lightning.callbacks import ModelCheckpoint, Callback
+from pytorch_lightning.strategies import DDPStrategy, DeepSpeedStrategy
+from pytorch_lightning.loggers import Logger, TensorBoardLogger
+from pytorch_lightning.utilities import rank_zero_info
+
+from hy3dshape.utils import get_config_from_file, instantiate_from_config
+
+
+class SetupCallback(Callback):
+ def __init__(self, config: DictConfig, basedir: Path, logdir: str = "log", ckptdir: str = "ckpt") -> None:
+ super().__init__()
+ self.logdir = basedir / logdir
+ self.ckptdir = basedir / ckptdir
+ self.config = config
+
+ def on_fit_start(self, trainer: pl.trainer.Trainer, pl_module: pl.LightningModule) -> None:
+ if trainer.global_rank == 0:
+ os.makedirs(self.logdir, exist_ok=True)
+ os.makedirs(self.ckptdir, exist_ok=True)
+
+
+def setup_callbacks(config: DictConfig) -> Tuple[List[Callback], Logger]:
+ training_cfg = config.training
+ basedir = Path(training_cfg.output_dir)
+ os.makedirs(basedir, exist_ok=True)
+ all_callbacks = []
+
+ setup_callback = SetupCallback(config, basedir)
+ all_callbacks.append(setup_callback)
+
+ checkpoint_callback = ModelCheckpoint(
+ dirpath=setup_callback.ckptdir,
+ filename="ckpt-{step:08d}",
+ monitor=training_cfg.monitor,
+ mode="max",
+ save_top_k=-1,
+ verbose=False,
+ every_n_train_steps=training_cfg.every_n_train_steps)
+ all_callbacks.append(checkpoint_callback)
+
+ if "callbacks" in config:
+ for key, value in config['callbacks'].items():
+ custom_callback = instantiate_from_config(value)
+ all_callbacks.append(custom_callback)
+
+ logger = TensorBoardLogger(save_dir=str(setup_callback.logdir), name="tensorboard")
+
+ return all_callbacks, logger
+
+
+def merge_cfg(cfg, arg_cfg):
+ for key in arg_cfg.keys():
+ if key in cfg.training:
+ arg_cfg[key] = cfg.training[key]
+ cfg.training = DictConfig(arg_cfg)
+ return cfg
+
+
+def get_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--fast", action='store_true')
+ parser.add_argument("-c", "--config", type=str, required=True)
+ parser.add_argument("-s", "--seed", type=int, default=0)
+ parser.add_argument("-nn", "--num_nodes", type=int, default=1)
+ parser.add_argument("-ng", "--num_gpus", type=int, default=1)
+ parser.add_argument("-u", "--update_every", type=int, default=1)
+ parser.add_argument("-st", "--steps", type=int, default=50000000)
+ parser.add_argument("-lr", "--base_lr", type=float, default=4.5e-6)
+ parser.add_argument("-a", "--use_amp", default=False, action="store_true")
+ parser.add_argument("--amp_type", type=str, default="16")
+ parser.add_argument("--gradient_clip_val", type=float, default=None)
+ parser.add_argument("--gradient_clip_algorithm", type=str, default=None)
+ parser.add_argument("--every_n_train_steps", type=int, default=50000)
+ parser.add_argument("--log_every_n_steps", type=int, default=50)
+ parser.add_argument("--val_check_interval", type=int, default=1024)
+ parser.add_argument("--limit_val_batches", type=int, default=64)
+ parser.add_argument("--monitor", type=str, default="val/total_loss")
+ parser.add_argument("--output_dir", type=str, help="the output directory to save everything.")
+ parser.add_argument("--ckpt_path", type=str, default="", help="the restore checkpoints.")
+ parser.add_argument("--deepspeed", default=False, action="store_true")
+ parser.add_argument("--deepspeed2", default=False, action="store_true")
+ parser.add_argument("--scale_lr", type=bool, nargs="?", const=True, default=False,
+ help="scale base-lr by ngpu * batch_size * n_accumulate")
+ return parser.parse_args()
+
+
+if __name__ == "__main__":
+
+ args = get_args()
+
+ if args.fast:
+ torch.backends.cudnn.allow_tf32 = True
+ torch.backends.cuda.matmul.allow_tf32 = True
+ torch.set_float32_matmul_precision('medium')
+ torch.utils.data._utils.MP_STATUS_CHECK_INTERVAL = 0.05
+
+ # Set random seed
+ pl.seed_everything(args.seed, workers=True)
+
+ # Load configuration
+ config = get_config_from_file(args.config)
+ config = merge_cfg(config, vars(args))
+ training_cfg = config.training
+
+ # print config
+ rank_zero_info("Begin to print configuration ...")
+ rank_zero_info(OmegaConf.to_yaml(config))
+ rank_zero_info("Finish print ...")
+
+ # Setup callbacks
+ callbacks, loggers = setup_callbacks(config)
+
+ # Build data modules
+ data: pl.LightningDataModule = instantiate_from_config(config.dataset)
+
+ # Build model
+ model: pl.LightningModule = instantiate_from_config(config.model)
+
+ nodes = args.num_nodes
+ ngpus = args.num_gpus
+ base_lr = training_cfg.base_lr
+ accumulate_grad_batches = training_cfg.update_every
+ batch_size = config.dataset.params.batch_size
+
+ if 'NNODES' in os.environ:
+ nodes = int(os.environ['NNODES'])
+ training_cfg.num_nodes = nodes
+ args.num_nodes = nodes
+
+ if args.scale_lr:
+ model.learning_rate = accumulate_grad_batches * nodes * ngpus * batch_size * base_lr
+ info = f"Setting learning rate to {model.learning_rate:.2e} = {accumulate_grad_batches} (accumulate)"
+ info += f" * {nodes} (nodes) * {ngpus} (num_gpus) * {batch_size} (batchsize) * {base_lr:.2e} (base_lr)"
+ rank_zero_info(info)
+ else:
+ model.learning_rate = base_lr
+ rank_zero_info("++++ NOT USING LR SCALING ++++")
+ rank_zero_info(f"Setting learning rate to {model.learning_rate:.2e}")
+
+ # Build trainer
+ if args.num_nodes > 1 or args.num_gpus > 1:
+ if args.deepspeed:
+ ddp_strategy = DeepSpeedStrategy(stage=1)
+ elif args.deepspeed2:
+ ddp_strategy = 'deepspeed_stage_2'
+ else:
+ ddp_strategy = DDPStrategy(find_unused_parameters=False, bucket_cap_mb=1500)
+ else:
+ ddp_strategy = None # 'auto'
+
+ rank_zero_info(f'*' * 100)
+ if training_cfg.use_amp:
+ amp_type = training_cfg.amp_type
+ assert amp_type in ['bf16', '16', '32'], f"Invalid amp_type: {amp_type}"
+ rank_zero_info(f'Using {amp_type} precision')
+ else:
+ amp_type = 32
+ rank_zero_info(f'Using 32 bit precision')
+ rank_zero_info(f'*' * 100)
+
+ trainer = pl.Trainer(
+ max_steps=training_cfg.steps,
+ precision=amp_type,
+ callbacks=callbacks,
+ accelerator="gpu",
+ devices=training_cfg.num_gpus,
+ num_nodes=training_cfg.num_nodes,
+ strategy=ddp_strategy,
+ gradient_clip_val=training_cfg.get('gradient_clip_val'),
+ gradient_clip_algorithm=training_cfg.get('gradient_clip_algorithm'),
+ accumulate_grad_batches=args.update_every,
+ logger=loggers,
+ log_every_n_steps=training_cfg.log_every_n_steps,
+ val_check_interval=training_cfg.val_check_interval,
+ limit_val_batches=training_cfg.limit_val_batches,
+ check_val_every_n_epoch=None
+ )
+
+ # Train
+ if training_cfg.ckpt_path == '':
+ training_cfg.ckpt_path = None
+ trainer.fit(model, datamodule=data, ckpt_path=training_cfg.ckpt_path)
diff --git a/Hunyuan3D-2.1/hy3dshape/minimal_demo.py b/Hunyuan3D-2.1/hy3dshape/minimal_demo.py
new file mode 100644
index 0000000000000000000000000000000000000000..a4bfe8fe49e866e2d66b3a7b44cfd6dbfe21fd92
--- /dev/null
+++ b/Hunyuan3D-2.1/hy3dshape/minimal_demo.py
@@ -0,0 +1,30 @@
+# Hunyuan 3D is licensed under the TENCENT HUNYUAN NON-COMMERCIAL LICENSE AGREEMENT
+# except for the third-party components listed below.
+# Hunyuan 3D does not impose any additional limitations beyond what is outlined
+# in the repsective licenses of these third-party components.
+# Users must comply with all terms and conditions of original licenses of these third-party
+# components and must ensure that the usage of the third party components adheres to
+# all relevant laws and regulations.
+
+# For avoidance of doubts, Hunyuan 3D means the large language models and
+# their software and algorithms, including trained model weights, parameters (including
+# optimizer states), machine-learning model code, inference-enabling code, training-enabling code,
+# fine-tuning enabling code and other elements of the foregoing made publicly available
+# by Tencent in accordance with TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT.
+
+from PIL import Image
+
+from hy3dshape.rembg import BackgroundRemover
+from hy3dshape.pipelines import Hunyuan3DDiTFlowMatchingPipeline
+
+model_path = 'tencent/Hunyuan3D-2.1'
+pipeline_shapegen = Hunyuan3DDiTFlowMatchingPipeline.from_pretrained(model_path)
+
+image_path = 'demos/demo.png'
+image = Image.open(image_path).convert("RGBA")
+if image.mode == 'RGB':
+ rembg = BackgroundRemover()
+ image = rembg(image)
+
+mesh = pipeline_shapegen(image=image)[0]
+mesh.export('demo.glb')
diff --git a/Hunyuan3D-2.1/hy3dshape/minimal_vae_demo.py b/Hunyuan3D-2.1/hy3dshape/minimal_vae_demo.py
new file mode 100644
index 0000000000000000000000000000000000000000..2b398cb7e37fa2a81b930a35b2bb70cd293707df
--- /dev/null
+++ b/Hunyuan3D-2.1/hy3dshape/minimal_vae_demo.py
@@ -0,0 +1,51 @@
+# Hunyuan 3D is licensed under the TENCENT HUNYUAN NON-COMMERCIAL LICENSE AGREEMENT
+# except for the third-party components listed below.
+# Hunyuan 3D does not impose any additional limitations beyond what is outlined
+# in the repsective licenses of these third-party components.
+# Users must comply with all terms and conditions of original licenses of these third-party
+# components and must ensure that the usage of the third party components adheres to
+# all relevant laws and regulations.
+
+# For avoidance of doubts, Hunyuan 3D means the large language models and
+# their software and algorithms, including trained model weights, parameters (including
+# optimizer states), machine-learning model code, inference-enabling code, training-enabling code,
+# fine-tuning enabling code and other elements of the foregoing made publicly available
+# by Tencent in accordance with TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT.
+
+import torch
+
+from hy3dshape.surface_loaders import SharpEdgeSurfaceLoader
+from hy3dshape.models.autoencoders import ShapeVAE
+from hy3dshape.pipelines import export_to_trimesh
+
+
+vae = ShapeVAE.from_pretrained(
+ 'tencent/Hunyuan3D-2.1',
+ use_safetensors=False,
+ variant='fp16',
+)
+
+
+loader = SharpEdgeSurfaceLoader(
+ num_sharp_points=0,
+ num_uniform_points=81920,
+)
+mesh_demo = 'demos/demo.glb'
+surface = loader(mesh_demo).to('cuda', dtype=torch.float16)
+print(surface.shape)
+
+latents = vae.encode(surface)
+latents = vae.decode(latents)
+mesh = vae.latents2mesh(
+ latents,
+ output_type='trimesh',
+ bounds=1.01,
+ mc_level=0.0,
+ num_chunks=20000,
+ octree_resolution=256,
+ mc_algo='mc',
+ enable_pbar=True
+)
+
+mesh = export_to_trimesh(mesh)[0]
+mesh.export('output.obj')
diff --git a/Hunyuan3D-2.1/hy3dshape/requirements.txt b/Hunyuan3D-2.1/hy3dshape/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..abdab84e043a82dc588767e52e925971dd183d4f
--- /dev/null
+++ b/Hunyuan3D-2.1/hy3dshape/requirements.txt
@@ -0,0 +1,40 @@
+ninja
+pybind11
+
+diffusers
+einops
+opencv-python
+numpy
+torch
+transformers
+torchvision
+#taming-transformers-rom1504
+#ConfigArgParse
+#ipdb
+omegaconf
+
+#sentencepiece
+tqdm
+
+# Mesh Processing
+trimesh
+pymeshlab
+pygltflib
+xatlas
+#kornia
+#facexlib
+
+# Training
+accelerate
+#pytorch_lightning
+#scikit-learn
+#scikit-image
+
+# Demo only
+gradio
+fastapi
+uvicorn
+rembg
+onnxruntime
+#gevent
+#geventhttpclient
\ No newline at end of file
diff --git a/Hunyuan3D-2.1/hy3dshape/scripts/train_deepspeed.sh b/Hunyuan3D-2.1/hy3dshape/scripts/train_deepspeed.sh
new file mode 100644
index 0000000000000000000000000000000000000000..ed6b7c7c3b8dded37891016fe6bcbab3fb9ba282
--- /dev/null
+++ b/Hunyuan3D-2.1/hy3dshape/scripts/train_deepspeed.sh
@@ -0,0 +1,70 @@
+# If: ImportError: /usr/lib64/libstdc++.so.6: version `GLIBCXX_3.4.20' not found
+# Do: ln /usr/local/gcc-8.3/lib64/libstdc++.so.6 -sf /usr/lib64/libstdc++.so.6
+
+export NCCL_IB_TIMEOUT=24
+export NCCL_NVLS_ENABLE=0
+NET_TYPE="high"
+if [[ "${NET_TYPE}" = "low" ]]; then
+ export NCCL_SOCKET_IFNAME=eth1
+ export NCCL_IB_GID_INDEX=3
+ export NCCL_IB_HCA=mlx5_2:1,mlx5_2:1
+ export NCCL_IB_SL=3
+ export NCCL_CHECK_DISABLE=1
+ export NCCL_P2P_DISABLE=0
+ export NCCL_LL_THRESHOLD=16384
+ export NCCL_IB_CUDA_SUPPORT=1
+else
+ export NCCL_IB_GID_INDEX=3
+ export NCCL_IB_SL=3
+ export NCCL_CHECK_DISABLE=1
+ export NCCL_P2P_DISABLE=0
+ export NCCL_IB_DISABLE=0
+ export NCCL_LL_THRESHOLD=16384
+ export NCCL_IB_CUDA_SUPPORT=1
+ export NCCL_SOCKET_IFNAME=bond1
+ export UCX_NET_DEVICES=bond1
+ export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
+ export NCCL_COLLNET_ENABLE=0
+ export SHARP_COLL_ENABLE_SAT=0
+ export NCCL_NET_GDR_LEVEL=2
+ export NCCL_IB_QPS_PER_CONNECTION=4
+ export NCCL_IB_TC=160
+ export NCCL_PXN_DISABLE=0
+fi
+export NCCL_DEBUG=WARN
+
+node_num=$1
+node_rank=$2
+master_ip=$3
+config=$4
+output_dir=$5
+
+# config='configs/dit-from-scratch-overfitting-flowmatching-dinog518-bf16-lr1e4-1024.yaml'
+# output_dir='output_folder/dit/overfitting_10'
+
+echo node_num $node_num
+echo node_rank $node_rank
+echo master_ip $master_ip
+echo config $config
+echo output_dir $output_dir
+
+if test -d "$output_dir"; then
+ cp $config $output_dir
+else
+ mkdir -p "$output_dir"
+ cp $config $output_dir
+fi
+
+NODE_RANK=$node_rank \
+HF_HUB_OFFLINE=0 \
+MASTER_PORT=12348 \
+MASTER_ADDR=$master_ip \
+NCCL_SOCKET_IFNAME=bond1 \
+NCCL_IB_GID_INDEX=3 \
+NCCL_NVLS_ENABLE=0 \
+python3 main.py \
+ --num_nodes $node_num \
+ --num_gpus 8 \
+ --config $config \
+ --output_dir $output_dir \
+ --deepspeed
diff --git a/Hunyuan3D-2.1/hy3dshape/tools/README-ch.md b/Hunyuan3D-2.1/hy3dshape/tools/README-ch.md
new file mode 100644
index 0000000000000000000000000000000000000000..af1c480aef1b6381e424608c52e2f794cd984b92
--- /dev/null
+++ b/Hunyuan3D-2.1/hy3dshape/tools/README-ch.md
@@ -0,0 +1,94 @@
+# 数据处理
+
+这是用于3D形状和纹理生成的数据处理流程。
+
+**注意事项**：
+1. 该实现是我们工业流程的简化版本。
+2. 渲染脚本基于[TRELLIS](https://github.com/microsoft/TRELLIS/blob/main/dataset_toolkits/blender_script/render.py)。
+
+## 渲染
+
+### 动机
+渲染脚本`render/render.py`主要有三个目的：
+1. 使用Blender将复杂的3D格式转换为PLY文件，以便进行进一步处理。
+2. 为DiT训练渲染条件图像。
+3. 渲染正交图像、PBR材质以及用于纹理生成的条件信号（世界空间法线和位置）。
+
+### 需求
+渲染脚本使用Blender 4.1执行。你需要使用Blender的Python安装`opencv`、`OpenEXR`和`Imath`。以下是Macbook上的示例：
+```bash
+/Applications/Blender.app/Contents/Resources/4.1/python/bin/python3.11 -m pip install OpenEXR Imath opencv-python
+```
+
+### 执行
+前两个目的可以通过以下单一命令执行：
+```bash
+$BLENDER_PATH -b -P render/render.py -- \
+ --object ${INPUT_FILE} --geo_mode --resolution 512 \
+ --output_folder $OUTPUT_FOLDER
+```
+对于第三个目的，只需移除`--geo_mode`标志。
+
+## 水密网格处理和采样
+
+### 动机
+为了学习3DShape2VecSets的SDF表示，我们需要一个水密输入网格。该流程处理原始三角网格，生成三种必要的数据类型：
+1. **表面采样** - 编码器的输入点。
+2. **体积采样** - 解码器中SDF评估的查询点。
+3. **体积SDFs** - VAE训练的地面真实有符号距离值。
+
+### 执行
+处理三角网格（OBJ/OFF格式），生成以下内容：
+1. 水密网格（`${OUTPUT_NAME}_watertight.obj`）。
+2. 表面点采样（`${OUTPUT_NAME}_surface.npz`）。
+3. 带有SDF的体积采样（`${OUTPUT_NAME}_sdf.npz`）。
+
+**命令：**
+```bash
+python3 watertight/watertight_and_sample.py \
+ --input_obj ${INPUT_MESH} \
+ --output_prefix ${OUTPUT_NAME}
+```
+
+### 输出数据格式
+
+#### 1. 表面采样（`${OUTPUT_NAME}_surface.npz`）
+包含两个点云数组，以numpy NPZ格式存储：
+
+| 键 | 形状 | 格式 | 描述 |
+|-----------------|----------|----------|---------------------------------|
+| `random_surface` | `(N, 6)` | `float16`| 表面上的均匀点采样 |
+| `sharp_surface` | `(M, 6)` | `float16`| 靠近网格锐边的采样 |
+
+#### 2. 体积SDF采样（`${OUTPUT_NAME}_sdf.npz`）
+包含三种采样类型，以数组对的形式存储。对于每种类型`${type}`：
+
+| 采样类型 | 点数组 | SDF标签数组 | 形状 | 格式 | 描述 |
+|-----------------|----------------------|----------------------|----------|----------|-------------------------|
+| `vol` | `vol_points` | `vol_label` | `(P, 3)/(P,)` | `float16`| 随机空间采样 |
+| `random_near` | `random_near_points` | `random_near_label` | `(Q, 3)/(Q,)` | `float16`| 靠近表面的采样 |
+| `sharp_near` | `sharp_near_points` | `sharp_near_label` | `(R, 3)/(R,)` | `float16`| 靠近锐边的采样 |
+
+**数据规格**：
+- 所有点坐标（`*_points`数组）包含以`float16`值存储的3D位置。
+- 所有SDF值（`*_label`数组）是表示以下内容的`float16`标量：
+ - **正值**：在表面外。
+ - **负值**：在表面内。
+ - **零值**：在表面上。
+- 数组维度：
+ - `N`、`M`、`P`、`Q`、`R`表示采样数量（因形状而异）。
+ - `3`表示XYZ坐标。
+ - `6`表示XYZ/法线坐标。
+- 所有数组均以未压缩形式存储在numpy的NPZ格式中。
+
+## 整体脚本
+修改pipeline.sh里面这4个变量，
+1. **INPUT_FILE** 每个3D数据的路径。
+2. **OUTPUT_FOLDER** 输出数据集的总路径。
+3. **NAME** 每个数据的输出路径命名。
+4. **BLENDER_PATH** Blender可执行路径。
+
+然后运行以下脚本：
+```bash
+bash pipeline.sh
+```
\ No newline at end of file
diff --git a/Hunyuan3D-2.1/hy3dshape/tools/README.md b/Hunyuan3D-2.1/hy3dshape/tools/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..38b70546fe2ec922ce735fdc89360aa2770e450e
--- /dev/null
+++ b/Hunyuan3D-2.1/hy3dshape/tools/README.md
@@ -0,0 +1,94 @@
+# Data Processing
+
+This is the data processing pipeline for 3D shape and texture generation.
+
+**Notes**:
+1. This implementation is a simplified version of our industrial pipeline.
+2. The rendering script is based on [TRELLIS](https://github.com/microsoft/TRELLIS/blob/main/dataset_toolkits/blender_script/render.py).
+
+## Rendering
+
+### Motivation
+The rendering script `render/render.py` serves three main purposes:
+1. Converting complex 3D formats to PLY files using Blender for further processing.
+2. Rendering condition images for DiT training.
+3. Rendering orthogonal images, PBR materials, and conditional signals (world-space normals and positions) for texture generation.
+
+### Requirements
+The rendering scripts are executed with Blender 4.1. You need to install `opencv`, `OpenEXR`, and `Imath` using Blender's Python. Here is an example for a Macbook:
+```bash
+/Applications/Blender.app/Contents/Resources/4.1/python/bin/python3.11 -m pip install OpenEXR Imath opencv-python
+```
+
+### Execution
+The first two purposes can be executed with a single command:
+```bash
+$BLENDER_PATH -b -P render/render.py -- \
+ --object ${INPUT_FILE} --geo_mode --resolution 512 \
+ --output_folder $OUTPUT_FOLDER
+```
+For the third purpose, simply remove the `--geo_mode` flag.
+
+## Watertight Mesh Processing and Sampling
+
+### Motivation
+To learn an SDF representation for 3DShape2VecSets, we require a watertight input mesh. This pipeline processes raw triangle meshes to generate three essential data types:
+1. **Surface samples** - Input points for the encoder.
+2. **Volume samples** - Query points for SDF evaluation in the decoder.
+3. **Volume SDFs** - Ground-truth signed distance values for VAE training.
+
+### Execution
+Process a triangle mesh (OBJ/OFF format) to generate:
+1. Watertight mesh (`${OUTPUT_NAME}_watertight.obj`).
+2. Surface point samples (`${OUTPUT_NAME}_surface.npz`).
+3. Volume samples with SDFs (`${OUTPUT_NAME}_sdf.npz`).
+
+**Command:**
+```bash
+python3 watertight/watertight_and_sample.py \
+ --input_obj ${INPUT_MESH} \
+ --output_prefix ${OUTPUT_NAME}
+```
+
+### Output Data Format
+
+#### 1. Surface Samples (`${OUTPUT_NAME}_surface.npz`)
+Contains two point cloud arrays in numpy NPZ format:
+
+| Key | Shape | Format | Description |
+|-----------------|----------|----------|---------------------------------|
+| `random_surface` | `(N, 6)` | `float16`| Uniform point samples on surface |
+| `sharp_surface` | `(M, 6)` | `float16`| Samples near sharp mesh edges |
+
+#### 2. Volume SDF Samples (`${OUTPUT_NAME}_sdf.npz`)
+Contains three sample types stored as array pairs. For each type `${type}`:
+
+| Sample Type | Points Array | SDF Labels Array | Shape | Format | Description |
+|-----------------|----------------------|----------------------|----------|----------|-------------------------|
+| `vol` | `vol_points` | `vol_label` | `(P, 3)/(P,)` | `float16`| Random spatial samples |
+| `random_near` | `random_near_points` | `random_near_label` | `(Q, 3)/(Q,)` | `float16`| Samples near surface |
+| `sharp_near` | `sharp_near_points` | `sharp_near_label` | `(R, 3)/(R,)` | `float16`| Samples near sharp edges |
+
+**Data Specifications**:
+- All point coordinates (`*_points` arrays) contain 3D positions stored as `float16` values.
+- All SDF values (`*_label` arrays) are `float16` scalars representing:
+ - **Positive values**: Outside the surface.
+ - **Negative values**: Inside the surface.
+ - **Zero values**: On the surface.
+- Array dimensions:
+ - `N`, `M`, `P`, `Q`, `R` represent sample counts (vary per shape).
+ - `3` indicates XYZ coordinates.
+ - `6` indicates XYZ/Normal coordinates.
+- All arrays are stored uncompressed in numpy's NPZ format.
+
+## Overall Script
+Modify the first four variables in `pipeline.sh`:
+1. **INPUT_FILE** The path to each 3D data file.
+2. **OUTPUT_FOLDER** The overall path for the output dataset.
+3. **NAME** The naming for the output path of each data.
+4. **BLENDER_PATH** The executable path for Blender.
+
+Then run the following script:
+```bash
+bash pipeline.sh
+```
\ No newline at end of file
diff --git a/Hunyuan3D-2.1/hy3dshape/tools/pipeline.sh b/Hunyuan3D-2.1/hy3dshape/tools/pipeline.sh
new file mode 100644
index 0000000000000000000000000000000000000000..01d24785d9c6a8241f00023faf8be1880a57b1e7
--- /dev/null
+++ b/Hunyuan3D-2.1/hy3dshape/tools/pipeline.sh
@@ -0,0 +1,10 @@
+export OPENCV_IO_ENABLE_OPENEXR=1
+export OUTPUT_FOLDER=dataset/preprocessed
+export BLENDER_PATH=/path/to/blender-4.0.2-linux-x64/blender
+
+export INPUT_FILE=dataset/raw/uuid.glb
+export NAME=uuid
+
+$BLENDER_PATH -b -P render/render.py -- --object ${INPUT_FILE} --output_folder $OUTPUT_FOLDER/$NAME/render_cond --geo_mode --resolution 512
+# $BLENDER_PATH -b -P render/render.py -- --object ${INPUT_FILE} --output_folder $OUTPUT_FOLDER/$NAME/render_tex --resolution 512
+python3 watertight/watertight_and_sample.py --input_obj $OUTPUT_FOLDER/$NAME/render_cond/mesh.ply --output_prefix $OUTPUT_FOLDER/$NAME/geo_data/$NAME
diff --git a/Hunyuan3D-2.1/hy3dshape/tools/render/render.py b/Hunyuan3D-2.1/hy3dshape/tools/render/render.py
new file mode 100644
index 0000000000000000000000000000000000000000..17011e6e825b3f7977f910812bc59eda9f0c98b2
--- /dev/null
+++ b/Hunyuan3D-2.1/hy3dshape/tools/render/render.py
@@ -0,0 +1,925 @@
+# Hunyuan 3D is licensed under the TENCENT HUNYUAN NON-COMMERCIAL LICENSE AGREEMENT
+# except for the third-party components listed below.
+# Hunyuan 3D does not impose any additional limitations beyond what is outlined
+# in the repsective licenses of these third-party components.
+# Users must comply with all terms and conditions of original licenses of these third-party
+# components and must ensure that the usage of the third party components adheres to
+# all relevant laws and regulations.
+
+# For avoidance of doubts, Hunyuan 3D means the large language models and
+# their software and algorithms, including trained model weights, parameters (including
+# optimizer states), machine-learning model code, inference-enabling code, training-enabling code,
+# fine-tuning enabling code and other elements of the foregoing made publicly available
+# by Tencent in accordance with TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT.
+
+
+import argparse, sys, os, math, re, glob
+from typing import *
+import bpy
+from mathutils import Vector, Matrix
+import numpy as np
+import json
+import glob
+import random
+import shutil
+import mathutils
+import cv2
+
+"""=============== BLENDER ==============="""
+
+IMPORT_FUNCTIONS: Dict[str, Callable] = {
+ "obj": bpy.ops.import_scene.obj,
+ "glb": bpy.ops.import_scene.gltf,
+ "gltf": bpy.ops.import_scene.gltf,
+ "usd": bpy.ops.import_scene.usd,
+ "fbx": bpy.ops.import_scene.fbx,
+ "stl": bpy.ops.import_mesh.stl,
+ "usda": bpy.ops.import_scene.usda,
+ "dae": bpy.ops.wm.collada_import,
+ "ply": bpy.ops.import_mesh.ply,
+ "abc": bpy.ops.wm.alembic_import,
+ "blend": bpy.ops.wm.append,
+}
+
+EXT = {
+ 'PNG': 'png',
+ 'JPEG': 'jpg',
+ 'OPEN_EXR': 'exr',
+ 'TIFF': 'tiff',
+ 'BMP': 'bmp',
+ 'HDR': 'hdr',
+ 'TARGA': 'tga'
+}
+
+PRIMES = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53]
+
+def radical_inverse(base, n):
+ val = 0
+ inv_base = 1.0 / base
+ inv_base_n = inv_base
+ while n > 0:
+ digit = n % base
+ val += digit * inv_base_n
+ n //= base
+ inv_base_n *= inv_base
+ return val
+
+def halton_sequence(dim, n):
+ return [radical_inverse(PRIMES[dim], n) for dim in range(dim)]
+
+def hammersley_sequence(dim, n, num_samples):
+ return [n / num_samples] + halton_sequence(dim - 1, n)
+
+def sphere_hammersley_sequence(n, num_samples, offset=(0, 0)):
+ u, v = hammersley_sequence(2, n, num_samples)
+ u += offset[0] / num_samples
+ v += offset[1]
+ u = 2 * u if u < 0.25 else 2 / 3 * u + 1 / 3
+ theta = np.arccos(1 - 2 * u) - np.pi / 2
+ phi = v * 2 * np.pi
+ return [phi, theta]
+
+def trellis_cond_camera_sequence(num_cond_views):
+ yaws = []
+ pitchs = []
+ offset = (np.random.rand(), np.random.rand())
+ for i in range(num_cond_views):
+ y, p = sphere_hammersley_sequence(i, num_cond_views, offset)
+ yaws.append(y)
+ pitchs.append(p)
+ fov_min, fov_max = 10, 70
+ radius_min = np.sqrt(3) / 2 / np.sin(fov_max / 360 * np.pi)
+ radius_max = np.sqrt(3) / 2 / np.sin(fov_min / 360 * np.pi)
+ k_min = 1 / radius_max**2
+ k_max = 1 / radius_min**2
+ ks = np.random.uniform(k_min, k_max, (1000000,))
+ radius = [1 / np.sqrt(k) for k in ks]
+ fov = [2 * np.arcsin(np.sqrt(3) / 2 / r) for r in radius]
+
+ views = [{'hangle': y, 'vangle': p, 'cam_dis': r, 'fov': f, 'proj_type': 0} \
+ for y, p, r, f in zip(yaws, pitchs, radius, fov)]
+ return views
+
+def orthogonal_camera_sequence():
+ yaws = [-0.5 * np.pi, 0, 0.5 * np.pi, np.pi, -0.5 * np.pi, -0.5 * np.pi]
+ pitchs = [0, 0, 0, 0, 0.5 * np.pi, -0.5 * np.pi]
+ radius = [1.5 for i in range(6)]
+ fov = [1.5 * np.arcsin(np.sqrt(3) / 2 / r) for r in radius]
+ views = [{'hangle': y, 'vangle': p, 'cam_dis': r, 'fov': f, 'proj_type': 1} \
+ for y, p, r, f in zip(yaws, pitchs, radius, fov)]
+ return views
+
+
+def switch_to_mr_render(render_base_color, output_nodes):
+ bpy.context.scene.view_settings.view_transform = 'Raw'
+ bpy.context.scene.use_nodes = True
+ tree = bpy.context.scene.node_tree
+ links = tree.links
+
+ for i in range(len(output_nodes)):
+ if i + 1 != len(output_nodes):
+ for l in output_nodes[i][1].links:
+ links.remove(l)
+ else:
+ links.new(output_nodes[i][0], output_nodes[i][1])
+
+ for material in bpy.data.materials:
+ if not material.use_nodes:
+ continue
+ bsdf_node = None
+ output_node = None
+ node_tree = material.node_tree
+ links = material.node_tree.links
+ nodes = node_tree.nodes
+ for node in node_tree.nodes:
+ # Check if the node is a BSDF node
+ if node.type == 'BSDF_PRINCIPLED':
+ bsdf_node = node
+ if node.type == 'OUTPUT_MATERIAL':
+ output_node = node
+ if bsdf_node is None or output_node is None:
+ continue
+ #bsdf_node.inputs['Emission'].default_value = 0
+ bsdf_node.inputs['Emission Strength'].default_value = 0
+ mr_node = None
+ bc_node = None
+ for node in node_tree.nodes:
+ # Check if the node is a BSDF node
+ if node.name == 'COMBINE_METALLIC_ROUGHNESS':
+ mr_node = node
+ if node.name == 'COMBINE_BASE_COLOR':
+ bc_node = node
+ if mr_node is None:
+ combine_rgb_node = nodes.new('ShaderNodeCombineColor')
+ #combine_rgb_node.name = 'COMBINE_METALLIC_ROUGHNESS'
+
+ # Optionally, set the RGB values
+ combine_rgb_node.inputs['Red'].default_value = 1.0
+ combine_rgb_node.inputs['Green'].default_value = 0.5
+ combine_rgb_node.inputs['Blue'].default_value = 0.0
+ metallic_input = bsdf_node.inputs["Metallic"]
+
+ if metallic_input.links:
+ source_endpoint = metallic_input.links[0].from_socket
+ links.new(source_endpoint, combine_rgb_node.inputs['Blue'])
+
+ roughness_input = bsdf_node.inputs['Roughness']
+ if roughness_input.links:
+ source_endpoint = roughness_input.links[0].from_socket
+ links.new(source_endpoint, combine_rgb_node.inputs['Green'])
+
+ emission_shader = nodes.new("ShaderNodeEmission")
+ emission_shader.inputs["Strength"].default_value = 1
+ links.new(combine_rgb_node.outputs["Color"], emission_shader.inputs["Color"])
+
+ mix_shader = nodes.new("ShaderNodeMixShader")
+ mix_shader.name = 'COMBINE_METALLIC_ROUGHNESS'
+ links.new(bsdf_node.outputs["BSDF"], mix_shader.inputs[1])
+ links.new(emission_shader.outputs["Emission"], mix_shader.inputs[2])
+ mr_node = mix_shader
+
+ mix_shader_bc = nodes.new("ShaderNodeMixShader")
+ mix_shader_bc.name = 'COMBINE_BASE_COLOR'
+
+ if len(bsdf_node.inputs['Base Color'].links) > 0:
+ socket = bsdf_node.inputs['Base Color'].links[0].from_socket
+ gamma_node = node_tree.nodes.new(type='ShaderNodeGamma')
+
+ gamma_node.inputs[1].default_value = 0.454
+ node_tree.links.new(socket, gamma_node.inputs[0])
+ node_tree.links.new(gamma_node.outputs[0], mix_shader_bc.inputs[1])
+
+ links.new(mix_shader.outputs["Shader"], mix_shader_bc.inputs[2])
+ bc_node = mix_shader_bc
+
+ for l in output_node.inputs['Surface'].links:
+ links.remove(l)
+ links.new(mix_shader_bc.outputs["Shader"], output_node.inputs["Surface"])
+
+ mr_node.inputs["Fac"].default_value = 1.0
+ if render_base_color:
+ bc_node.inputs['Fac'].default_value = 0.0
+ else:
+ bc_node.inputs['Fac'].default_value = 1.0
+
+def switch_to_color_render(output_nodes):
+ bpy.context.scene.view_settings.view_transform = 'Standard'
+ bpy.context.scene.use_nodes = True
+ tree = bpy.context.scene.node_tree
+ links = tree.links
+
+ for i in range(len(output_nodes)):
+ if i + 1 == len(output_nodes):
+ for l in output_nodes[i][1].links:
+ links.remove(l)
+ else:
+ links.new(output_nodes[i][0], output_nodes[i][1])
+
+ for material in bpy.data.materials:
+ if not material.use_nodes:
+ continue
+ node_tree = material.node_tree
+ links = material.node_tree.links
+ nodes = node_tree.nodes
+ mr_node = None
+ bc_node = None
+ for node in node_tree.nodes:
+ if node.name == 'COMBINE_METALLIC_ROUGHNESS':
+ mr_node = node
+ if node.name == 'COMBINE_BASE_COLOR':
+ bc_node = node
+ if mr_node is not None and bc_node is not None:
+ mr_node.inputs["Fac"].default_value = 0.0
+ if len(bc_node.inputs[1].links) > 0:
+ try:
+ node = bc_node.inputs[1].links[0].from_socket.node
+ node.image.colorspace_settings.name = 'sRGB'
+ except:
+ pass
+
+# def ConvertNormalMap(input_exr, output_jpg):
+# import OpenEXR
+# import Imath
+# file = OpenEXR.InputFile(input_exr)
+# channels = file.header()['channels'].keys()
+
+# # Get the image data
+# data_window = file.header()['dataWindow']
+# width = data_window.max.x - data_window.min.x + 1
+# height = data_window.max.y - data_window.min.y + 1
+
+# # Read the X, Y, and Z channels as 32-bit floats
+# x_channel = np.frombuffer(file.channel('X', Imath.PixelType(Imath.PixelType.FLOAT)), dtype=np.float32)
+# y_channel = np.frombuffer(file.channel('Y', Imath.PixelType(Imath.PixelType.FLOAT)), dtype=np.float32)
+# z_channel = np.frombuffer(file.channel('Z', Imath.PixelType(Imath.PixelType.FLOAT)), dtype=np.float32)
+
+# # Reshape the channels into 2D arrays
+# x_channel = x_channel.reshape((height, width))
+# y_channel = y_channel.reshape((height, width))
+# z_channel = z_channel.reshape((height, width))
+
+# # Stack the channels to create a 3D array
+# normal = np.stack((x_channel, y_channel, z_channel), axis=-1)
+# normal = ((normal * 0.5 + 0.5) * 255).astype('uint8')
+# cv2.imwrite(output_jpg, normal)
+
+
+def ConvertNormalMap(input_exr, output_jpg):
+ # Read EXR file with OpenCV (returns float32 image)
+ exr_img = cv2.imread(input_exr, cv2.IMREAD_UNCHANGED)
+ if exr_img is None:
+ raise RuntimeError(f"Failed to load EXR file: {input_exr}")
+ print(f"EXR shape: {exr_img.shape}, dtype: {exr_img.dtype}")
+ normal = ((exr_img * 0.5 + 0.5) * 255).clip(0, 255).astype(np.uint8)
+ cv2.imwrite(output_jpg, normal)
+ print(f"Saved normal map to {output_jpg}")
+
+
+gidx = 0
+def ConvertDepthMap(input_exr, output_png):
+ import bpy
+
+ # cam = bpy.data.objects.get('Camera')
+ cams = [obj for obj in bpy.data.objects if obj.type == 'CAMERA']
+ print("All cameras in scene:")
+ if not cams:
+ raise RuntimeError("No camera objects found in the scene")
+ for c in cams:
+ print(f" {c.name} - type: {c.type}")
+ cam = cams[0]
+
+ print('cam', cam)
+ print('cam.type', cam.type) # should be 'CAMERA'
+ print('cam_data', cam.data) # should not be None
+ print(f"Using camera: {cam.name}")
+
+ cam_data = cam.data
+
+ exr_img = cv2.imread(input_exr, cv2.IMREAD_UNCHANGED)
+ if exr_img is None:
+ raise RuntimeError(f"Failed to load EXR file: {input_exr}")
+
+ print(f"EXR shape: {exr_img.shape}, dtype: {exr_img.dtype}")
+
+ depth_channel = exr_img[:, :, 0] if exr_img.ndim == 3 else exr_img
+
+ # filter
+ depth_channel = depth_channel.copy()
+ depth_channel[depth_channel > 1e9] = 0
+
+ extrinsic_matrix = np.array(cam.matrix_world.copy())
+
+ scene = bpy.context.scene
+ render = scene.render
+ cam_data = cam.data
+
+ resolution_x = render.resolution_x * render.pixel_aspect_x
+ resolution_y = render.resolution_y * render.pixel_aspect_y
+
+ cx = resolution_x / 2.0
+ cy = resolution_y / 2.0
+
+ if cam_data.type == 'ORTHO':
+ aspect_ratio = render.resolution_x / render.resolution_y
+ ortho_scale = cam_data.ortho_scale
+ near = cam_data.clip_start
+ far = cam_data.clip_end
+
+ left = -ortho_scale / 2
+ right = ortho_scale / 2
+ top = (ortho_scale / 2) / aspect_ratio
+ bottom = -top
+
+ proj_matrix = np.array((
+ (2/(right-left), 0, 0, -(right+left)/(right-left)),
+ (0, 2/(top-bottom), 0, -(top+bottom)/(top-bottom)),
+ (0, 0, -2/(far-near), -(far+near)/(far-near)),
+ (0, 0, 0, 1)
+ ))
+ else:
+ if cam_data.sensor_fit == 'VERTICAL':
+ sensor_size = cam_data.sensor_height
+ fit = 'VERTICAL'
+ else:
+ sensor_size = cam_data.sensor_width
+ fit = 'HORIZONTAL'
+
+ focal_length = cam_data.lens
+
+ if fit == 'HORIZONTAL':
+ scale = resolution_x / sensor_size
+ else:
+ scale = resolution_y / sensor_size
+
+ fx = focal_length * scale
+ fy = focal_length * scale
+
+ K = np.array([
+ [fx, 0, cx],
+ [0, fy, cy],
+ [0, 0, 1]
+ ])
+
+ mask = (depth_channel.reshape(-1) == 0)
+ jj, ii = np.meshgrid(np.arange(resolution_x), np.arange(resolution_y))
+ jj = jj + 0.5
+ ii = ii + 0.5
+
+ if cam_data.type == 'ORTHO':
+ cam_pos = np.stack((
+ (jj - cx) * (1.0 / (resolution_x - 1) * ortho_scale),
+ (ii - cy) * (1.0 / (resolution_y - 1) * ortho_scale),
+ depth_channel
+ ), axis=-1)
+ else:
+ image_pos = np.stack((jj * depth_channel, ii * depth_channel, depth_channel), axis=-1)
+ cam_pos = image_pos @ np.linalg.inv(K).T
+
+ cam_pos[..., 1:] = -cam_pos[..., 1:]
+
+ world_pos = cam_pos @ extrinsic_matrix[:3, :3].T + extrinsic_matrix[:3, 3].reshape(1, 1, 3)
+ world_pos = world_pos.reshape(-1, 3)
+ world_pos[mask] = 0
+ world_pos = world_pos.reshape(cam_pos.shape)
+ world_pos = np.stack((world_pos[..., 0], world_pos[..., 2], -world_pos[..., 1]), axis=-1)
+
+ img_out = np.clip((0.5 + world_pos) * 255, 0, 255).astype('uint8')
+ cv2.imwrite(output_png, img_out)
+ print(f"Saved depth map to {output_png}")
+
+
+def init_render(engine='CYCLES', resolution=512, geo_mode=False):
+ bpy.context.scene.render.engine = engine
+ bpy.context.scene.render.resolution_x = resolution
+ bpy.context.scene.render.resolution_y = resolution
+ bpy.context.scene.render.resolution_percentage = 100
+ bpy.context.scene.render.image_settings.file_format = 'PNG'
+ bpy.context.scene.render.image_settings.color_mode = 'RGBA'
+ bpy.context.scene.render.film_transparent = True
+
+ bpy.context.scene.cycles.device = 'GPU'
+ #bpy.context.scene.cycles.samples = 128 if not geo_mode else 1
+ bpy.context.scene.cycles.filter_type = 'BOX'
+ bpy.context.scene.cycles.filter_width = 1
+ bpy.context.scene.cycles.diffuse_bounces = 1
+ bpy.context.scene.cycles.glossy_bounces = 1
+ # bpy.context.scene.cycles.transparent_max_bounces = 3 if not geo_mode else 0
+ # bpy.context.scene.cycles.transmission_bounces = 3 if not geo_mode else 1
+ bpy.context.scene.cycles.use_denoising = True
+
+ bpy.context.preferences.addons['cycles'].preferences.get_devices()
+ # bpy.context.preferences.addons['cycles'].preferences.compute_device_type = 'CUDA'
+
+def init_nodes(save_depth=False, save_normal=False, save_albedo=False, save_mr = False, save_mist=False):
+ if not any([save_depth, save_normal, save_albedo, save_mist]):
+ return {}, {}, []
+ outputs = {}
+ spec_nodes = {}
+ composite_nodes = []
+ bpy.context.scene.use_nodes = True
+ bpy.context.scene.view_layers['ViewLayer'].use_pass_z = save_depth
+ bpy.context.scene.view_layers['ViewLayer'].use_pass_normal = save_normal
+ bpy.context.scene.view_layers['ViewLayer'].use_pass_diffuse_color = save_albedo
+ bpy.context.scene.view_layers['ViewLayer'].use_pass_mist = save_mist
+
+ nodes = bpy.context.scene.node_tree.nodes
+ links = bpy.context.scene.node_tree.links
+ for n in nodes:
+ nodes.remove(n)
+
+ render_layers = nodes.new('CompositorNodeRLayers')
+
+ if save_depth:
+ depth_file_output = nodes.new('CompositorNodeOutputFile')
+ depth_file_output.base_path = ''
+ depth_file_output.file_slots[0].use_node_format = True
+ depth_file_output.format.file_format = "OPEN_EXR"
+ links.new(render_layers.outputs['Depth'], depth_file_output.inputs[0])
+
+ outputs['depth'] = depth_file_output
+ composite_nodes.append((render_layers.outputs['Depth'], depth_file_output.inputs[0]))
+
+ if save_normal:
+ normal_file_output = nodes.new('CompositorNodeOutputFile')
+ normal_file_output.base_path = ''
+ normal_file_output.file_slots[0].use_node_format = True
+ normal_file_output.format.file_format = 'OPEN_EXR'
+ links.new(render_layers.outputs['Normal'], normal_file_output.inputs[0])
+
+ outputs['normal'] = normal_file_output
+ composite_nodes.append((render_layers.outputs['Normal'], normal_file_output.inputs[0]))
+
+ if save_albedo:
+ albedo_file_output = nodes.new('CompositorNodeOutputFile')
+ albedo_file_output.base_path = ''
+ albedo_file_output.file_slots[0].use_node_format = True
+ albedo_file_output.format.file_format = 'PNG'
+ albedo_file_output.format.color_mode = 'RGBA'
+ albedo_file_output.format.color_depth = '8'
+
+ alpha_albedo = nodes.new('CompositorNodeSetAlpha')
+
+ links.new(render_layers.outputs['DiffCol'], alpha_albedo.inputs['Image'])
+ links.new(render_layers.outputs['Alpha'], alpha_albedo.inputs['Alpha'])
+ links.new(alpha_albedo.outputs['Image'], albedo_file_output.inputs[0])
+
+ outputs['albedo'] = albedo_file_output
+ #composite_nodes.append((alpha_albedo.outputs['Image'], albedo_file_output.inputs[0]))
+
+ if save_mr:
+ mr_file_output = tree.nodes.new(type='CompositorNodeOutputFile')
+ mr_file_output.base_path = ''
+ mr_file_output.file_slots[0].use_node_format = True
+ mr_file_output.format.file_format = 'OPEN_EXR'
+
+ links.new(render_layers.outputs['Image'], mr_file_output.inputs[0])
+
+ outputs['mr'] = mr_file_output
+ composite_nodes.append((render_layers.outputs['Image'], mr_file_output.inputs[0]))
+
+ if save_mist:
+ bpy.data.worlds['World'].mist_settings.start = 0
+ bpy.data.worlds['World'].mist_settings.depth = 10
+
+ mist_file_output = nodes.new('CompositorNodeOutputFile')
+ mist_file_output.base_path = ''
+ mist_file_output.file_slots[0].use_node_format = True
+ mist_file_output.format.file_format = 'PNG'
+ mist_file_output.format.color_mode = 'BW'
+ mist_file_output.format.color_depth = '16'
+
+ links.new(render_layers.outputs['Mist'], mist_file_output.inputs[0])
+
+ outputs['mist'] = mist_file_output
+ composite_nodes.append((render_layers.outputs['Mist'], mist_file_output.inputs[0]))
+
+ return outputs, spec_nodes, composite_nodes
+
+def init_scene() -> None:
+ """Resets the scene to a clean state.
+
+ Returns:
+ None
+ """
+ # delete everything
+ for obj in bpy.data.objects:
+ bpy.data.objects.remove(obj, do_unlink=True)
+
+ # delete all the materials
+ for material in bpy.data.materials:
+ bpy.data.materials.remove(material, do_unlink=True)
+
+ # delete all the textures
+ for texture in bpy.data.textures:
+ bpy.data.textures.remove(texture, do_unlink=True)
+
+ # delete all the images
+ for image in bpy.data.images:
+ bpy.data.images.remove(image, do_unlink=True)
+
+def init_camera():
+ cam = bpy.data.objects.new('Camera', bpy.data.cameras.new('Camera'))
+ bpy.context.collection.objects.link(cam)
+ bpy.context.scene.camera = cam
+ cam.data.sensor_height = cam.data.sensor_width = 32
+ cam_constraint = cam.constraints.new(type='TRACK_TO')
+ cam_constraint.track_axis = 'TRACK_NEGATIVE_Z'
+ cam_constraint.up_axis = 'UP_Y'
+ cam_empty = bpy.data.objects.new("Empty", None)
+ cam_empty.location = (0, 0, 0)
+ bpy.context.scene.collection.objects.link(cam_empty)
+ cam_constraint.target = cam_empty
+ return cam
+
+def init_lighting():
+ # Clear existing lights
+ bpy.ops.object.select_all(action="DESELECT")
+ bpy.ops.object.select_by_type(type="LIGHT")
+ bpy.ops.object.delete()
+
+ # Create key light
+ default_light = bpy.data.objects.new("Default_Light", bpy.data.lights.new("Default_Light", type="POINT"))
+ bpy.context.collection.objects.link(default_light)
+ default_light.data.energy = 1000
+ default_light.location = (4, 1, 6)
+ default_light.rotation_euler = (0, 0, 0)
+
+ # create top light
+ top_light = bpy.data.objects.new("Top_Light", bpy.data.lights.new("Top_Light", type="AREA"))
+ bpy.context.collection.objects.link(top_light)
+ top_light.data.energy = 10000
+ top_light.location = (0, 0, 10)
+ top_light.scale = (100, 100, 100)
+
+ # create bottom light
+ bottom_light = bpy.data.objects.new("Bottom_Light", bpy.data.lights.new("Bottom_Light", type="AREA"))
+ bpy.context.collection.objects.link(bottom_light)
+ bottom_light.data.energy = 1000
+ bottom_light.location = (0, 0, -10)
+ bottom_light.rotation_euler = (0, 0, 0)
+
+ return {
+ "default_light": default_light,
+ "top_light": top_light,
+ "bottom_light": bottom_light
+ }
+
+
+def load_object(object_path: str) -> None:
+ """Loads a model with a supported file extension into the scene.
+
+ Args:
+ object_path (str): Path to the model file.
+
+ Raises:
+ ValueError: If the file extension is not supported.
+
+ Returns:
+ None
+ """
+ file_extension = object_path.split(".")[-1].lower()
+ if file_extension is None:
+ raise ValueError(f"Unsupported file type: {object_path}")
+
+ if file_extension == "usdz":
+ # install usdz io package
+ dirname = os.path.dirname(os.path.realpath(__file__))
+ usdz_package = os.path.join(dirname, "io_scene_usdz.zip")
+ bpy.ops.preferences.addon_install(filepath=usdz_package)
+ # enable it
+ addon_name = "io_scene_usdz"
+ bpy.ops.preferences.addon_enable(module=addon_name)
+ # import the usdz
+ from io_scene_usdz.import_usdz import import_usdz
+
+ import_usdz(context, filepath=object_path, materials=True, animations=True)
+ return None
+
+ # load from existing import functions
+ import_function = IMPORT_FUNCTIONS[file_extension]
+
+ print(f"Loading object from {object_path}")
+ if file_extension == "blend":
+ import_function(directory=object_path, link=False)
+ elif file_extension in {"glb", "gltf"}:
+ import_function(filepath=object_path, merge_vertices=True, import_shading='NORMALS')
+ else:
+ import_function(filepath=object_path)
+
+def delete_invisible_objects() -> None:
+ """Deletes all invisible objects in the scene.
+
+ Returns:
+ None
+ """
+ # bpy.ops.object.mode_set(mode="OBJECT")
+ bpy.ops.object.select_all(action="DESELECT")
+ for obj in bpy.context.scene.objects:
+ if obj.hide_viewport or obj.hide_render:
+ obj.hide_viewport = False
+ obj.hide_render = False
+ obj.hide_select = False
+ obj.select_set(True)
+ bpy.ops.object.delete()
+
+ # Delete invisible collections
+ invisible_collections = [col for col in bpy.data.collections if col.hide_viewport]
+ for col in invisible_collections:
+ bpy.data.collections.remove(col)
+
+def split_mesh_normal():
+ bpy.ops.object.select_all(action="DESELECT")
+ objs = [obj for obj in bpy.context.scene.objects if obj.type == "MESH"]
+ bpy.context.view_layer.objects.active = objs[0]
+ for obj in objs:
+ obj.select_set(True)
+ bpy.ops.object.mode_set(mode="EDIT")
+ bpy.ops.mesh.select_all(action='SELECT')
+ bpy.ops.mesh.split_normals()
+ bpy.ops.object.mode_set(mode='OBJECT')
+ bpy.ops.object.select_all(action="DESELECT")
+
+def delete_custom_normals():
+ for this_obj in bpy.data.objects:
+ if this_obj.type == "MESH":
+ bpy.context.view_layer.objects.active = this_obj
+ bpy.ops.mesh.customdata_custom_splitnormals_clear()
+
+def override_material():
+ new_mat = bpy.data.materials.new(name="Override0123456789")
+ new_mat.use_nodes = True
+ new_mat.node_tree.nodes.clear()
+ bsdf = new_mat.node_tree.nodes.new('ShaderNodeBsdfDiffuse')
+ bsdf.inputs[0].default_value = (0.5, 0.5, 0.5, 1)
+ bsdf.inputs[1].default_value = 1
+ output = new_mat.node_tree.nodes.new('ShaderNodeOutputMaterial')
+ new_mat.node_tree.links.new(bsdf.outputs['BSDF'], output.inputs['Surface'])
+ bpy.context.scene.view_layers['ViewLayer'].material_override = new_mat
+
+def unhide_all_objects() -> None:
+ """Unhides all objects in the scene.
+
+ Returns:
+ None
+ """
+ for obj in bpy.context.scene.objects:
+ obj.hide_set(False)
+
+def convert_to_meshes() -> None:
+ """Converts all objects in the scene to meshes.
+
+ Returns:
+ None
+ """
+ bpy.ops.object.select_all(action="DESELECT")
+ bpy.context.view_layer.objects.active = [obj for obj in bpy.context.scene.objects if obj.type == "MESH"][0]
+ for obj in bpy.context.scene.objects:
+ obj.select_set(True)
+ bpy.ops.object.convert(target="MESH")
+
+def triangulate_meshes() -> None:
+ """Triangulates all meshes in the scene.
+
+ Returns:
+ None
+ """
+ bpy.ops.object.select_all(action="DESELECT")
+ objs = [obj for obj in bpy.context.scene.objects if obj.type == "MESH"]
+ bpy.context.view_layer.objects.active = objs[0]
+ for obj in objs:
+ obj.select_set(True)
+ bpy.ops.object.mode_set(mode="EDIT")
+ bpy.ops.mesh.reveal()
+ bpy.ops.mesh.select_all(action="SELECT")
+ bpy.ops.mesh.quads_convert_to_tris(quad_method="BEAUTY", ngon_method="BEAUTY")
+ bpy.ops.object.mode_set(mode="OBJECT")
+ bpy.ops.object.select_all(action="DESELECT")
+
+def scene_bbox() -> Tuple[Vector, Vector]:
+ """Returns the bounding box of the scene.
+
+ Taken from Shap-E rendering script
+ (https://github.com/openai/shap-e/blob/main/shap_e/rendering/blender/blender_script.py#L68-L82)
+
+ Returns:
+ Tuple[Vector, Vector]: The minimum and maximum coordinates of the bounding box.
+ """
+ bbox_min = (math.inf,) * 3
+ bbox_max = (-math.inf,) * 3
+ found = False
+ scene_meshes = [obj for obj in bpy.context.scene.objects.values() if isinstance(obj.data, bpy.types.Mesh)]
+ for obj in scene_meshes:
+ found = True
+ for coord in obj.bound_box:
+ coord = Vector(coord)
+ coord = obj.matrix_world @ coord
+ bbox_min = tuple(min(x, y) for x, y in zip(bbox_min, coord))
+ bbox_max = tuple(max(x, y) for x, y in zip(bbox_max, coord))
+ if not found:
+ raise RuntimeError("no objects in scene to compute bounding box for")
+ return Vector(bbox_min), Vector(bbox_max)
+
+def normalize_scene() -> Tuple[float, Vector]:
+ """Normalizes the scene by scaling and translating it to fit in a unit cube centered
+ at the origin.
+
+ Mostly taken from the Point-E / Shap-E rendering script
+ (https://github.com/openai/point-e/blob/main/point_e/evals/scripts/blender_script.py#L97-L112),
+ but fix for multiple root objects: (see bug report here:
+ https://github.com/openai/shap-e/pull/60).
+
+ Returns:
+ Tuple[float, Vector]: The scale factor and the offset applied to the scene.
+ """
+ scene_root_objects = [obj for obj in bpy.context.scene.objects.values() if not obj.parent]
+ if len(scene_root_objects) > 1:
+ # create an empty object to be used as a parent for all root objects
+ scene = bpy.data.objects.new("ParentEmpty", None)
+ bpy.context.scene.collection.objects.link(scene)
+
+ # parent all root objects to the empty object
+ for obj in scene_root_objects:
+ obj.parent = scene
+ else:
+ scene = scene_root_objects[0]
+
+ bbox_min, bbox_max = scene_bbox()
+ scale = 1 / max(bbox_max - bbox_min)
+ scene.scale = scene.scale * scale
+
+ # Apply scale to matrix_world.
+ bpy.context.view_layer.update()
+ bbox_min, bbox_max = scene_bbox()
+ offset = -(bbox_min + bbox_max) / 2
+ scene.matrix_world.translation += offset
+ bpy.ops.object.select_all(action="DESELECT")
+
+ return scale, offset
+
+def get_transform_matrix(obj: bpy.types.Object) -> list:
+ pos, rt, _ = obj.matrix_world.decompose()
+ rt = rt.to_matrix()
+ matrix = []
+ for ii in range(3):
+ a = []
+ for jj in range(3):
+ a.append(rt[ii][jj])
+ a.append(pos[ii])
+ matrix.append(a)
+ matrix.append([0, 0, 0, 1])
+ return matrix
+
+
+def main(arg):
+ os.makedirs(arg.output_folder, exist_ok=True)
+
+ if arg.geo_mode:
+ views = trellis_cond_camera_sequence(arg.views)
+ arg.save_mesh = True
+ else:
+ views = orthogonal_camera_sequence()
+ arg.save_albedo = True
+ arg.save_mr = True
+ arg.save_normal = True
+ arg.save_depth = True
+ arg.save_mesh = False
+
+ # Initialize context
+ init_render(engine=arg.engine, resolution=arg.resolution, geo_mode=arg.geo_mode)
+ outputs, spec_nodes, composite_nodes = init_nodes(
+ save_depth=arg.save_depth,
+ save_normal=arg.save_normal,
+ save_albedo=arg.save_albedo,
+ save_mist=arg.save_mist
+ )
+ if arg.object.endswith(".blend"):
+ delete_invisible_objects()
+ else:
+ init_scene()
+ load_object(arg.object)
+ if arg.split_normal:
+ split_mesh_normal()
+ # delete_custom_normals()
+ print('[INFO] Scene initialized.')
+
+ # normalize scene
+ scale, offset = normalize_scene()
+ print('[INFO] Scene normalized.')
+
+ # Initialize camera and lighting
+ cam = init_camera()
+ init_lighting()
+ print('[INFO] Camera and lighting initialized.')
+
+ # Override material
+ #if arg.geo_mode:
+ # override_material()
+
+ # Create a list of views
+ to_export = {
+ "aabb": [[-0.5, -0.5, -0.5], [0.5, 0.5, 0.5]],
+ "scale": scale,
+ "offset": [offset.x, offset.y, offset.z],
+ "frames": []
+ }
+
+ for i, view in enumerate(views):
+ cam.location = (
+ view['cam_dis'] * np.cos(view['hangle']) * np.cos(view['vangle']),
+ view['cam_dis'] * np.sin(view['hangle']) * np.cos(view['vangle']),
+ view['cam_dis'] * np.sin(view['vangle'])
+ )
+ cam.data.lens = 16 / np.tan(view['fov'] / 2)
+
+ if view['proj_type'] == 1:
+ cam.data.type = "ORTHO"
+ cam.data.ortho_scale = 1.2
+
+ bpy.context.scene.render.filepath = os.path.join(arg.output_folder, f'{i:03d}.png')
+ for name, output in outputs.items():
+ output.file_slots[0].path = os.path.join(arg.output_folder, f'{i:03d}_{name}')
+
+ # Render the scene
+ if not arg.geo_mode:
+ switch_to_mr_render(False, composite_nodes)
+ bpy.ops.render.render(write_still=True)
+ shutil.copyfile(bpy.context.scene.render.filepath,
+ bpy.context.scene.render.filepath.replace('.png', '_mr.png'))
+ switch_to_color_render(composite_nodes)
+
+ bpy.ops.render.render(write_still=True)
+ bpy.context.view_layer.update()
+ for name, output in outputs.items():
+ ext = EXT[output.format.file_format]
+ path = glob.glob(f'{output.file_slots[0].path}*.{ext}')[0]
+ os.rename(path, f'{output.file_slots[0].path}.{ext}')
+
+ if not arg.geo_mode:
+ ConvertNormalMap(os.path.join(arg.output_folder, f'{i:03d}_normal.exr'),
+ os.path.join(arg.output_folder, f'{i:03d}_normal.jpg'))
+ ConvertDepthMap(os.path.join(arg.output_folder, f'{i:03d}_depth.exr'),
+ os.path.join(arg.output_folder, f'{i:03d}_pos.jpg'))
+ os.remove(os.path.join(arg.output_folder, f'{i:03d}_normal.exr'))
+ os.remove(os.path.join(arg.output_folder, f'{i:03d}_depth.exr'))
+
+ # Save camera parameters
+ metadata = {
+ "file_path": f'{i:03d}.png',
+ "camera_angle_x": view['fov'],
+ 'proj_type': view['proj_type'],
+ 'azimuth': view['hangle'],
+ 'elevation': view['vangle'],
+ 'cam_dis': view['cam_dis'],
+ "transform_matrix": get_transform_matrix(cam)
+ }
+ to_export["frames"].append(metadata)
+
+ # Save the camera parameters
+ transform_path = os.path.join(arg.output_folder, 'transforms.json')
+ with open(transform_path, 'w') as f:
+ json.dump(to_export, f, indent=4)
+
+ if arg.save_mesh:
+ # triangulate meshes
+ unhide_all_objects()
+ convert_to_meshes()
+ triangulate_meshes()
+ print('[INFO] Meshes triangulated.')
+
+ # export ply mesh
+ bpy.ops.wm.ply_export(filepath=os.path.join(arg.output_folder, 'mesh.ply'),
+ export_triangulated_mesh=True, up_axis='Y',
+ forward_axis='NEGATIVE_Z')
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description='Renders given obj file by rotation a camera around it.')
+ parser.add_argument('--views', type=int, default=24,
+ help='JSON string of views. Contains a list of {yaw, pitch, radius, fov} object.')
+ parser.add_argument('--object', type=str,
+ help='Path to the 3D model file to be rendered.')
+ parser.add_argument('--output_folder', type=str, default='/tmp',
+ help='The path the output will be dumped to.')
+ parser.add_argument('--resolution', type=int, default=512,
+ help='Resolution of the images.')
+ parser.add_argument('--engine', type=str, default='CYCLES',
+ help='Blender internal engine for rendering. E.g. CYCLES, BLENDER_EEVEE, ...')
+ parser.add_argument('--geo_mode', action='store_true',
+ help='Geometry mode for rendering.')
+ parser.add_argument('--save_depth', action='store_true',
+ help='Save the depth maps.')
+ parser.add_argument('--save_normal', action='store_true',
+ help='Save the normal maps.')
+ parser.add_argument('--save_albedo', action='store_true',
+ help='Save the albedo maps.')
+ parser.add_argument('--save_mr', action='store_true',
+ help='Save the MR maps.')
+ parser.add_argument('--save_mist', action='store_true',
+ help='Save the mist distance maps.')
+ parser.add_argument('--split_normal', action='store_true',
+ help='Split the normals of the mesh.')
+ parser.add_argument('--save_mesh', action='store_true',
+ help='Save the mesh as a .ply file.')
+ argv = sys.argv[sys.argv.index("--") + 1:]
+ args = parser.parse_args(argv)
+
+ main(args)
diff --git a/Hunyuan3D-2.1/hy3dshape/tools/watertight/watertight_and_sample.py b/Hunyuan3D-2.1/hy3dshape/tools/watertight/watertight_and_sample.py
new file mode 100644
index 0000000000000000000000000000000000000000..9a1e684774db475931af9078a3cb153d935834c7
--- /dev/null
+++ b/Hunyuan3D-2.1/hy3dshape/tools/watertight/watertight_and_sample.py
@@ -0,0 +1,223 @@
+# Hunyuan 3D is licensed under the TENCENT HUNYUAN NON-COMMERCIAL LICENSE AGREEMENT
+# except for the third-party components listed below.
+# Hunyuan 3D does not impose any additional limitations beyond what is outlined
+# in the repsective licenses of these third-party components.
+# Users must comply with all terms and conditions of original licenses of these third-party
+# components and must ensure that the usage of the third party components adheres to
+# all relevant laws and regulations.
+
+# For avoidance of doubts, Hunyuan 3D means the large language models and
+# their software and algorithms, including trained model weights, parameters (including
+# optimizer states), machine-learning model code, inference-enabling code, training-enabling code,
+# fine-tuning enabling code and other elements of the foregoing made publicly available
+# by Tencent in accordance with TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT.
+
+import argparse
+import igl
+import numpy as np
+import os
+from scipy.stats import truncnorm
+import trimesh
+
+def random_sample_pointcloud(mesh, num = 30000):
+ points, face_idx = mesh.sample(num, return_index=True)
+ normals = mesh.face_normals[face_idx]
+ rng = np.random.default_rng()
+ index = rng.choice(num, num, replace=False)
+ return points[index], normals[index]
+
+def sharp_sample_pointcloud(mesh, num=16384):
+ V = mesh.vertices
+ N = mesh.face_normals
+ VN = mesh.vertex_normals
+ F = mesh.faces
+ VN2 = np.ones(V.shape[0])
+ for i in range(3):
+ dot = np.stack((VN2[F[:,i]], np.sum(VN[F[:,i]] * N, axis=-1)), axis=-1)
+ VN2[F[:,i]] = np.min(dot, axis=-1)
+
+ sharp_mask = VN2<0.985
+ # collect edge
+ edge_a = np.concatenate((F[:,0],F[:,1],F[:,2]))
+ edge_b = np.concatenate((F[:,1],F[:,2],F[:,0]))
+ sharp_edge = ((sharp_mask[edge_a] * sharp_mask[edge_b]))
+ edge_a = edge_a[sharp_edge>0]
+ edge_b = edge_b[sharp_edge>0]
+
+ sharp_verts_a = V[edge_a]
+ sharp_verts_b = V[edge_b]
+ sharp_verts_an = VN[edge_a]
+ sharp_verts_bn = VN[edge_b]
+
+ weights = np.linalg.norm(sharp_verts_b - sharp_verts_a, axis=-1)
+ weights /= np.sum(weights)
+
+ random_number = np.random.rand(num)
+ w = np.random.rand(num,1)
+ index = np.searchsorted(weights.cumsum(), random_number)
+ samples = w * sharp_verts_a[index] + (1 - w) * sharp_verts_b[index]
+ normals = w * sharp_verts_an[index] + (1 - w) * sharp_verts_bn[index]
+ return samples, normals
+
+def sample_sdf(mesh, random_surface, sharp_surface):
+ n_volume_points = sharp_surface.shape[0] * 2
+ vol_points = (np.random.rand(n_volume_points, 3) - 0.5) * 2 * 1.05
+
+ a, b = -0.25, 0.25
+ mu = 0
+
+ # get near points (add offset on surface points)
+ offset1 = truncnorm.rvs((a - mu) / 0.005, (b - mu) / 0.005, loc=mu, scale=0.005, size=(len(random_surface), 3))
+ offset2 = truncnorm.rvs((a - mu) / 0.05, (b - mu) / 0.05, loc=mu, scale=0.05, size=(len(random_surface), 3))
+ random_near_points = np.concatenate([
+ random_surface + offset1,
+ random_surface + offset2
+ ], axis=0)
+
+ unit_num = len(sharp_surface) // 6
+ sharp_near_points = np.concatenate([
+ sharp_surface[:unit_num] + np.random.normal(scale=0.001, size=(unit_num, 3)),
+ sharp_surface[unit_num:unit_num*2] + np.random.normal(scale=0.003, size=(unit_num,3)),
+ sharp_surface[unit_num*2:unit_num*3] + np.random.normal(scale=0.06, size=(unit_num,3)),
+ sharp_surface[unit_num*3:unit_num*4] + np.random.normal(scale=0.01, size=(unit_num,3)),
+ sharp_surface[unit_num*4:unit_num*5] + np.random.normal(scale=0.02, size=(unit_num,3)),
+ sharp_surface[unit_num*5:] + np.random.normal(scale=0.04, size=(len(sharp_surface)-5*unit_num,3))
+ ], axis=0)
+
+ np.random.shuffle(random_near_points)
+ np.random.shuffle(sharp_near_points)
+
+ sign_type = igl.SIGNED_DISTANCE_TYPE_FAST_WINDING_NUMBER
+ try:
+ vol_sdf, I, C = igl.signed_distance(
+ vol_points.astype(np.float32),
+ mesh.vertices, mesh.faces,
+ return_normals=False,
+ sign_type=sign_type)
+ except:
+ vol_sdf, I, C = igl.signed_distance(
+ vol_points.astype(np.float32),
+ mesh.vertices, mesh.faces,
+ return_normals=False)
+ try:
+ random_near_sdf, I, C = igl.signed_distance(
+ random_near_points.astype(np.float32),
+ mesh.vertices, mesh.faces,
+ return_normals=False,
+ sign_type=sign_type)
+ except:
+ random_near_sdf, I, C = igl.signed_distance(
+ random_near_points.astype(np.float32),
+ mesh.vertices, mesh.faces,
+ return_normals=False)
+ try:
+ sharp_near_sdf, I, C = igl.signed_distance(
+ sharp_near_points.astype(np.float32),
+ mesh.vertices, mesh.faces,
+ return_normals=False,
+ sign_type=sign_type)
+ except:
+ sharp_near_sdf, I, C = igl.signed_distance(
+ sharp_near_points.astype(np.float32),
+ mesh.vertices, mesh.faces,
+ return_normals=False)
+
+ vol_label = -vol_sdf
+ random_near_label = -random_near_sdf
+ sharp_near_label = -sharp_near_sdf
+
+ data = {
+ "vol_points": vol_points.astype(np.float16),
+ "vol_label": vol_label.astype(np.float16),
+ "random_near_points": random_near_points.astype(np.float16),
+ "random_near_label": random_near_label.astype(np.float16),
+ "sharp_near_points": sharp_near_points.astype(np.float16),
+ "sharp_near_label": sharp_near_label.astype(np.float16)
+ }
+ return data
+
+def SampleMesh(V, F):
+ mesh = trimesh.Trimesh(vertices=V, faces=F)
+
+ area = mesh.area
+ sample_num = 499712//4
+
+ random_surface, random_normal = random_sample_pointcloud(mesh, num=sample_num)
+ random_sharp_surface, sharp_normal = sharp_sample_pointcloud(mesh, num=sample_num)
+
+ #save_surface
+ surface = np.concatenate((random_surface, random_normal), axis = 1).astype(np.float16)
+ sharp_surface = np.concatenate((random_sharp_surface, sharp_normal), axis=1).astype(np.float16)
+
+ surface_data = {
+ "random_surface": surface,
+ "sharp_surface": sharp_surface,
+ }
+
+ sdf_data = sample_sdf(mesh, random_surface, random_sharp_surface)
+ return surface_data, sdf_data
+
+def normalize_to_unit_box(V):
+ """
+ Normalize the vertices V to fit inside a unit bounding box [0,1]^3.
+ V: (n,3) numpy array of vertex positions.
+ Returns: normalized V
+ """
+ V_min = V.min(axis=0)
+ V_max = V.max(axis=0)
+ scale = (V_max - V_min).max() * 1.01
+ V_normalized = (V - V_min) / scale
+ return V_normalized
+
+# Given: V (n x 3 array of vertices), F (m x 3 array of faces)
+# Parameters epsilon/grid_res
+def Watertight(V, F, epsilon = 2.0/256, grid_res = 256):
+ # Compute bounding box
+ min_corner = V.min(axis=0)
+ max_corner = V.max(axis=0)
+ padding = 0.05 * (max_corner - min_corner)
+ min_corner -= padding
+ max_corner += padding
+
+ # Create a uniform grid
+ x = np.linspace(min_corner[0], max_corner[0], grid_res)
+ y = np.linspace(min_corner[1], max_corner[1], grid_res)
+ z = np.linspace(min_corner[2], max_corner[2], grid_res)
+ X, Y, Z = np.meshgrid(x, y, z, indexing='ij')
+ grid_points = np.vstack([X.ravel(), Y.ravel(), Z.ravel()]).T
+
+ # Compute SDF at grid points using igl.signed_distance with pseudo normals
+ sdf, _, _ = igl.signed_distance(
+ grid_points, V, F, sign_type=igl.SIGNED_DISTANCE_TYPE_PSEUDONORMAL
+ )
+
+ # igl.marching_cubes returns (vertices, faces)
+ mc_verts, mc_faces = igl.marching_cubes(epsilon - np.abs(sdf), grid_points, grid_res, grid_res, grid_res, 0.0)
+
+ # mc_verts: (k x 3) array of vertices of the epsilon contour
+ # mc_faces: (l x 3) array of faces of the epsilon contour
+ return mc_verts, mc_faces
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description='Process an OBJ file and output surface and SDF data.')
+ parser.add_argument('--input_obj', type=str, help='Path to the input OBJ file')
+ parser.add_argument('--output_prefix', type=str, default=None,
+ help='Base name for output files (default: input OBJ filename without extension)')
+ args = parser.parse_args()
+
+ input_obj = args.input_obj
+ name = args.output_prefix
+
+ V, F = igl.read_triangle_mesh(input_obj)
+ V = normalize_to_unit_box(V)
+
+ mc_verts, mc_faces = Watertight(V, F)
+ surface_data, sdf_data = SampleMesh(mc_verts, mc_faces)
+
+ parent_folder = os.path.dirname(args.output_prefix)
+ os.makedirs(parent_folder, exist_ok=True)
+ export_surface = f'{name}_surface.npz'
+ np.savez(export_surface, **surface_data)
+ export_sdf = f'{name}_sdf.npz'
+ np.savez(export_sdf, **sdf_data)
+ igl.write_obj(f'{name}_watertight.obj', mc_verts, mc_faces)
diff --git a/Hunyuan3D-2.1/requirements.txt b/Hunyuan3D-2.1/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..7225d5d78602440a75a790991d07dc1332fcac2d
--- /dev/null
+++ b/Hunyuan3D-2.1/requirements.txt
@@ -0,0 +1,70 @@
+--extra-index-url https://download.blender.org/pypi/
+
+# Build Tools
+ninja==1.11.1.1
+pybind11==2.13.4
+
+# Core ML/Deep Learning
+transformers==4.46.0
+diffusers==0.30.0
+accelerate==1.1.1
+pytorch-lightning==1.9.5
+huggingface-hub==0.30.2
+safetensors==0.4.4
+
+# Scientific Computing
+numpy==1.24.4
+scipy==1.14.1
+einops==0.8.0
+pandas==2.2.2
+
+# Computer Vision & Image Processing
+opencv-python==4.10.0.84
+imageio==2.36.0
+scikit-image==0.24.0
+rembg==2.0.65
+realesrgan==0.3.0
+tb_nightly==2.18.0a20240726
+basicsr==1.4.2
+
+# 3D Mesh Processing
+trimesh==4.4.7
+pymeshlab==2022.2.post3
+pygltflib==1.16.3
+xatlas==0.0.9
+open3d==0.18.0
+
+# Configuration Management
+omegaconf==2.3.0
+pyyaml==6.0.2
+configargparse==1.7
+
+# Web Framework (for demo)
+gradio==5.33.0
+fastapi==0.115.12
+uvicorn==0.34.3
+
+# Utilities
+tqdm==4.66.5
+psutil==6.0.0
+
+# GPU Computing (requires CUDA)
+cupy-cuda12x==13.4.1
+
+# Blender
+bpy==4.0
+
+# ONNX Runtime
+onnxruntime==1.16.3
+torchmetrics==1.6.0
+
+timm
+pythreejs
+
+# torchdiffeq
+# deepspeed
+# torch_cluster
+
+pydantic==2.10.6
+
+
diff --git a/Hunyuan3D-2.1/torchvision_fix.py b/Hunyuan3D-2.1/torchvision_fix.py
new file mode 100644
index 0000000000000000000000000000000000000000..4ea6fb43fb61e3d1f201f78dfe0ef2a25a44b4e8
--- /dev/null
+++ b/Hunyuan3D-2.1/torchvision_fix.py
@@ -0,0 +1,104 @@
+# Torchvision compatibility fix for functional_tensor module
+# This file helps resolve compatibility issues between different torchvision versions
+
+import sys
+import torchvision
+
+def fix_torchvision_functional_tensor():
+ """
+ Fix torchvision.transforms.functional_tensor import issue
+ """
+ try:
+ # Check if the module exists in the expected location
+ import torchvision.transforms.functional_tensor
+ print("torchvision.transforms.functional_tensor is available")
+ return True
+ except ImportError:
+ print("torchvision.transforms.functional_tensor not found, applying compatibility fix...")
+
+ try:
+ # Create a mock functional_tensor module with the required functions
+ import torch
+ import torchvision.transforms.functional as F
+
+ class FunctionalTensorMock:
+ """Mock module to replace functional_tensor"""
+
+ @staticmethod
+ def rgb_to_grayscale(img, num_output_channels=1):
+ """Convert RGB image to grayscale"""
+ if hasattr(F, 'rgb_to_grayscale'):
+ return F.rgb_to_grayscale(img, num_output_channels)
+ else:
+ # Fallback implementation
+ if len(img.shape) == 4: # Batch of images
+ weights = torch.tensor([0.299, 0.587, 0.114],
+ device=img.device, dtype=img.dtype).view(1, 3, 1, 1)
+ else: # Single image
+ weights = torch.tensor([0.299, 0.587, 0.114],
+ device=img.device, dtype=img.dtype).view(3, 1, 1)
+
+ grayscale = torch.sum(img * weights, dim=-3, keepdim=True)
+
+ if num_output_channels == 3:
+ if len(img.shape) == 4:
+ grayscale = grayscale.repeat(1, 3, 1, 1)
+ else:
+ grayscale = grayscale.repeat(3, 1, 1)
+
+ return grayscale
+
+ @staticmethod
+ def resize(img, size, interpolation=2, antialias=None):
+ """Resize function wrapper"""
+ try:
+ from torchvision.transforms.v2.functional import resize as v2_resize
+ return v2_resize(img, size,
+ interpolation=interpolation,
+ antialias=antialias)
+ except ImportError:
+ if hasattr(F, 'resize'):
+ return F.resize(img, size, interpolation=interpolation)
+ else:
+ import torch.nn.functional as torch_F
+ if isinstance(size, int):
+ size = (size, size)
+ return torch_F.interpolate(
+ img.unsqueeze(0) if len(img.shape) == 3 else img,
+ size=size, mode='bilinear', align_corners=False)
+
+ def __getattr__(self, name):
+ """Fallback to regular functional module"""
+ if hasattr(F, name):
+ return getattr(F, name)
+ else:
+ # Try v2.functional
+ try:
+ import torchvision.transforms.v2.functional as v2_F
+ if hasattr(v2_F, name):
+ return getattr(v2_F, name)
+ except ImportError:
+ pass
+
+ raise AttributeError(f"'{name}' not found in functional_tensor mock")
+
+ # Create the mock module instance
+ functional_tensor_mock = FunctionalTensorMock()
+
+ # Monkey patch the old location
+ sys.modules['torchvision.transforms.functional_tensor'] = functional_tensor_mock
+ print("Applied compatibility fix: created functional_tensor mock module")
+ return True
+
+ except Exception as e:
+ print(f"Failed to create functional_tensor mock: {e}")
+ return False
+
+def apply_fix():
+ """Apply the torchvision compatibility fix"""
+ print(f"Torchvision version: {torchvision.__version__}")
+ return fix_torchvision_functional_tensor()
+
+if __name__ == "__main__":
+ apply_fix()
+
\ No newline at end of file
diff --git a/PLAN.md b/PLAN.md
new file mode 100644
index 0000000000000000000000000000000000000000..8b5a7b33d7726e2f86570981b6ece0c88dcee807
--- /dev/null
+++ b/PLAN.md
@@ -0,0 +1,551 @@
+# Open3DForge — Full Build Plan
+
+> **Project:** Personal image-to-game-ready 3D asset pipeline
+> **Owner:** Basel · Solo dev for "What Remains" (UE5.7)
+> **Hosting:** Single HF Pro Space, ZeroGPU H200, 25 min/day quota
+> **SDK:** Gradio 6.x (currently 6.14.0)
+> **Repo:** `Reverb/open3dforge`
+
+---
+
+## Architectural Decisions (Locked In)
+
+These were debated and decided during early development. Don't relitigate them mid-build.
+
+1. **One Space, not multiple.** Option B: vendor TRELLIS.2 + Hunyuan3D-2 + UniRig into this Space rather than orchestrating across multiple Spaces via `gradio_client`. Larger repo, but single deployment, no inter-Space latency, no auth juggling.
+
+2. **HF Space only, no local fallback.** Don't build a path for running on the RTX 4070. Quota is enough for personal use.
+
+3. **Gradio 6.x.** Match the `sdk_version` in `README.md`. No upper bound in `requirements.txt`.
+
+4. **UE5-first export defaults.** DirectX normals, ORM packing, cm units, Z-up, `SM_/SK_/T_` naming.
+
+5. **Drop the custom website.** Standard Gradio tabs. `gradio.Server` is not worth the work for one user.
+
+6. **Drop CHORD.** Research-only license. Use TRELLIS.2's own metallic/roughness volume attributes instead, which are already correct and license-compatible.
+
+7. **nvdiffrast for all baking.** Not Blender headless. Fast (~2-5s per bake), GPU-based, fits inside `@spaces.GPU`, no apt install needed beyond what TRELLIS already requires.
+
+8. **Solo-user workspace pattern.** One `workspace/current/` folder. No session IDs, no multi-tenancy.
+
+---
+
+## Reference Code: Pipeline Stage Order
+
+```
+INPUT: 1-4 images
+ │
+ ├── rembg (background removal, CPU)
+ │
+ ▼
+[Stage 1] GENERATION [GPU]
+ ├── TRELLIS.2 (hard surface) or Hunyuan3D-2 (organic)
+ ├── SAVE high_poly.glb (kept for normal baking)
+ └── Extract: albedo + metallic + roughness volume attrs
+ │
+ ▼
+[Stage 2] POST-PROCESSING [CPU + GPU baking]
+ 2A Mesh repair pymeshfix CPU
+ 2B Geometry cleanup PyMeshLab CPU
+ 2C Decimation CPU
+ ├── Preview: fast-simplification
+ └── Final: PyMeshLab quality pass
+ 2D Symmetry (characters) PyMeshLab CPU
+ 2E UV unwrap xatlas CPU
+ (texels_per_unit packing)
+ 2F Normal bake nvdiffrast GPU
+ DX + GL outputs
+ 2G Albedo bake nvdiffrast GPU
+ vertex color → UV atlas
+ 2H Material bake nvdiffrast GPU
+ TRELLIS volume attrs → UV
+ 2I AO bake nvdiffrast GPU
+ ray-occlusion → UV
+ 2J Texture inpaint (opt) SDXL inpaint GPU
+ hidden UV regions
+ 2K Channel pack numpy CPU
+ Unreal ORM / Unity MetSmooth
+ 2L LOD generation PyMeshLab CPU
+ LOD0/LOD1/LOD2 only (UE5 HLOD handles billboards)
+ 2M Collision mesh CoACD CPU
+ 2N Pivot correction trimesh CPU
+ 2O Scale validation trimesh CPU
+ │
+ ▼
+[Stage 3] AUTO-RIGGING (Optional) [GPU]
+ UniRig → rigged.glb / rigged.fbx
+ │
+ ▼
+[Stage 4] EXPORT
+ UE5 preset (default) → DX normals + ORM
+ Naming: SM_/SK_/T_ convention
+ → export_AssetName_UE5.zip
+```
+
+---
+
+## Milestone Plan
+
+Each milestone is sized to be one focused work session. Push at the end of each, verify, then move on.
+
+---
+
+### ✅ Milestone 1 — Foundation (COMPLETE)
+
+**Status:** Deployed and verified. ZeroGPU smoke test passes.
+
+What got built:
+- HF Space scaffolded with Gradio 6 + ZeroGPU
+- 5 tabs: Generate / Post-Process / Auto-Rig / Export / Presets + Diagnostics
+- `workspace/` folder pattern (current/exports/presets/history)
+- `src/workspace.py` — AssetState, preset save/load
+- `src/quota.py` — daily quota tracking
+- `src/ui_helpers.py` — status bar, asset summary, viewer model picker
+- `gr.Model3D` viewer wired up
+- Pipeline stubs returning placeholder messages
+- Diagnostics tab with `@spaces.GPU` smoke test
+
+Files in repo:
+```
+open3dforge/
+├── README.md ← sdk_version: 6.14.0
+├── requirements.txt ← gradio>=5.0, spaces, numpy, pillow
+├── .gitignore
+├── app.py ← main entry, all UI wiring
+├── src/
+│ ├── __init__.py
+│ ├── workspace.py
+│ ├── quota.py
+│ └── ui_helpers.py
+└── workspace/
+ ├── current/.gitkeep
+ ├── exports/.gitkeep
+ ├── presets/.gitkeep
+ └── history/.gitkeep
+```
+
+---
+
+### 🟡 Milestone 2 — Stage 1: TRELLIS.2 Generation (NEXT)
+
+**Goal:** Real image-to-3D generation working end-to-end. Upload image → get a GLB in the viewer.
+
+**Approach:** Option B — duplicate the microsoft/TRELLIS.2 Space, merge its contents into our repo, then refactor app.py to integrate with our tab structure.
+
+**Step-by-step:**
+
+1. **Duplicate microsoft/TRELLIS.2 Space** to get a known-good baseline:
+ - On HF: `huggingface.co/spaces/microsoft/TRELLIS.2` → Duplicate this Space → name it `open3dforge-trellis-staging`
+ - This is a *staging* copy — we don't deploy it, we just clone it locally for the merge
+ - Confirm it builds and runs in your duplicate before touching anything
+
+2. **Clone both repos locally:**
+ ```bash
+ git clone https://huggingface.co/spaces/Reverb/open3dforge
+ git clone https://huggingface.co/spaces/baselanaya/open3dforge-trellis-staging
+ ```
+
+3. **Copy TRELLIS.2 assets into open3dforge:**
+ ```bash
+ cp -r open3dforge-trellis-staging/trellis2/ open3dforge/
+ cp -r open3dforge-trellis-staging/assets/ open3dforge/
+ cp open3dforge-trellis-staging/autotune_cache.json open3dforge/
+ cp open3dforge-trellis-staging/packages.txt open3dforge/
+ ```
+ This gives us the vendored `trellis2/` Python package, HDRI envmaps, FlexGemm cache, and apt deps.
+
+4. **Merge requirements.txt:**
+ Combine the TRELLIS.2 requirements with our existing ones. Add to `requirements.txt`:
+ ```
+ # TRELLIS.2 deps (from microsoft/TRELLIS.2 Space)
+ torch
+ torchvision
+ cv2 / opencv-python-headless
+ imageio
+ imageio-ffmpeg
+ rembg
+ # plus the custom wheels they install at build time
+ ```
+ Copy theirs verbatim and add to ours. Inspect the resolved `requirements.txt` in the staging duplicate first.
+
+5. **Refactor app.py to integrate the TRELLIS handlers:**
+ - Move TRELLIS pipeline init to module level (per ZeroGPU rules — must be on CUDA at module-level)
+ - Wrap their `image_to_3d` + `extract_glb` functions as the implementation of our existing `stub_generate` handler
+ - Update the Generate tab to match TRELLIS parameter names (resolution, ss_sampling_steps, etc.)
+ - Hide most TRELLIS knobs behind the "Advanced" accordion; expose only Quality preset + Seed at top level
+ - Keep our quality presets (Fast/Balanced/Hero) mapping to their parameter sets
+ - Hook the output GLB into `workspace.get_state().raw_gen_glb` and save the high-poly separately
+
+6. **Critical: save `high_poly.glb` before decimation.** TRELLIS's `extract_glb` calls `o_voxel.postprocess.to_glb(decimation_target=...)`. We need to call it once with no decimation (or a very high target like 16M faces — the nvdiffrast limit they use) to get the high-poly we'll bake from in Stage 2, then call it again with the user's chosen decimation_target for the working low-poly.
+
+7. **Update workspace state on success:**
+ ```python
+ state = workspace.get_state()
+ state.high_poly_glb = Path("workspace/current/high_poly.glb")
+ state.raw_gen_glb = Path("workspace/current/raw_gen.glb")
+ state.face_count = len(mesh.faces)
+ state.vertex_count = len(mesh.vertices)
+ state.model_used = "TRELLIS.2"
+ ```
+
+8. **Test:**
+ - Push to Space
+ - Wait for build (~10-15 min due to CUDA wheels compiling)
+ - Upload a test image
+ - Confirm the GLB appears in the viewer
+ - Confirm Diagnostics quota tracker shows time consumed
+
+**Quality presets to wire up (map to TRELLIS params):**
+
+| Preset | resolution | ss_steps | shape_steps | tex_steps | Expected time |
+|---|---|---|---|---|---|
+| Fast | 512 | 8 | 8 | 8 | ~30s |
+| Balanced | 1024 | 12 | 12 | 12 | ~60s |
+| Hero | 1536 | 16 | 16 | 16 | ~90s |
+
+**Risk mitigation:**
+- TRELLIS.2 build can fail in many ways (CUDA wheel compilation, flash-attn install). If a build fails, check the build logs for which wheel failed. The staging duplicate is the reference — if it built there, the issue is in *your* merge.
+- Don't move anything into `@spaces.GPU` functions that should be at module level. Pipeline init goes at module level.
+
+---
+
+### Milestone 2b — Hunyuan3D-2 Alternative Generator
+
+**Goal:** Second generator option for organic shapes (characters, creatures).
+
+**Approach:** Same duplicate-and-vendor pattern as Milestone 2.
+
+1. Duplicate `tencent/Hunyuan3D-2` to staging Space
+2. Clone, copy the `hy3dgen/` package into open3dforge
+3. Merge requirements (most overlap with TRELLIS.2 — torch, diffusers)
+4. Add Hunyuan pipeline init at module level
+5. The model dropdown in the Generate tab routes between `image_to_3d_trellis()` and `image_to_3d_hunyuan()`
+6. Hunyuan needs 16GB VRAM — fits alongside TRELLIS in H200's 70GB but only load one at a time via lazy module-level guards
+
+**Decision deferred to this milestone:** Whether to keep both models in VRAM at module load (faster, more memory) or lazy-load per call (slower first call, less memory). Test both.
+
+---
+
+### Milestone 3 — Stage 2A-2C: Mesh Cleanup
+
+**Goal:** Working CPU-side mesh repair, cleanup, and decimation with live preview.
+
+**Dependencies to add:**
+```
+trimesh[easy]
+pymeshfix
+pymeshlab
+fast-simplification
+```
+
+**Files to create:**
+- `src/stages/__init__.py`
+- `src/stages/stage2_repair.py` — pymeshfix wrapper
+- `src/stages/stage2_cleanup.py` — PyMeshLab filter chain
+- `src/stages/stage2_decimate.py` — both fast-simplification (preview) and PyMeshLab (final)
+
+**UI work in app.py:**
+- Wire the existing checkboxes/sliders in Tab 2 to call the real implementations
+- Live preview: slider `.change()` event fires `fast-simplification`, updates face count display
+- Run button: actually runs full pipeline on the current GLB
+
+**Workspace state updates:**
+- `state.repaired_glb`, `state.cleaned_glb`, `state.low_poly_glb` all get populated as steps complete
+
+**Test criteria:**
+- Generate a TRELLIS asset (50k faces)
+- Run repair → no errors
+- Run cleanup → no errors
+- Set decimation slider to 10k → live preview updates face count
+- Click "Run final" → produces low_poly.glb at 10k faces
+- Viewer auto-refreshes to show the cleaned mesh
+
+---
+
+### Milestone 4 — Stage 2D-2E: Symmetry + UV Unwrap
+
+**Goal:** Symmetry enforcement + xatlas UV unwrapping with consistent texel density.
+
+**Dependencies to add:**
+```
+xatlas
+```
+
+**Files to create:**
+- `src/stages/stage2_symmetry.py` — PyMeshLab `apply_filter_mesh_symmetrize`
+- `src/stages/stage2_uv.py` — xatlas with `texels_per_unit` packing
+
+**UI work:**
+- Symmetry: off / bilateral-X / bilateral-Y / radial dropdown
+- UV: atlas resolution, texels_per_unit, padding
+
+**Test criteria:**
+- Run on a human-character GLB → symmetry produces clean mirror
+- UV unwrap produces `unwrapped.glb` with valid UV0 coords visible if you inspect via trimesh
+- No overlapping UV islands (check with PyMeshLab's quality measure)
+
+---
+
+### Milestone 5 — Stage 2F: Normal Baking with nvdiffrast
+
+**Goal:** High-poly → low-poly normal map baking, GPU-accelerated, 2-5 second bakes.
+
+**Dependencies to add:**
+- `nvdiffrast` (already installed via TRELLIS.2 wheels — verify in the staging duplicate)
+
+**Files to create:**
+- `src/stages/stage2_bake_normal.py` — full nvdiffrast pipeline
+
+**Algorithm (from the plan doc):**
+```python
+@spaces.GPU(duration=60)
+def bake_normal_map(high_poly_path, low_poly_path, uv_coords, map_size=2048):
+ ctx = dr.RasterizeCudaContext()
+ # 1. UV → clip space
+ # 2. Rasterize low-poly UVs → per-pixel world position + tri ID
+ # 3. For each pixel: nearest-on-surface from high-poly
+ # 4. Sample high-poly normal at that point
+ # 5. Transform to tangent space (low-poly tangent frame)
+ # 6. Pack RGB [0,1], save PNG
+ # 7. Dilate edges past UV island boundaries
+```
+
+**Output:** Two PNGs — `normal_gl.png` and `normal_dx.png` (DX has Y-flipped green channel).
+
+**Test criteria:**
+- Run on TRELLIS character output (50k high-poly → 10k low-poly)
+- Bake completes in <10 seconds
+- Open the normal map in any image viewer — should be bluish/purple with surface detail visible
+- Both DX and GL versions are produced
+- Quota shows 5-10 seconds consumed
+
+---
+
+### Milestone 6 — Stage 2G-2I: Albedo, Material, AO Baking
+
+**Goal:** Three more nvdiffrast bakes producing the full PBR texture set.
+
+**Files to create:**
+- `src/stages/stage2_bake_albedo.py`
+- `src/stages/stage2_bake_material.py` — uses TRELLIS.2's stored metallic+roughness attrs
+- `src/stages/stage2_bake_ao.py` — ray-occlusion in hemisphere
+
+**Key reuse:** Same nvdiffrast rasterization pattern as Milestone 5 — refactor that code into a shared helper `_rasterize_uv_atlas()` in `src/stages/_baking_helpers.py`.
+
+**Workspace state:** All texture paths populated on the AssetState.
+
+**Test criteria:**
+- All four maps (normal, albedo, metallic, roughness, AO) viewable as PNG thumbnails in Tab 2
+- Total Stage 2 baking time < 30 seconds for a Balanced-quality asset
+
+---
+
+### Milestone 7 — Stage 2J: SDXL Inpainting for Hidden UVs
+
+**Goal:** Detect stretched/synthetic UV regions and inpaint them with SDXL.
+
+**Dependencies to add:**
+```
+diffusers
+accelerate
+safetensors
+```
+
+**Files to create:**
+- `src/stages/stage2_inpaint.py`
+ - `detect_hidden_regions(albedo, uvs, faces)` — variance analysis
+ - `inpaint_hidden_uvs(...)` — SDXL inpainting pipeline
+
+**UI:** Toggle off by default (costs ~30s quota). Prompt input. Strength slider.
+
+**Test criteria:**
+- Generate an asset with a clear "back side" (e.g., a humanoid character)
+- Without inpainting: back of character has visible texture stretching
+- With inpainting: back is plausibly filled in
+- Quota cost: ~30s per inpaint
+
+---
+
+### Milestone 8 — Stage 2K-2O: Finalization Steps
+
+**Goal:** Channel packing, LODs, collision, pivot, scale — all CPU-side, fast.
+
+**Dependencies to add:**
+```
+coacd==1.0.4
+```
+
+**Files to create:**
+- `src/stages/stage2_channel_pack.py` — numpy ORM / MetallicSmoothness packing
+- `src/stages/stage2_lods.py` — PyMeshLab quality-aware LOD0/1/2
+- `src/stages/stage2_collision.py` — CoACD with `trimesh.convex_hull` fallback
+- `src/stages/stage2_pivot.py` — bottom_center / geometric_center / custom
+- `src/stages/stage2_scale.py` — height presets, UE5 cm units
+
+**UI:** All controls already scaffolded in Milestone 1's Post-Process tab. Just wire to real implementations.
+
+**Test criteria:**
+- ORM packed as RGB with AO/Roughness/Metallic in correct channels
+- LOD0/LOD1/LOD2 all generated, all share same UV layout
+- Collision mesh has <1% the triangle count of LOD0
+- Pivot at bottom_center for a generated human character results in feet at world origin Y=0
+- Scale: human asset is 1.8m tall = 180cm in UE5 export
+
+---
+
+### Milestone 9 — Stage 3: UniRig Auto-Rigging
+
+**Goal:** Generate a skeleton + skinning weights for character meshes.
+
+**Approach:** Same vendor-the-Space pattern as Milestone 2.
+
+1. Duplicate `MohamedRashad/UniRig` Space → staging
+2. Verify it builds in the staging duplicate
+3. Copy `UniRig/` package into our repo
+4. Merge requirements
+5. Wire to the Auto-Rig tab handler
+6. Output: rigged FBX (UE5 default) or GLB
+
+**Test criteria:**
+- Run on a humanoid character (after full Stage 2 processing)
+- Output FBX imports into UE5 as a Skeletal Mesh
+- Drag into Mixamo → animations auto-attach correctly
+
+---
+
+### Milestone 10 — Stage 4: UE5 Export
+
+**Goal:** Bundle everything into a UE5-ready zip with proper naming and packing.
+
+**Dependencies to add:**
+```
+pygltflib
+```
+
+**Files to create:**
+- `src/stages/stage4_export.py`
+ - `export_ue5(asset_state, asset_name, asset_type) → zip_path`
+ - Handles FBX conversion via trimesh
+ - Applies naming convention (`SM_`, `SK_`, `T_`)
+ - Writes ORM-packed textures to correct paths
+ - Zip + drop in `workspace/exports/`
+
+**Engine presets (only UE5 fully implemented):**
+- UE5: FBX, DX normals, ORM, Z-up, cm — the default
+- Unity HDRP: FBX, GL normals, MetallicSmoothness, Y-up, m — stub for later
+- Godot/Blender/Web: stubs
+
+**Test criteria:**
+- Export a character → unzip → 6-7 files following naming convention
+- Import to UE5: drag-drop the zip's contents → no warnings, materials auto-create from textures
+- Both Static Mesh and Skeletal Mesh paths work
+
+---
+
+### Milestone 11 — Presets System
+
+**Goal:** Save and load named parameter configurations across tabs.
+
+**Files to update:**
+- `src/workspace.py` — already has `save_preset/load_preset/delete_preset`, just needs the JSON schema fleshed out
+- `app.py` — wire the Presets tab's Save button to actually read all current tab values
+
+**Schema:**
+```json
+{
+ "name": "character_UE5_hero",
+ "stage1": { ... },
+ "stage2": { ... },
+ "stage3": { ... },
+ "stage4": { ... }
+}
+```
+
+**Ship five default presets:**
+- `character_UE5_hero.json`
+- `character_UE5_npc.json`
+- `prop_UE5_hero.json`
+- `prop_UE5_standard.json`
+- `environment_UE5_background.json`
+
+---
+
+### Milestone 12 — Polish & Production Hardening
+
+- Error handling on every stage (don't crash the app, show clear error in UI)
+- Progress bars during long ops (`gr.Progress(track_tqdm=True)`)
+- Quota cost shown *before* each GPU operation (warning if it would exceed remaining)
+- Game-ready checklist passes shown before allowing Export
+- Asset history sidebar (last 5 generated assets with thumbnails)
+- Session cleanup of `workspace/current/` on new generation
+
+---
+
+## Working with Claude Code
+
+When you continue in Claude Code, you'll have the full repo locally. Key things to remember:
+
+### Project conventions
+
+1. **Each stage = its own module** in `src/stages/`. Don't dump pipeline logic into `app.py`.
+2. **Workspace state is the single source of truth.** Every stage reads from and writes to `workspace.get_state()`.
+3. **GPU functions live where they're needed**, not all in app.py. The `@spaces.GPU` decorator works in any file as long as `spaces` is imported.
+4. **No `if __name__ == "__main__":` on `demo.launch()`.** HF Spaces imports app.py at module level.
+5. **Gradio 6 specifics:**
+ - `theme` and `css` go in `launch()`, not `Blocks()`
+ - `show_api` is gone — use `footer_links=["gradio", "settings"]`
+ - `api_visibility` replaces `api_name=False` on events
+6. **The 3 global components** (`viewer`, `summary`, `status_bar`) get refreshed via `_global_refresh()` chained off every pipeline action button. Don't forget to add new buttons to that list.
+
+### Useful commands
+
+```bash
+# Pull the latest Space state
+cd open3dforge
+git pull
+
+# Make changes, syntax-check before push
+python -c "import ast; ast.parse(open('app.py').read())"
+
+# Push to deploy
+git add -A
+git commit -m "Milestone N: ● Render Mode - Click on the circular buttons to switch between different render modes.
+● View Angle - Drag the slider to change the view angle.
+