Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Sync from huggingface/lerobot-dataset-visualizer via hub-sync
Browse files- src/components/urdf-viewer.tsx +148 -94
src/components/urdf-viewer.tsx
CHANGED
|
@@ -20,6 +20,8 @@ import URDFLoader from "urdf-loader";
|
|
| 20 |
import type { URDFRobot } from "urdf-loader";
|
| 21 |
import { STLLoader } from "three/examples/jsm/loaders/STLLoader.js";
|
| 22 |
import { ColladaLoader } from "three/examples/jsm/loaders/ColladaLoader.js";
|
|
|
|
|
|
|
| 23 |
import { Line2 } from "three/examples/jsm/lines/Line2.js";
|
| 24 |
import { LineMaterial } from "three/examples/jsm/lines/LineMaterial.js";
|
| 25 |
import { LineGeometry } from "three/examples/jsm/lines/LineGeometry.js";
|
|
@@ -57,6 +59,69 @@ function loadStlGeometry(url: string): Promise<THREE.BufferGeometry> {
|
|
| 57 |
return loading;
|
| 58 |
}
|
| 59 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
/// One download and parse per DAE file per session: the OpenArm URDF references finger.dae 4 times
|
| 61 |
/// and each arm link twice, and the viewer remounts on every visit to the 3D Replay tab.
|
| 62 |
const daeSceneCache = new Map<string, Promise<THREE.Object3D>>();
|
|
@@ -87,62 +152,7 @@ function loadDaeScene(url: string): Promise<THREE.Object3D> {
|
|
| 87 |
});
|
| 88 |
for (const l of strayLights) l.parent?.remove(l);
|
| 89 |
|
| 90 |
-
collada.scene
|
| 91 |
-
if (!(child instanceof THREE.Mesh) || !child.material) return;
|
| 92 |
-
|
| 93 |
-
const originals = Array.isArray(child.material)
|
| 94 |
-
? child.material
|
| 95 |
-
: [child.material];
|
| 96 |
-
|
| 97 |
-
const rebuilt = originals.map((orig) => {
|
| 98 |
-
const srcColor =
|
| 99 |
-
(orig as THREE.MeshStandardMaterial).color ??
|
| 100 |
-
new THREE.Color("#c0c4cc");
|
| 101 |
-
const hsl = { h: 0, s: 0, l: 0 };
|
| 102 |
-
srcColor.getHSL(hsl);
|
| 103 |
-
|
| 104 |
-
// Archetype classification by original lightness.
|
| 105 |
-
let color: THREE.Color;
|
| 106 |
-
let metalness: number;
|
| 107 |
-
let roughness: number;
|
| 108 |
-
let envMapIntensity: number;
|
| 109 |
-
if (hsl.l < 0.3) {
|
| 110 |
-
// Carbon / anodised structural parts
|
| 111 |
-
color = new THREE.Color().setHSL(hsl.h, 0.02, 0.09);
|
| 112 |
-
metalness = 0.15;
|
| 113 |
-
roughness = 0.75;
|
| 114 |
-
envMapIntensity = 0.6;
|
| 115 |
-
} else if (hsl.l < 0.7) {
|
| 116 |
-
// Brushed metal joint collars / accents
|
| 117 |
-
color = new THREE.Color().setHSL(hsl.h, 0.04, 0.42);
|
| 118 |
-
metalness = 0.75;
|
| 119 |
-
roughness = 0.35;
|
| 120 |
-
envMapIntensity = 1.1;
|
| 121 |
-
} else {
|
| 122 |
-
// Off-white painted plates
|
| 123 |
-
color = new THREE.Color().setHSL(hsl.h, 0.03, 0.6);
|
| 124 |
-
metalness = 0.1;
|
| 125 |
-
roughness = 0.5;
|
| 126 |
-
envMapIntensity = 0.9;
|
| 127 |
-
}
|
| 128 |
-
|
| 129 |
-
const mat = new THREE.MeshStandardMaterial({
|
| 130 |
-
color,
|
| 131 |
-
metalness,
|
| 132 |
-
roughness,
|
| 133 |
-
envMapIntensity,
|
| 134 |
-
side: THREE.DoubleSide,
|
| 135 |
-
});
|
| 136 |
-
orig.dispose?.();
|
| 137 |
-
return mat;
|
| 138 |
-
});
|
| 139 |
-
|
| 140 |
-
child.material = Array.isArray(child.material)
|
| 141 |
-
? rebuilt
|
| 142 |
-
: rebuilt[0];
|
| 143 |
-
child.castShadow = true;
|
| 144 |
-
child.receiveShadow = true;
|
| 145 |
-
});
|
| 146 |
}
|
| 147 |
return collada.scene;
|
| 148 |
})
|
|
@@ -155,6 +165,30 @@ function loadDaeScene(url: string): Promise<THREE.Object3D> {
|
|
| 155 |
return loading;
|
| 156 |
}
|
| 157 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
// URDFs + meshes are hosted in the Hub bucket at
|
| 159 |
// https://huggingface.co/buckets/lerobot/robot-urdfs. URDFLoader resolves
|
| 160 |
// relative mesh paths against the URDF's own URL, so the bucket layout
|
|
@@ -189,9 +223,9 @@ function loadUrdfText(url: string): Promise<string> {
|
|
| 189 |
}
|
| 190 |
|
| 191 |
/**
|
| 192 |
-
* Starts downloading a robot's visual
|
| 193 |
-
*
|
| 194 |
-
*
|
| 195 |
*/
|
| 196 |
export function prefetchRobotModel(robotType: string | null) {
|
| 197 |
const { urdfUrl } = getRobotConfig(robotType);
|
|
@@ -205,8 +239,11 @@ export function prefetchRobotModel(robotType: string | null) {
|
|
| 205 |
const filename = mesh.getAttribute("filename");
|
| 206 |
/// Same resolution URDFLoader applies to relative paths; package:// and DAE meshes are left to it.
|
| 207 |
if (!filename || filename.startsWith("package://")) continue;
|
| 208 |
-
|
| 209 |
-
|
|
|
|
|
|
|
|
|
|
| 210 |
}
|
| 211 |
})
|
| 212 |
.catch(() => prefetchedUrdfs.delete(urdfUrl));
|
|
@@ -219,12 +256,12 @@ function getRobotConfig(robotType: string | null) {
|
|
| 219 |
}
|
| 220 |
if (lower.includes("openarm")) {
|
| 221 |
return {
|
| 222 |
-
urdfUrl: `${URDF_BASE_URL}/openarm/
|
| 223 |
scale: 3,
|
| 224 |
};
|
| 225 |
}
|
| 226 |
return {
|
| 227 |
-
urdfUrl: `${URDF_BASE_URL}/so101/
|
| 228 |
scale: 10,
|
| 229 |
};
|
| 230 |
}
|
|
@@ -508,28 +545,7 @@ function RobotScene({
|
|
| 508 |
return group;
|
| 509 |
};
|
| 510 |
loader.loadMeshCb = (url, mgr, onLoad) => {
|
| 511 |
-
|
| 512 |
-
/// URDFLoader re-parents and re-orients what it is handed, so it gets a clone (sharing
|
| 513 |
-
/// geometry and materials) and the cached scene stays intact.
|
| 514 |
-
mgr.itemStart(url);
|
| 515 |
-
loadDaeScene(url)
|
| 516 |
-
.then((daeScene) => onLoad(daeScene.clone()))
|
| 517 |
-
.catch((err) => onLoad(new THREE.Object3D(), err as Error))
|
| 518 |
-
.finally(() => mgr.itemEnd(url));
|
| 519 |
-
return;
|
| 520 |
-
}
|
| 521 |
-
// STL files — apply final PBR materials directly here. We used to do a
|
| 522 |
-
// post-load archetype rebuild in manager.onLoad, but STLLoader calls
|
| 523 |
-
// `manager.itemEnd` *before* our Promise resolves — so when the last
|
| 524 |
-
// STL completes, manager.onLoad fires synchronously, our traverse runs,
|
| 525 |
-
// and THEN URDFLoader's inner `group.add(obj)` + `obj.material = urdf`
|
| 526 |
-
// runs in a microtask. Last-batch meshes ended up gold/green because
|
| 527 |
-
// they were added to the robot after our rebuild passed.
|
| 528 |
-
//
|
| 529 |
-
// Fix: pick the archetype color here, wrap the mesh in a Group so
|
| 530 |
-
// URDFLoader won't override our material (its override only triggers
|
| 531 |
-
// for direct `THREE.Mesh` instances), and skip the onLoad rebuild.
|
| 532 |
-
const makeMesh = (geometry: THREE.BufferGeometry) => {
|
| 533 |
// Defaults: neutral off-white plastic, matches OpenArm "light" archetype
|
| 534 |
let color = "#9ba1ab";
|
| 535 |
let metalness = 0.1;
|
|
@@ -558,16 +574,54 @@ function RobotScene({
|
|
| 558 |
roughness = 0.6;
|
| 559 |
side = THREE.DoubleSide;
|
| 560 |
}
|
| 561 |
-
return new THREE.
|
| 562 |
-
|
| 563 |
-
|
| 564 |
-
|
| 565 |
-
|
| 566 |
-
|
| 567 |
-
|
| 568 |
-
}),
|
| 569 |
-
);
|
| 570 |
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 571 |
|
| 572 |
/// Tracked on this manager even when the geometry is cached or a prefetch already started
|
| 573 |
/// the download: manager.onLoad (shadows, camera fit) only fires for tracked loads, and
|
|
|
|
| 20 |
import type { URDFRobot } from "urdf-loader";
|
| 21 |
import { STLLoader } from "three/examples/jsm/loaders/STLLoader.js";
|
| 22 |
import { ColladaLoader } from "three/examples/jsm/loaders/ColladaLoader.js";
|
| 23 |
+
import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js";
|
| 24 |
+
import { MeshoptDecoder } from "three/examples/jsm/libs/meshopt_decoder.module.js";
|
| 25 |
import { Line2 } from "three/examples/jsm/lines/Line2.js";
|
| 26 |
import { LineMaterial } from "three/examples/jsm/lines/LineMaterial.js";
|
| 27 |
import { LineGeometry } from "three/examples/jsm/lines/LineGeometry.js";
|
|
|
|
| 59 |
return loading;
|
| 60 |
}
|
| 61 |
|
| 62 |
+
/**
|
| 63 |
+
* Rebuilds every mesh with a MeshStandardMaterial bucketed into one of three archetypes (carbon-black,
|
| 64 |
+
* brushed metal, off-white paint) by the original base-colour lightness. `flatShading` is for the GLB
|
| 65 |
+
* meshes, which carry no normals.
|
| 66 |
+
*/
|
| 67 |
+
function applyOpenArmMaterials(root: THREE.Object3D, flatShading: boolean) {
|
| 68 |
+
root.traverse((child) => {
|
| 69 |
+
if (!(child instanceof THREE.Mesh) || !child.material) return;
|
| 70 |
+
|
| 71 |
+
const originals = Array.isArray(child.material)
|
| 72 |
+
? child.material
|
| 73 |
+
: [child.material];
|
| 74 |
+
|
| 75 |
+
const rebuilt = originals.map((orig) => {
|
| 76 |
+
const srcColor =
|
| 77 |
+
(orig as THREE.MeshStandardMaterial).color ??
|
| 78 |
+
new THREE.Color("#c0c4cc");
|
| 79 |
+
const hsl = { h: 0, s: 0, l: 0 };
|
| 80 |
+
srcColor.getHSL(hsl);
|
| 81 |
+
|
| 82 |
+
// Archetype classification by original lightness.
|
| 83 |
+
let color: THREE.Color;
|
| 84 |
+
let metalness: number;
|
| 85 |
+
let roughness: number;
|
| 86 |
+
let envMapIntensity: number;
|
| 87 |
+
if (hsl.l < 0.3) {
|
| 88 |
+
// Carbon / anodised structural parts
|
| 89 |
+
color = new THREE.Color().setHSL(hsl.h, 0.02, 0.09);
|
| 90 |
+
metalness = 0.15;
|
| 91 |
+
roughness = 0.75;
|
| 92 |
+
envMapIntensity = 0.6;
|
| 93 |
+
} else if (hsl.l < 0.7) {
|
| 94 |
+
// Brushed metal joint collars / accents
|
| 95 |
+
color = new THREE.Color().setHSL(hsl.h, 0.04, 0.42);
|
| 96 |
+
metalness = 0.75;
|
| 97 |
+
roughness = 0.35;
|
| 98 |
+
envMapIntensity = 1.1;
|
| 99 |
+
} else {
|
| 100 |
+
// Off-white painted plates
|
| 101 |
+
color = new THREE.Color().setHSL(hsl.h, 0.03, 0.6);
|
| 102 |
+
metalness = 0.1;
|
| 103 |
+
roughness = 0.5;
|
| 104 |
+
envMapIntensity = 0.9;
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
const mat = new THREE.MeshStandardMaterial({
|
| 108 |
+
color,
|
| 109 |
+
metalness,
|
| 110 |
+
roughness,
|
| 111 |
+
envMapIntensity,
|
| 112 |
+
side: THREE.DoubleSide,
|
| 113 |
+
flatShading,
|
| 114 |
+
});
|
| 115 |
+
orig.dispose?.();
|
| 116 |
+
return mat;
|
| 117 |
+
});
|
| 118 |
+
|
| 119 |
+
child.material = Array.isArray(child.material) ? rebuilt : rebuilt[0];
|
| 120 |
+
child.castShadow = true;
|
| 121 |
+
child.receiveShadow = true;
|
| 122 |
+
});
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
/// One download and parse per DAE file per session: the OpenArm URDF references finger.dae 4 times
|
| 126 |
/// and each arm link twice, and the viewer remounts on every visit to the 3D Replay tab.
|
| 127 |
const daeSceneCache = new Map<string, Promise<THREE.Object3D>>();
|
|
|
|
| 152 |
});
|
| 153 |
for (const l of strayLights) l.parent?.remove(l);
|
| 154 |
|
| 155 |
+
applyOpenArmMaterials(collada.scene, false);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
}
|
| 157 |
return collada.scene;
|
| 158 |
})
|
|
|
|
| 165 |
return loading;
|
| 166 |
}
|
| 167 |
|
| 168 |
+
/// The bucket holds meshopt-compressed GLB copies of the SO-101 and OpenArm meshes, about 16x and 20x
|
| 169 |
+
/// smaller than the STL/DAE originals. They carry no normals: the originals' are flat per face, and
|
| 170 |
+
/// flatShading derives the same ones.
|
| 171 |
+
const glbSceneCache = new Map<string, Promise<THREE.Object3D>>();
|
| 172 |
+
|
| 173 |
+
function loadGlbScene(url: string): Promise<THREE.Object3D> {
|
| 174 |
+
let loading = glbSceneCache.get(url);
|
| 175 |
+
if (!loading) {
|
| 176 |
+
loading = new GLTFLoader()
|
| 177 |
+
.setMeshoptDecoder(MeshoptDecoder)
|
| 178 |
+
.loadAsync(url)
|
| 179 |
+
.then((gltf) => {
|
| 180 |
+
if (url.includes("openarm")) applyOpenArmMaterials(gltf.scene, true);
|
| 181 |
+
return gltf.scene;
|
| 182 |
+
})
|
| 183 |
+
.catch((err) => {
|
| 184 |
+
glbSceneCache.delete(url);
|
| 185 |
+
throw err;
|
| 186 |
+
});
|
| 187 |
+
glbSceneCache.set(url, loading);
|
| 188 |
+
}
|
| 189 |
+
return loading;
|
| 190 |
+
}
|
| 191 |
+
|
| 192 |
// URDFs + meshes are hosted in the Hub bucket at
|
| 193 |
// https://huggingface.co/buckets/lerobot/robot-urdfs. URDFLoader resolves
|
| 194 |
// relative mesh paths against the URDF's own URL, so the bucket layout
|
|
|
|
| 223 |
}
|
| 224 |
|
| 225 |
/**
|
| 226 |
+
* Starts downloading a robot's visual meshes into the caches. The viewer only mounts once the episode
|
| 227 |
+
* data has loaded, and the meshes used to start downloading only then; calling this earlier overlaps
|
| 228 |
+
* the two.
|
| 229 |
*/
|
| 230 |
export function prefetchRobotModel(robotType: string | null) {
|
| 231 |
const { urdfUrl } = getRobotConfig(robotType);
|
|
|
|
| 239 |
const filename = mesh.getAttribute("filename");
|
| 240 |
/// Same resolution URDFLoader applies to relative paths; package:// and DAE meshes are left to it.
|
| 241 |
if (!filename || filename.startsWith("package://")) continue;
|
| 242 |
+
const lower = filename.toLowerCase();
|
| 243 |
+
if (lower.endsWith(".glb"))
|
| 244 |
+
loadGlbScene(base + filename).catch(() => {});
|
| 245 |
+
else if (lower.endsWith(".stl"))
|
| 246 |
+
loadStlGeometry(base + filename).catch(() => {});
|
| 247 |
}
|
| 248 |
})
|
| 249 |
.catch(() => prefetchedUrdfs.delete(urdfUrl));
|
|
|
|
| 256 |
}
|
| 257 |
if (lower.includes("openarm")) {
|
| 258 |
return {
|
| 259 |
+
urdfUrl: `${URDF_BASE_URL}/openarm/openarm_bimanual_web.urdf`,
|
| 260 |
scale: 3,
|
| 261 |
};
|
| 262 |
}
|
| 263 |
return {
|
| 264 |
+
urdfUrl: `${URDF_BASE_URL}/so101/so101_new_calib_web.urdf`,
|
| 265 |
scale: 10,
|
| 266 |
};
|
| 267 |
}
|
|
|
|
| 545 |
return group;
|
| 546 |
};
|
| 547 |
loader.loadMeshCb = (url, mgr, onLoad) => {
|
| 548 |
+
const makeMaterial = (flatShading: boolean) => {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 549 |
// Defaults: neutral off-white plastic, matches OpenArm "light" archetype
|
| 550 |
let color = "#9ba1ab";
|
| 551 |
let metalness = 0.1;
|
|
|
|
| 574 |
roughness = 0.6;
|
| 575 |
side = THREE.DoubleSide;
|
| 576 |
}
|
| 577 |
+
return new THREE.MeshStandardMaterial({
|
| 578 |
+
color,
|
| 579 |
+
metalness,
|
| 580 |
+
roughness,
|
| 581 |
+
side,
|
| 582 |
+
flatShading,
|
| 583 |
+
});
|
|
|
|
|
|
|
| 584 |
};
|
| 585 |
+
const makeMesh = (geometry: THREE.BufferGeometry) =>
|
| 586 |
+
new THREE.Mesh(geometry, makeMaterial(false));
|
| 587 |
+
if (url.endsWith(".glb")) {
|
| 588 |
+
mgr.itemStart(url);
|
| 589 |
+
loadGlbScene(url)
|
| 590 |
+
.then((glbScene) => {
|
| 591 |
+
const obj = glbScene.clone();
|
| 592 |
+
if (!isOpenArm) {
|
| 593 |
+
obj.traverse((child) => {
|
| 594 |
+
if (child instanceof THREE.Mesh)
|
| 595 |
+
child.material = makeMaterial(true);
|
| 596 |
+
});
|
| 597 |
+
}
|
| 598 |
+
onLoad(obj);
|
| 599 |
+
})
|
| 600 |
+
.catch((err) => onLoad(new THREE.Object3D(), err as Error))
|
| 601 |
+
.finally(() => mgr.itemEnd(url));
|
| 602 |
+
return;
|
| 603 |
+
}
|
| 604 |
+
if (url.endsWith(".dae")) {
|
| 605 |
+
/// URDFLoader re-parents and re-orients what it is handed, so it gets a clone (sharing
|
| 606 |
+
/// geometry and materials) and the cached scene stays intact.
|
| 607 |
+
mgr.itemStart(url);
|
| 608 |
+
loadDaeScene(url)
|
| 609 |
+
.then((daeScene) => onLoad(daeScene.clone()))
|
| 610 |
+
.catch((err) => onLoad(new THREE.Object3D(), err as Error))
|
| 611 |
+
.finally(() => mgr.itemEnd(url));
|
| 612 |
+
return;
|
| 613 |
+
}
|
| 614 |
+
// STL files — apply final PBR materials directly here. We used to do a
|
| 615 |
+
// post-load archetype rebuild in manager.onLoad, but STLLoader calls
|
| 616 |
+
// `manager.itemEnd` *before* our Promise resolves — so when the last
|
| 617 |
+
// STL completes, manager.onLoad fires synchronously, our traverse runs,
|
| 618 |
+
// and THEN URDFLoader's inner `group.add(obj)` + `obj.material = urdf`
|
| 619 |
+
// runs in a microtask. Last-batch meshes ended up gold/green because
|
| 620 |
+
// they were added to the robot after our rebuild passed.
|
| 621 |
+
//
|
| 622 |
+
// Fix: pick the archetype color here, wrap the mesh in a Group so
|
| 623 |
+
// URDFLoader won't override our material (its override only triggers
|
| 624 |
+
// for direct `THREE.Mesh` instances), and skip the onLoad rebuild.
|
| 625 |
|
| 626 |
/// Tracked on this manager even when the geometry is cached or a prefetch already started
|
| 627 |
/// the download: manager.onLoad (shadows, camera fit) only fires for tracked loads, and
|