mishig HF Staff commited on
Commit
707ca99
·
verified ·
1 Parent(s): ee4674d

Sync from huggingface/lerobot-dataset-visualizer via hub-sync

Browse files
Files changed (1) hide show
  1. src/components/urdf-viewer.tsx +148 -94
src/components/urdf-viewer.tsx CHANGED
@@ -20,6 +20,8 @@ import URDFLoader from "urdf-loader";
20
  import type { URDFRobot } from "urdf-loader";
21
  import { STLLoader } from "three/examples/jsm/loaders/STLLoader.js";
22
  import { ColladaLoader } from "three/examples/jsm/loaders/ColladaLoader.js";
 
 
23
  import { Line2 } from "three/examples/jsm/lines/Line2.js";
24
  import { LineMaterial } from "three/examples/jsm/lines/LineMaterial.js";
25
  import { LineGeometry } from "three/examples/jsm/lines/LineGeometry.js";
@@ -57,6 +59,69 @@ function loadStlGeometry(url: string): Promise<THREE.BufferGeometry> {
57
  return loading;
58
  }
59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
60
  /// One download and parse per DAE file per session: the OpenArm URDF references finger.dae 4 times
61
  /// and each arm link twice, and the viewer remounts on every visit to the 3D Replay tab.
62
  const daeSceneCache = new Map<string, Promise<THREE.Object3D>>();
@@ -87,62 +152,7 @@ function loadDaeScene(url: string): Promise<THREE.Object3D> {
87
  });
88
  for (const l of strayLights) l.parent?.remove(l);
89
 
90
- collada.scene.traverse((child) => {
91
- if (!(child instanceof THREE.Mesh) || !child.material) return;
92
-
93
- const originals = Array.isArray(child.material)
94
- ? child.material
95
- : [child.material];
96
-
97
- const rebuilt = originals.map((orig) => {
98
- const srcColor =
99
- (orig as THREE.MeshStandardMaterial).color ??
100
- new THREE.Color("#c0c4cc");
101
- const hsl = { h: 0, s: 0, l: 0 };
102
- srcColor.getHSL(hsl);
103
-
104
- // Archetype classification by original lightness.
105
- let color: THREE.Color;
106
- let metalness: number;
107
- let roughness: number;
108
- let envMapIntensity: number;
109
- if (hsl.l < 0.3) {
110
- // Carbon / anodised structural parts
111
- color = new THREE.Color().setHSL(hsl.h, 0.02, 0.09);
112
- metalness = 0.15;
113
- roughness = 0.75;
114
- envMapIntensity = 0.6;
115
- } else if (hsl.l < 0.7) {
116
- // Brushed metal joint collars / accents
117
- color = new THREE.Color().setHSL(hsl.h, 0.04, 0.42);
118
- metalness = 0.75;
119
- roughness = 0.35;
120
- envMapIntensity = 1.1;
121
- } else {
122
- // Off-white painted plates
123
- color = new THREE.Color().setHSL(hsl.h, 0.03, 0.6);
124
- metalness = 0.1;
125
- roughness = 0.5;
126
- envMapIntensity = 0.9;
127
- }
128
-
129
- const mat = new THREE.MeshStandardMaterial({
130
- color,
131
- metalness,
132
- roughness,
133
- envMapIntensity,
134
- side: THREE.DoubleSide,
135
- });
136
- orig.dispose?.();
137
- return mat;
138
- });
139
-
140
- child.material = Array.isArray(child.material)
141
- ? rebuilt
142
- : rebuilt[0];
143
- child.castShadow = true;
144
- child.receiveShadow = true;
145
- });
146
  }
147
  return collada.scene;
148
  })
@@ -155,6 +165,30 @@ function loadDaeScene(url: string): Promise<THREE.Object3D> {
155
  return loading;
156
  }
157
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
158
  // URDFs + meshes are hosted in the Hub bucket at
159
  // https://huggingface.co/buckets/lerobot/robot-urdfs. URDFLoader resolves
160
  // relative mesh paths against the URDF's own URL, so the bucket layout
@@ -189,9 +223,9 @@ function loadUrdfText(url: string): Promise<string> {
189
  }
190
 
191
  /**
192
- * Starts downloading a robot's visual STL meshes into the geometry cache. The viewer only mounts once
193
- * the episode data has loaded, and the meshes (15.7 MB for SO-101) used to start downloading only
194
- * then; calling this earlier overlaps the two.
195
  */
196
  export function prefetchRobotModel(robotType: string | null) {
197
  const { urdfUrl } = getRobotConfig(robotType);
@@ -205,8 +239,11 @@ export function prefetchRobotModel(robotType: string | null) {
205
  const filename = mesh.getAttribute("filename");
206
  /// Same resolution URDFLoader applies to relative paths; package:// and DAE meshes are left to it.
207
  if (!filename || filename.startsWith("package://")) continue;
208
- if (!filename.toLowerCase().endsWith(".stl")) continue;
209
- loadStlGeometry(base + filename).catch(() => {});
 
 
 
210
  }
211
  })
212
  .catch(() => prefetchedUrdfs.delete(urdfUrl));
@@ -219,12 +256,12 @@ function getRobotConfig(robotType: string | null) {
219
  }
220
  if (lower.includes("openarm")) {
221
  return {
222
- urdfUrl: `${URDF_BASE_URL}/openarm/openarm_bimanual.urdf`,
223
  scale: 3,
224
  };
225
  }
226
  return {
227
- urdfUrl: `${URDF_BASE_URL}/so101/so101_new_calib.urdf`,
228
  scale: 10,
229
  };
230
  }
@@ -508,28 +545,7 @@ function RobotScene({
508
  return group;
509
  };
510
  loader.loadMeshCb = (url, mgr, onLoad) => {
511
- if (url.endsWith(".dae")) {
512
- /// URDFLoader re-parents and re-orients what it is handed, so it gets a clone (sharing
513
- /// geometry and materials) and the cached scene stays intact.
514
- mgr.itemStart(url);
515
- loadDaeScene(url)
516
- .then((daeScene) => onLoad(daeScene.clone()))
517
- .catch((err) => onLoad(new THREE.Object3D(), err as Error))
518
- .finally(() => mgr.itemEnd(url));
519
- return;
520
- }
521
- // STL files — apply final PBR materials directly here. We used to do a
522
- // post-load archetype rebuild in manager.onLoad, but STLLoader calls
523
- // `manager.itemEnd` *before* our Promise resolves — so when the last
524
- // STL completes, manager.onLoad fires synchronously, our traverse runs,
525
- // and THEN URDFLoader's inner `group.add(obj)` + `obj.material = urdf`
526
- // runs in a microtask. Last-batch meshes ended up gold/green because
527
- // they were added to the robot after our rebuild passed.
528
- //
529
- // Fix: pick the archetype color here, wrap the mesh in a Group so
530
- // URDFLoader won't override our material (its override only triggers
531
- // for direct `THREE.Mesh` instances), and skip the onLoad rebuild.
532
- const makeMesh = (geometry: THREE.BufferGeometry) => {
533
  // Defaults: neutral off-white plastic, matches OpenArm "light" archetype
534
  let color = "#9ba1ab";
535
  let metalness = 0.1;
@@ -558,16 +574,54 @@ function RobotScene({
558
  roughness = 0.6;
559
  side = THREE.DoubleSide;
560
  }
561
- return new THREE.Mesh(
562
- geometry,
563
- new THREE.MeshStandardMaterial({
564
- color,
565
- metalness,
566
- roughness,
567
- side,
568
- }),
569
- );
570
  };
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
571
 
572
  /// Tracked on this manager even when the geometry is cached or a prefetch already started
573
  /// the download: manager.onLoad (shadows, camera fit) only fires for tracked loads, and
 
20
  import type { URDFRobot } from "urdf-loader";
21
  import { STLLoader } from "three/examples/jsm/loaders/STLLoader.js";
22
  import { ColladaLoader } from "three/examples/jsm/loaders/ColladaLoader.js";
23
+ import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js";
24
+ import { MeshoptDecoder } from "three/examples/jsm/libs/meshopt_decoder.module.js";
25
  import { Line2 } from "three/examples/jsm/lines/Line2.js";
26
  import { LineMaterial } from "three/examples/jsm/lines/LineMaterial.js";
27
  import { LineGeometry } from "three/examples/jsm/lines/LineGeometry.js";
 
59
  return loading;
60
  }
61
 
62
+ /**
63
+ * Rebuilds every mesh with a MeshStandardMaterial bucketed into one of three archetypes (carbon-black,
64
+ * brushed metal, off-white paint) by the original base-colour lightness. `flatShading` is for the GLB
65
+ * meshes, which carry no normals.
66
+ */
67
+ function applyOpenArmMaterials(root: THREE.Object3D, flatShading: boolean) {
68
+ root.traverse((child) => {
69
+ if (!(child instanceof THREE.Mesh) || !child.material) return;
70
+
71
+ const originals = Array.isArray(child.material)
72
+ ? child.material
73
+ : [child.material];
74
+
75
+ const rebuilt = originals.map((orig) => {
76
+ const srcColor =
77
+ (orig as THREE.MeshStandardMaterial).color ??
78
+ new THREE.Color("#c0c4cc");
79
+ const hsl = { h: 0, s: 0, l: 0 };
80
+ srcColor.getHSL(hsl);
81
+
82
+ // Archetype classification by original lightness.
83
+ let color: THREE.Color;
84
+ let metalness: number;
85
+ let roughness: number;
86
+ let envMapIntensity: number;
87
+ if (hsl.l < 0.3) {
88
+ // Carbon / anodised structural parts
89
+ color = new THREE.Color().setHSL(hsl.h, 0.02, 0.09);
90
+ metalness = 0.15;
91
+ roughness = 0.75;
92
+ envMapIntensity = 0.6;
93
+ } else if (hsl.l < 0.7) {
94
+ // Brushed metal joint collars / accents
95
+ color = new THREE.Color().setHSL(hsl.h, 0.04, 0.42);
96
+ metalness = 0.75;
97
+ roughness = 0.35;
98
+ envMapIntensity = 1.1;
99
+ } else {
100
+ // Off-white painted plates
101
+ color = new THREE.Color().setHSL(hsl.h, 0.03, 0.6);
102
+ metalness = 0.1;
103
+ roughness = 0.5;
104
+ envMapIntensity = 0.9;
105
+ }
106
+
107
+ const mat = new THREE.MeshStandardMaterial({
108
+ color,
109
+ metalness,
110
+ roughness,
111
+ envMapIntensity,
112
+ side: THREE.DoubleSide,
113
+ flatShading,
114
+ });
115
+ orig.dispose?.();
116
+ return mat;
117
+ });
118
+
119
+ child.material = Array.isArray(child.material) ? rebuilt : rebuilt[0];
120
+ child.castShadow = true;
121
+ child.receiveShadow = true;
122
+ });
123
+ }
124
+
125
  /// One download and parse per DAE file per session: the OpenArm URDF references finger.dae 4 times
126
  /// and each arm link twice, and the viewer remounts on every visit to the 3D Replay tab.
127
  const daeSceneCache = new Map<string, Promise<THREE.Object3D>>();
 
152
  });
153
  for (const l of strayLights) l.parent?.remove(l);
154
 
155
+ applyOpenArmMaterials(collada.scene, false);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
156
  }
157
  return collada.scene;
158
  })
 
165
  return loading;
166
  }
167
 
168
+ /// The bucket holds meshopt-compressed GLB copies of the SO-101 and OpenArm meshes, about 16x and 20x
169
+ /// smaller than the STL/DAE originals. They carry no normals: the originals' are flat per face, and
170
+ /// flatShading derives the same ones.
171
+ const glbSceneCache = new Map<string, Promise<THREE.Object3D>>();
172
+
173
+ function loadGlbScene(url: string): Promise<THREE.Object3D> {
174
+ let loading = glbSceneCache.get(url);
175
+ if (!loading) {
176
+ loading = new GLTFLoader()
177
+ .setMeshoptDecoder(MeshoptDecoder)
178
+ .loadAsync(url)
179
+ .then((gltf) => {
180
+ if (url.includes("openarm")) applyOpenArmMaterials(gltf.scene, true);
181
+ return gltf.scene;
182
+ })
183
+ .catch((err) => {
184
+ glbSceneCache.delete(url);
185
+ throw err;
186
+ });
187
+ glbSceneCache.set(url, loading);
188
+ }
189
+ return loading;
190
+ }
191
+
192
  // URDFs + meshes are hosted in the Hub bucket at
193
  // https://huggingface.co/buckets/lerobot/robot-urdfs. URDFLoader resolves
194
  // relative mesh paths against the URDF's own URL, so the bucket layout
 
223
  }
224
 
225
  /**
226
+ * Starts downloading a robot's visual meshes into the caches. The viewer only mounts once the episode
227
+ * data has loaded, and the meshes used to start downloading only then; calling this earlier overlaps
228
+ * the two.
229
  */
230
  export function prefetchRobotModel(robotType: string | null) {
231
  const { urdfUrl } = getRobotConfig(robotType);
 
239
  const filename = mesh.getAttribute("filename");
240
  /// Same resolution URDFLoader applies to relative paths; package:// and DAE meshes are left to it.
241
  if (!filename || filename.startsWith("package://")) continue;
242
+ const lower = filename.toLowerCase();
243
+ if (lower.endsWith(".glb"))
244
+ loadGlbScene(base + filename).catch(() => {});
245
+ else if (lower.endsWith(".stl"))
246
+ loadStlGeometry(base + filename).catch(() => {});
247
  }
248
  })
249
  .catch(() => prefetchedUrdfs.delete(urdfUrl));
 
256
  }
257
  if (lower.includes("openarm")) {
258
  return {
259
+ urdfUrl: `${URDF_BASE_URL}/openarm/openarm_bimanual_web.urdf`,
260
  scale: 3,
261
  };
262
  }
263
  return {
264
+ urdfUrl: `${URDF_BASE_URL}/so101/so101_new_calib_web.urdf`,
265
  scale: 10,
266
  };
267
  }
 
545
  return group;
546
  };
547
  loader.loadMeshCb = (url, mgr, onLoad) => {
548
+ const makeMaterial = (flatShading: boolean) => {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
549
  // Defaults: neutral off-white plastic, matches OpenArm "light" archetype
550
  let color = "#9ba1ab";
551
  let metalness = 0.1;
 
574
  roughness = 0.6;
575
  side = THREE.DoubleSide;
576
  }
577
+ return new THREE.MeshStandardMaterial({
578
+ color,
579
+ metalness,
580
+ roughness,
581
+ side,
582
+ flatShading,
583
+ });
 
 
584
  };
585
+ const makeMesh = (geometry: THREE.BufferGeometry) =>
586
+ new THREE.Mesh(geometry, makeMaterial(false));
587
+ if (url.endsWith(".glb")) {
588
+ mgr.itemStart(url);
589
+ loadGlbScene(url)
590
+ .then((glbScene) => {
591
+ const obj = glbScene.clone();
592
+ if (!isOpenArm) {
593
+ obj.traverse((child) => {
594
+ if (child instanceof THREE.Mesh)
595
+ child.material = makeMaterial(true);
596
+ });
597
+ }
598
+ onLoad(obj);
599
+ })
600
+ .catch((err) => onLoad(new THREE.Object3D(), err as Error))
601
+ .finally(() => mgr.itemEnd(url));
602
+ return;
603
+ }
604
+ if (url.endsWith(".dae")) {
605
+ /// URDFLoader re-parents and re-orients what it is handed, so it gets a clone (sharing
606
+ /// geometry and materials) and the cached scene stays intact.
607
+ mgr.itemStart(url);
608
+ loadDaeScene(url)
609
+ .then((daeScene) => onLoad(daeScene.clone()))
610
+ .catch((err) => onLoad(new THREE.Object3D(), err as Error))
611
+ .finally(() => mgr.itemEnd(url));
612
+ return;
613
+ }
614
+ // STL files — apply final PBR materials directly here. We used to do a
615
+ // post-load archetype rebuild in manager.onLoad, but STLLoader calls
616
+ // `manager.itemEnd` *before* our Promise resolves — so when the last
617
+ // STL completes, manager.onLoad fires synchronously, our traverse runs,
618
+ // and THEN URDFLoader's inner `group.add(obj)` + `obj.material = urdf`
619
+ // runs in a microtask. Last-batch meshes ended up gold/green because
620
+ // they were added to the robot after our rebuild passed.
621
+ //
622
+ // Fix: pick the archetype color here, wrap the mesh in a Group so
623
+ // URDFLoader won't override our material (its override only triggers
624
+ // for direct `THREE.Mesh` instances), and skip the onLoad rebuild.
625
 
626
  /// Tracked on this manager even when the geometry is cached or a prefetch already started
627
  /// the download: manager.onLoad (shadows, camera fit) only fires for tracked loads, and