Spaces:

lerobot
/

glove_visualizer

Running

Homunculus glove viewer

af93715 verified about 2 months ago

9.24 kB

// Static URDF viewer for the Project Homunculus hardware.
//
// Everything is client side: each model is a kinematics.json (the tree) plus a
// packed GLB (the meshes), both produced by build_space.sh. There is no
// backend, so the sliders drive forward kinematics in the browser.
//
// Lighting and palette follow the g1 viewer: INK background, room environment
// for PBR reflections, one key light plus a warm rim.

import * as THREE from "three";
import { OrbitControls } from "three/addons/controls/OrbitControls.js";
import { RoomEnvironment } from "three/addons/environments/RoomEnvironment.js";
import { buildRig, loadKinematics, setJoint, applyPose } from "./robot.js";
import { buildPanel, bindKeys } from "./panel.js";
import { GloveController } from "./glove-panel.js";
import { gloveControlLayout } from "./glove.js";

const INK = 0x08080c;
const MODELS_URL = "./robot/models.json";

const el = (id) => document.getElementById(id);

function buildScene(canvasParent) {
  const renderer = new THREE.WebGLRenderer({ antialias: true });
  renderer.setPixelRatio(Math.min(devicePixelRatio, 2));
  renderer.setSize(innerWidth, innerHeight);
  renderer.setClearColor(INK, 1);
  renderer.outputColorSpace = THREE.SRGBColorSpace;
  renderer.toneMapping = THREE.ACESFilmicToneMapping;
  renderer.toneMappingExposure = 1.05;
  renderer.shadowMap.enabled = true;
  renderer.shadowMap.type = THREE.PCFSoftShadowMap;
  canvasParent.appendChild(renderer.domElement);

  const scene = new THREE.Scene();
  scene.background = new THREE.Color(INK);
  const pmrem = new THREE.PMREMGenerator(renderer);
  scene.environment = pmrem.fromScene(new RoomEnvironment()).texture;
  scene.environmentIntensity = 0.45;

  const camera = new THREE.PerspectiveCamera(40, innerWidth / innerHeight, 0.001, 200);

  scene.add(new THREE.AmbientLight(0xffffff, 0.6));
  const key = new THREE.DirectionalLight(0xffffff, 1.6);
  key.castShadow = true;
  key.shadow.mapSize.set(2048, 2048);
  key.shadow.bias = -0.0005;
  scene.add(key);
  scene.add(fill(0xffffff, 0.4, [-3, 2.5, 2]));
  scene.add(fill(0xffb366, 0.7, [0, 3, -3]));

  const controls = new OrbitControls(camera, renderer.domElement);
  controls.enableDamping = true;
  controls.dampingFactor = 0.08;
  controls.maxPolarAngle = Math.PI; // inspect the hand from above and underneath

  return { renderer, scene, camera, controls, key };
}

function fill(color, intensity, [x, y, z]) {
  const l = new THREE.DirectionalLight(color, intensity);
  l.position.set(x, y, z);
  return l;
}

/** Shadow catcher plus a two-tier grid, both sized to whatever is on stage. */
function buildFloor(scene, radius) {
  const group = new THREE.Group();
  const extent = radius * 16;

  const shadow = new THREE.Mesh(
    new THREE.PlaneGeometry(extent, extent),
    // depthWrite off: the catcher sits in the same plane as the grid, and
    // writing depth lets it z-fight the lines away.
    new THREE.ShadowMaterial({ opacity: 0.5, depthWrite: false }),
  );
  shadow.rotation.x = -Math.PI / 2;
  shadow.receiveShadow = true;
  group.add(shadow);

  // Cell size is derived from the subject: a 40 m grid of 0.1 m cells reads as
  // a floor under a humanoid and as solid fill under a 20 cm glove.
  const cell = radius / 2;
  const fine = new THREE.GridHelper(extent, Math.round(extent / cell), 0x2b323d, 0x2b323d);
  const section = new THREE.GridHelper(extent, Math.round(extent / (cell * 5)), 0x55637a, 0x55637a);
  fine.position.y = radius * 1e-3;
  section.position.y = radius * 2e-3;
  group.add(fine, section);

  scene.add(group);
  return group;
}

/** Sit the rig on the floor, centred, and return how big it turned out. */
function ground(rig) {
  const box = new THREE.Box3().setFromObject(rig.placer);
  const centre = box.getCenter(new THREE.Vector3());
  rig.placer.position.set(-centre.x, -box.min.y, -centre.z);

  const size = box.getSize(new THREE.Vector3());
  const radius = Math.max(size.x, size.y, size.z) / 2 || 0.1;
  return { radius, height: size.y };
}

function frame({ camera, controls, key }, radius, height) {
  const target = new THREE.Vector3(0, height / 2, 0);
  // Off-axis and slightly above: a straight-on view of a hand reads flat.
  camera.position.set(radius * 1.6, height / 2 + radius * 1.3, radius * 2.2);
  camera.near = radius / 100;
  camera.far = radius * 200;
  camera.updateProjectionMatrix();
  controls.target.copy(target);
  controls.minDistance = radius * 0.4;
  controls.maxDistance = radius * 20;
  controls.update();

  key.position.set(radius * 3, radius * 5, radius * 3);
  const s = radius * 1.6;
  Object.assign(key.shadow.camera, { left: -s, right: s, top: s, bottom: -s, far: radius * 20 });
  key.shadow.camera.updateProjectionMatrix();
}

/** Release the GPU buffers of a rig we are about to drop. */
function dispose(scene, rig) {
  scene.remove(rig.placer);
  rig.placer.traverse((o) => {
    if (!o.isMesh) return;
    o.geometry?.dispose();
    // Materials are shared across meshes by colour, so disposing per mesh
    // would free the same one repeatedly; harmless, and simpler than a set.
    o.material?.dispose();
  });
}

export async function main(progress) {
  progress("loading models…");
  const r = await fetch(MODELS_URL);
  if (!r.ok) throw new Error(`models.json fetch ${r.status}`);
  const models = await r.json();
  if (!models.length) throw new Error("models.json is empty");

  const view = buildScene(document.body);
  const { renderer, scene, camera, controls } = view;

  let current = null; // { rig, floor, panel }
  let busy = false;
  const buttons = new Map();

  const show = async (model) => {
    if (busy) return;
    busy = true;
    for (const [id, b] of buttons) {
      b.classList.toggle("on", id === model.id);
      b.disabled = true;
    }
    buttons.get(model.id).classList.add("loading");

    try {
      const k = await loadKinematics(model.kinematics);
      const rig = await buildRig(k, { glbUrl: model.glb });
      rig.root.scale.x = -1; // Mirror both the glove and hand before centring.
      scene.add(rig.placer);

      const { radius, height } = ground(rig);
      frame(view, radius, height);
      // Fog hides the grid's outer edge; tie it to the subject so it never
      // swallows the model or leaves the horizon hard.
      scene.fog = new THREE.Fog(INK, radius * 3, radius * 14);

      if (current) {
        await current.glove?.dispose();
        dispose(scene, current.rig);
        scene.remove(current.floor);
      }
      const floor = buildFloor(scene, radius);

      let glove = null;
      const panel = buildPanel(el("controls"), model.id === "glove" ? gloveControlLayout(k) : k, (joint, value, label) => {
        setJoint(rig, joint, value);
        el("joint").textContent = `${label} ${value >= 0 ? "+" : ""}${value.toFixed(2)}`;
        glove?.manualChanged();
      });
      if (model.id === "glove") glove = new GloveController({ rig, kinematics: k, panel, container: el("glove-input"), applyJointPose: pose => applyPose(rig, pose) });
      else el("glove-input").hidden = true;
      current = { rig, floor, panel, glove };
      el("hint").textContent = glove ? "Drag to orbit above or below · scroll to zoom · calibration at left" : "drag a slider to pose a joint";

      const meshes = k.bodies.reduce((n, b) => n + b.geoms.length, 0);
      el("model").textContent = k.name ?? model.label;
      el("subline").textContent = model.note ?? "";
      el("stats").textContent = `${panel.count} dof · ${rig.bodies.size} links · ${meshes} meshes`;
      el("joint").innerHTML = "&nbsp;";
    } finally {
      busy = false;
      for (const b of buttons.values()) b.disabled = false;
      buttons.get(model.id).classList.remove("loading");
    }
  };

  const tabs = el("tabs");
  for (const model of models) {
    const btn = document.createElement("button");
    btn.textContent = model.tab;
    btn.title = model.label;
    btn.className = "pbtn";
    btn.onclick = () => show(model).catch(error => { el("hint").textContent = `Model load failed: ${error.message}`; console.error(error); });
    buttons.set(model.id, btn);
    tabs.appendChild(btn);
  }

  const zero = document.createElement("button");
  zero.textContent = "ZERO";
  zero.className = "pbtn ghost";
  zero.title = "Return every joint to zero";
  zero.onclick = () => (current?.glove?.activePanel ?? current?.panel).zero();
  tabs.appendChild(zero);

  const fist = document.createElement("button");
  fist.textContent = "FIST";
  fist.className = "pbtn ghost";
  fist.title = "Curl the flexion joints";
  fist.onclick = () => (current?.glove?.activePanel ?? current?.panel).fist();
  tabs.appendChild(fist);

  bindKeys(() => current?.glove?.activePanel ?? current?.panel);

  progress("loading meshes…");
  await show(models[0]);

  addEventListener("resize", () => {
    camera.aspect = innerWidth / innerHeight;
    camera.updateProjectionMatrix();
    renderer.setSize(innerWidth, innerHeight);
  });

  renderer.setAnimationLoop(() => {
    controls.update();
    if (current) current.floor.visible = camera.position.y >= 0;
    renderer.render(scene, camera);
  });

  window.__viewer = { scene, camera, renderer, get rig() { return current?.rig; } };
  return view;
}

	// Static URDF viewer for the Project Homunculus hardware.
	//
	// Everything is client side: each model is a kinematics.json (the tree) plus a
	// packed GLB (the meshes), both produced by build_space.sh. There is no
	// backend, so the sliders drive forward kinematics in the browser.
	//
	// Lighting and palette follow the g1 viewer: INK background, room environment
	// for PBR reflections, one key light plus a warm rim.

	import * as THREE from "three";
	import { OrbitControls } from "three/addons/controls/OrbitControls.js";
	import { RoomEnvironment } from "three/addons/environments/RoomEnvironment.js";
	import { buildRig, loadKinematics, setJoint, applyPose } from "./robot.js";
	import { buildPanel, bindKeys } from "./panel.js";
	import { GloveController } from "./glove-panel.js";
	import { gloveControlLayout } from "./glove.js";

	const INK = 0x08080c;
	const MODELS_URL = "./robot/models.json";

	const el = (id) => document.getElementById(id);

	function buildScene(canvasParent) {
	const renderer = new THREE.WebGLRenderer({ antialias: true });
	renderer.setPixelRatio(Math.min(devicePixelRatio, 2));
	renderer.setSize(innerWidth, innerHeight);
	renderer.setClearColor(INK, 1);
	renderer.outputColorSpace = THREE.SRGBColorSpace;
	renderer.toneMapping = THREE.ACESFilmicToneMapping;
	renderer.toneMappingExposure = 1.05;
	renderer.shadowMap.enabled = true;
	renderer.shadowMap.type = THREE.PCFSoftShadowMap;
	canvasParent.appendChild(renderer.domElement);

	const scene = new THREE.Scene();
	scene.background = new THREE.Color(INK);
	const pmrem = new THREE.PMREMGenerator(renderer);
	scene.environment = pmrem.fromScene(new RoomEnvironment()).texture;
	scene.environmentIntensity = 0.45;

	const camera = new THREE.PerspectiveCamera(40, innerWidth / innerHeight, 0.001, 200);

	scene.add(new THREE.AmbientLight(0xffffff, 0.6));
	const key = new THREE.DirectionalLight(0xffffff, 1.6);
	key.castShadow = true;
	key.shadow.mapSize.set(2048, 2048);
	key.shadow.bias = -0.0005;
	scene.add(key);
	scene.add(fill(0xffffff, 0.4, [-3, 2.5, 2]));
	scene.add(fill(0xffb366, 0.7, [0, 3, -3]));

	const controls = new OrbitControls(camera, renderer.domElement);
	controls.enableDamping = true;
	controls.dampingFactor = 0.08;
	controls.maxPolarAngle = Math.PI; // inspect the hand from above and underneath

	return { renderer, scene, camera, controls, key };
	}

	function fill(color, intensity, [x, y, z]) {
	const l = new THREE.DirectionalLight(color, intensity);
	l.position.set(x, y, z);
	return l;
	}

	/** Shadow catcher plus a two-tier grid, both sized to whatever is on stage. */
	function buildFloor(scene, radius) {
	const group = new THREE.Group();
	const extent = radius * 16;

	const shadow = new THREE.Mesh(
	new THREE.PlaneGeometry(extent, extent),
	// depthWrite off: the catcher sits in the same plane as the grid, and
	// writing depth lets it z-fight the lines away.
	new THREE.ShadowMaterial({ opacity: 0.5, depthWrite: false }),
	);
	shadow.rotation.x = -Math.PI / 2;
	shadow.receiveShadow = true;
	group.add(shadow);

	// Cell size is derived from the subject: a 40 m grid of 0.1 m cells reads as
	// a floor under a humanoid and as solid fill under a 20 cm glove.
	const cell = radius / 2;
	const fine = new THREE.GridHelper(extent, Math.round(extent / cell), 0x2b323d, 0x2b323d);
	const section = new THREE.GridHelper(extent, Math.round(extent / (cell * 5)), 0x55637a, 0x55637a);
	fine.position.y = radius * 1e-3;
	section.position.y = radius * 2e-3;
	group.add(fine, section);

	scene.add(group);
	return group;
	}

	/** Sit the rig on the floor, centred, and return how big it turned out. */
	function ground(rig) {
	const box = new THREE.Box3().setFromObject(rig.placer);
	const centre = box.getCenter(new THREE.Vector3());
	rig.placer.position.set(-centre.x, -box.min.y, -centre.z);

	const size = box.getSize(new THREE.Vector3());
	const radius = Math.max(size.x, size.y, size.z) / 2 \|\| 0.1;
	return { radius, height: size.y };
	}

	function frame({ camera, controls, key }, radius, height) {
	const target = new THREE.Vector3(0, height / 2, 0);
	// Off-axis and slightly above: a straight-on view of a hand reads flat.
	camera.position.set(radius * 1.6, height / 2 + radius * 1.3, radius * 2.2);
	camera.near = radius / 100;
	camera.far = radius * 200;
	camera.updateProjectionMatrix();
	controls.target.copy(target);
	controls.minDistance = radius * 0.4;
	controls.maxDistance = radius * 20;
	controls.update();

	key.position.set(radius * 3, radius * 5, radius * 3);
	const s = radius * 1.6;
	Object.assign(key.shadow.camera, { left: -s, right: s, top: s, bottom: -s, far: radius * 20 });
	key.shadow.camera.updateProjectionMatrix();
	}

	/** Release the GPU buffers of a rig we are about to drop. */
	function dispose(scene, rig) {
	scene.remove(rig.placer);
	rig.placer.traverse((o) => {
	if (!o.isMesh) return;
	o.geometry?.dispose();
	// Materials are shared across meshes by colour, so disposing per mesh
	// would free the same one repeatedly; harmless, and simpler than a set.
	o.material?.dispose();
	});
	}

	export async function main(progress) {
	progress("loading models…");
	const r = await fetch(MODELS_URL);
	if (!r.ok) throw new Error(`models.json fetch ${r.status}`);
	const models = await r.json();
	if (!models.length) throw new Error("models.json is empty");

	const view = buildScene(document.body);
	const { renderer, scene, camera, controls } = view;

	let current = null; // { rig, floor, panel }
	let busy = false;
	const buttons = new Map();

	const show = async (model) => {
	if (busy) return;
	busy = true;
	for (const [id, b] of buttons) {
	b.classList.toggle("on", id === model.id);
	b.disabled = true;
	}
	buttons.get(model.id).classList.add("loading");

	try {
	const k = await loadKinematics(model.kinematics);
	const rig = await buildRig(k, { glbUrl: model.glb });
	rig.root.scale.x = -1; // Mirror both the glove and hand before centring.
	scene.add(rig.placer);

	const { radius, height } = ground(rig);
	frame(view, radius, height);
	// Fog hides the grid's outer edge; tie it to the subject so it never
	// swallows the model or leaves the horizon hard.
	scene.fog = new THREE.Fog(INK, radius * 3, radius * 14);

	if (current) {
	await current.glove?.dispose();
	dispose(scene, current.rig);
	scene.remove(current.floor);
	}
	const floor = buildFloor(scene, radius);

	let glove = null;
	const panel = buildPanel(el("controls"), model.id === "glove" ? gloveControlLayout(k) : k, (joint, value, label) => {
	setJoint(rig, joint, value);
	el("joint").textContent = `${label} ${value >= 0 ? "+" : ""}${value.toFixed(2)}`;
	glove?.manualChanged();
	});
	if (model.id === "glove") glove = new GloveController({ rig, kinematics: k, panel, container: el("glove-input"), applyJointPose: pose => applyPose(rig, pose) });
	else el("glove-input").hidden = true;
	current = { rig, floor, panel, glove };
	el("hint").textContent = glove ? "Drag to orbit above or below · scroll to zoom · calibration at left" : "drag a slider to pose a joint";

	const meshes = k.bodies.reduce((n, b) => n + b.geoms.length, 0);
	el("model").textContent = k.name ?? model.label;
	el("subline").textContent = model.note ?? "";
	el("stats").textContent = `${panel.count} dof · ${rig.bodies.size} links · ${meshes} meshes`;
	el("joint").innerHTML = " ";
	} finally {
	busy = false;
	for (const b of buttons.values()) b.disabled = false;
	buttons.get(model.id).classList.remove("loading");
	}
	};

	const tabs = el("tabs");
	for (const model of models) {
	const btn = document.createElement("button");
	btn.textContent = model.tab;
	btn.title = model.label;
	btn.className = "pbtn";
	btn.onclick = () => show(model).catch(error => { el("hint").textContent = `Model load failed: ${error.message}`; console.error(error); });
	buttons.set(model.id, btn);
	tabs.appendChild(btn);
	}

	const zero = document.createElement("button");
	zero.textContent = "ZERO";
	zero.className = "pbtn ghost";
	zero.title = "Return every joint to zero";
	zero.onclick = () => (current?.glove?.activePanel ?? current?.panel).zero();
	tabs.appendChild(zero);

	const fist = document.createElement("button");
	fist.textContent = "FIST";
	fist.className = "pbtn ghost";
	fist.title = "Curl the flexion joints";
	fist.onclick = () => (current?.glove?.activePanel ?? current?.panel).fist();
	tabs.appendChild(fist);

	bindKeys(() => current?.glove?.activePanel ?? current?.panel);

	progress("loading meshes…");
	await show(models[0]);

	addEventListener("resize", () => {
	camera.aspect = innerWidth / innerHeight;
	camera.updateProjectionMatrix();
	renderer.setSize(innerWidth, innerHeight);
	});

	renderer.setAnimationLoop(() => {
	controls.update();
	if (current) current.floor.visible = camera.position.y >= 0;
	renderer.render(scene, camera);
	});

	window.__viewer = { scene, camera, renderer, get rig() { return current?.rig; } };
	return view;
	}