// Reachy Remote — the remote-control UI and all robot logic. // // Works with any object that has the ReachyMini SDK's runtime API // (the real robot from connectToHost(), or MockRobot in demo mode). // // Units: the UI speaks degrees. Everything sent to the robot is converted // at the boundary (in `toWire`) to radians + a flat 4×4 head matrix. import { flattenMatrix } from "./math.js"; // ---- Tunables --------------------------------------------------------------- const LIMITS = { yaw: 40, // head left/right, degrees pitch: 25, // head up/down roll: 20, // head tilt body: 120, // body rotation antenna: 70, // each antenna }; // Sign conventions (SDK frame: x forward, y left, z up). // Positive yaw turns the head to its left; positive pitch tips the nose down. // If your robot moves the opposite way on the pad, flip these to -1. const YAW_SIGN = 1; const PITCH_SIGN = 1; // Rest pose: head level, antennas ~10° outward (Pollen's anti-resonance offset). const HOME = Object.freeze({ roll: 0, pitch: 0, yaw: 0, body: 0, antR: -10, antL: 10 }); const SEND_INTERVAL_MS = 40; // ~25 Hz max for live control const KEY_STEP = 4; // degrees per key press // Emotes: keyframes are offsets (degrees) added to the current pose, each // reached with a smooth daemon-side goto over `t` seconds. const EMOTES = [ { id: "nod", key: "1", label: "Nod", icon: "↕", frames: [ { pitch: 15, t: 0.25 }, { pitch: -8, t: 0.25 }, { pitch: 15, t: 0.25 }, { pitch: 0, t: 0.3 }, ] }, { id: "shake", key: "2", label: "Shake", icon: "↔", frames: [ { yaw: 20, t: 0.25 }, { yaw: -20, t: 0.35 }, { yaw: 20, t: 0.35 }, { yaw: 0, t: 0.3 }, ] }, { id: "curious", key: "3", label: "Curious", icon: "?", frames: [ { roll: 15, pitch: -5, antR: -40, antL: 10, t: 0.5 }, { roll: 15, pitch: -5, antR: -40, antL: 10, t: 0.8 }, { t: 0.5 }, ] }, { id: "happy", key: "4", label: "Happy", icon: "✦", frames: [ { pitch: -8, antR: -50, antL: 50, t: 0.3 }, { pitch: -8, antR: 30, antL: -30, t: 0.25 }, { pitch: -8, antR: -50, antL: 50, t: 0.25 }, { pitch: -8, antR: 30, antL: -30, t: 0.25 }, { t: 0.4 }, ] }, { id: "look", key: "5", label: "Explore", icon: "◎", frames: [ { yaw: 30, pitch: -5, t: 0.7 }, { yaw: 30, pitch: -5, t: 0.4 }, { yaw: -30, pitch: -5, t: 1.0 }, { yaw: -30, pitch: -5, t: 0.4 }, { t: 0.7 }, ] }, { id: "surprised", key: "6", label: "Surprised", icon: "!", frames: [ { pitch: -15, antR: -65, antL: 65, t: 0.2 }, { pitch: -15, antR: -65, antL: 65, t: 0.7 }, { t: 0.6 }, ] }, ]; // ---- Helpers ---------------------------------------------------------------- const clamp = (v, lo, hi) => Math.min(hi, Math.max(lo, v)); const d2r = (d) => (d * Math.PI) / 180; const r2d = (r) => (r * 180) / Math.PI; const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); function el(tag, attrs = {}, ...children) { const node = document.createElement(tag); for (const [k, v] of Object.entries(attrs)) { if (k === "class") node.className = v; else if (k.startsWith("on")) node.addEventListener(k.slice(2).toLowerCase(), v); else if (v === true) node.setAttribute(k, ""); else if (v !== false && v != null) node.setAttribute(k, v); } for (const c of children.flat()) node.append(c instanceof Node ? c : document.createTextNode(String(c))); return node; } // ---- Main ------------------------------------------------------------------- /** * Mount the remote into `mount` and start controlling `robot`. * Returns { shutdown } — call it when the session ends. */ export function startRemote({ robot, mount, math, robotName = "Reachy Mini", demo = false }) { const cmd = { ...HOME }; // last commanded pose (degrees), head yaw is relative to body let touched = false; // has the user moved anything yet? let emoteToken = 0; // bump to cancel a running emote let tracking = false; let puppet = false; let lastSend = 0; let sendTimer = null; const cleanups = []; // ---- Wire conversion ------------------------------------------------------ function toWire(p) { // The head matrix is in the world frame, so add the body yaw to make // the head turn with the body. const head = flattenMatrix(math.rpyToMatrix(p.roll, p.pitch, p.yaw + p.body)); return { head, antennas: [d2r(p.antR), d2r(p.antL)], body_yaw: d2r(p.body) }; } function sendNow() { sendTimer = null; lastSend = performance.now(); try { robot.setTarget(toWire(cmd)); } catch (err) { log(`Command failed: ${err.message || err}`, "warn"); } } function scheduleSend() { const wait = SEND_INTERVAL_MS - (performance.now() - lastSend); if (wait <= 0) sendNow(); else if (!sendTimer) sendTimer = setTimeout(sendNow, wait); } /** Apply a manual change from the UI: cancels emotes, updates controls, sends. */ function setCmd(patch) { touched = true; emoteToken++; setBusy(null); Object.assign(cmd, patch); cmd.roll = clamp(cmd.roll, -LIMITS.roll, LIMITS.roll); cmd.pitch = clamp(cmd.pitch, -LIMITS.pitch, LIMITS.pitch); cmd.yaw = clamp(cmd.yaw, -LIMITS.yaw, LIMITS.yaw); cmd.body = clamp(cmd.body, -LIMITS.body, LIMITS.body); cmd.antR = clamp(cmd.antR, -LIMITS.antenna, LIMITS.antenna); cmd.antL = clamp(cmd.antL, -LIMITS.antenna, LIMITS.antenna); renderControls(); scheduleSend(); } async function gotoPose(p, seconds) { robot.gotoTarget({ ...toWire(p), duration: seconds }); await sleep(seconds * 1000); } /** Read the robot's actual pose into `cmd` so the next input doesn't jump. */ function syncFromState() { const s = robot.robotState; if (!s || !s.head) return false; const world = math.matrixToRpy(s.head); const body = typeof s.body_yaw === "number" ? r2d(s.body_yaw) : 0; Object.assign(cmd, { roll: clamp(world.roll, -LIMITS.roll, LIMITS.roll), pitch: clamp(world.pitch, -LIMITS.pitch, LIMITS.pitch), yaw: clamp(world.yaw - body, -LIMITS.yaw, LIMITS.yaw), body: clamp(body, -LIMITS.body, LIMITS.body), }); if (Array.isArray(s.antennas) && s.antennas.length === 2) { cmd.antR = clamp(r2d(s.antennas[0]), -LIMITS.antenna, LIMITS.antenna); cmd.antL = clamp(r2d(s.antennas[1]), -LIMITS.antenna, LIMITS.antenna); } renderControls(); return true; } // ---- Actions ---------------------------------------------------------------- async function playEmote(emote) { if (puppet) return log("Turn off puppet mode to play emotes.", "warn"); const token = ++emoteToken; setBusy(emote.id); log(`Emote: ${emote.label}`); const base = { ...cmd }; try { for (const f of emote.frames) { if (token !== emoteToken) return; const target = { ...base }; for (const k of ["roll", "pitch", "yaw", "body", "antR", "antL"]) { if (typeof f[k] === "number") target[k] = base[k] + f[k]; } await gotoPose(target, f.t); } } catch (err) { log(`Emote failed: ${err.message || err}`, "warn"); } finally { if (token === emoteToken) setBusy(null); } } async function goHome() { emoteToken++; setBusy("home"); Object.assign(cmd, HOME); renderControls(); log("Returning home"); const token = emoteToken; await gotoPose(cmd, 1.0); if (token === emoteToken) setBusy(null); } async function wake() { setBusy("wake"); log("Waking up…"); try { await robot.wakeUp(); puppet = false; Object.assign(cmd, HOME); renderControls(); log("Awake"); } catch (err) { log(`Wake-up failed: ${err.message || err}`, "warn"); } setBusy(null); renderToggles(); } async function goSleep() { emoteToken++; if (tracking) toggleTracking(false); setBusy("sleep"); log("Going to sleep…"); try { await robot.gotoSleep(); log("Asleep. Press Wake up to continue."); } catch (err) { log(`Sleep failed: ${err.message || err}`, "warn"); } setBusy(null); } function toggleTracking(on = !tracking) { tracking = on; try { if (on) robot.startHeadTracking(1.0); else robot.stopHeadTracking(); log(on ? "Face tracking on" : "Face tracking off"); } catch (err) { tracking = false; log(`Face tracking unavailable: ${err.message || err}`, "warn"); } renderToggles(); } function togglePuppet(on = !puppet) { emoteToken++; setBusy(null); puppet = on; robot.setMotorMode(on ? "gravity_compensation" : "enabled"); log(on ? "Puppet mode: move Reachy by hand" : "Motors back on"); if (!on) { // Torque-on holds the current pose; adopt it so sliders match. robot.requestState(); setTimeout(syncFromState, 300); } renderToggles(); } // ---- UI ----------------------------------------------------------------------- mount.innerHTML = ""; const ui = {}; // Header ui.status = el("span", { class: "pill" }, "Connected"); const header = el("header", { class: "topline" }, el("div", { class: "brand" }, el("img", { src: "./public/icon.svg", alt: "", width: 28, height: 28 }), el("div", {}, el("h1", {}, "Reachy Remote"), el("p", { class: "muted" }, robotName), ), ), ui.status, ); // Camera / preview ui.video = el("video", { autoplay: true, playsinline: true, muted: true }); ui.videoNote = el("div", { class: "video-note" }, demo ? "" : "Waiting for camera…"); ui.avatar = buildAvatar(); const camera = el("section", { class: "card camera" }, el("div", { class: "camera-frame" }, demo ? ui.avatar.svg : ui.video, ui.videoNote), el("div", { class: "camera-meta" }, el("span", { class: "muted" }, demo ? "Simulated robot" : "Live camera"), ui.mode = el("span", { class: "muted" }, ""), ), ); // Head pad ui.dot = el("div", { class: "pad-dot" }); ui.pad = el("div", { class: "pad", tabindex: 0, role: "slider", "aria-label": "Head direction. Drag to aim; double-click to center." }, el("span", { class: "pad-label top" }, "up"), el("span", { class: "pad-label bottom" }, "down"), el("span", { class: "pad-label left" }, "left"), el("span", { class: "pad-label right" }, "right"), el("div", { class: "pad-cross" }), ui.dot, ); wirePad(ui.pad); const head = el("section", { class: "card" }, el("h2", {}, "Head"), ui.pad, slider("roll", "Tilt", -LIMITS.roll, LIMITS.roll, (v) => setCmd({ roll: v }), "Q / E"), ); // Body & antennas ui.linkAnt = el("input", { type: "checkbox" }); const bodyCard = el("section", { class: "card" }, el("h2", {}, "Body & antennas"), slider("body", "Body rotation", -LIMITS.body, LIMITS.body, (v) => setCmd({ body: -v }), "A / D", true), slider("antL", "Left antenna", -LIMITS.antenna, LIMITS.antenna, (v) => { setCmd(ui.linkAnt.checked ? { antL: v, antR: -v } : { antL: v }); }), // Right antenna is shown inverted so both sliders read "+ = outward". slider("antR", "Right antenna", -LIMITS.antenna, LIMITS.antenna, (v) => { setCmd(ui.linkAnt.checked ? { antR: -v, antL: v } : { antR: -v }); }, null, true), el("label", { class: "check" }, ui.linkAnt, "Mirror antennas"), ); // Emotes ui.emoteButtons = {}; const emoteGrid = el("div", { class: "emotes" }, EMOTES.map((e) => (ui.emoteButtons[e.id] = el("button", { type: "button", class: "emote", onclick: () => playEmote(e), title: `Shortcut: ${e.key}`, }, el("span", { class: "emote-icon", "aria-hidden": "true" }, e.icon), e.label, el("kbd", {}, e.key)))), ); ui.homeBtn = el("button", { type: "button", class: "secondary wide", onclick: goHome }, "Home pose ", el("kbd", {}, "H")); const emotes = el("section", { class: "card" }, el("h2", {}, "Emotes"), emoteGrid, ui.homeBtn); // Robot controls ui.trackBtn = el("button", { type: "button", class: "toggle", onclick: () => toggleTracking() }, "Face tracking"); ui.puppetBtn = el("button", { type: "button", class: "toggle", onclick: () => togglePuppet() }, "Puppet mode"); ui.wakeBtn = el("button", { type: "button", class: "secondary", onclick: wake }, "Wake up"); ui.sleepBtn = el("button", { type: "button", class: "secondary", onclick: goSleep }, "Sleep"); ui.volume = el("input", { type: "range", min: 0, max: 100, step: 1, value: 50, "aria-label": "Speaker volume" }); ui.volumeOut = el("output", {}, "50"); let volTimer = null; ui.volume.addEventListener("input", () => { ui.volumeOut.textContent = ui.volume.value; clearTimeout(volTimer); volTimer = setTimeout(() => robot.setVolume(Number(ui.volume.value)), 200); }); const controls = el("section", { class: "card" }, el("h2", {}, "Robot"), el("div", { class: "row2" }, ui.trackBtn, ui.puppetBtn), el("div", { class: "row2" }, ui.wakeBtn, ui.sleepBtn), el("div", { class: "slider" }, el("div", { class: "slider-head" }, el("span", {}, "Speaker volume"), ui.volumeOut), ui.volume, ), ); // Activity log ui.log = el("ol", { class: "log", "aria-live": "polite" }); const activity = el("section", { class: "card log-card" }, el("h2", {}, "Activity"), ui.log, el("p", { class: "muted small" }, "Keys: arrows aim · Q/E tilt · A/D body · 1–6 emotes · H home · T tracking"), ); const banner = demo ? el("div", { class: "banner" }, "Demo mode: no robot connected. The drawing shows what Reachy would do.") : null; mount.append(el("main", { class: "app" }, header, banner || "", el("div", { class: "grid" }, el("div", { class: "col" }, camera, activity), el("div", { class: "col" }, head, emotes), el("div", { class: "col" }, bodyCard, controls), ), )); // ---- UI builders ------------------------------------------------------------ function slider(key, label, min, max, onInput, hint, invert = false) { const input = el("input", { type: "range", min, max, step: 1, value: 0, "aria-label": label }); const out = el("output", {}, "0°"); input.addEventListener("input", () => onInput(Number(input.value))); ui[`slider_${key}`] = { input, out, invert }; return el("div", { class: "slider" }, el("div", { class: "slider-head" }, el("span", {}, label, hint ? el("kbd", {}, hint) : ""), out, ), input, ); } function wirePad(pad) { let dragging = false; const update = (e) => { const r = pad.getBoundingClientRect(); const x = clamp(((e.clientX - r.left) / r.width) * 2 - 1, -1, 1); const y = clamp(((e.clientY - r.top) / r.height) * 2 - 1, -1, 1); setCmd({ yaw: -x * LIMITS.yaw * YAW_SIGN, pitch: y * LIMITS.pitch * PITCH_SIGN }); }; pad.addEventListener("pointerdown", (e) => { dragging = true; pad.setPointerCapture(e.pointerId); pad.classList.add("active"); update(e); }); pad.addEventListener("pointermove", (e) => dragging && update(e)); const end = () => { dragging = false; pad.classList.remove("active"); }; pad.addEventListener("pointerup", end); pad.addEventListener("pointercancel", end); pad.addEventListener("dblclick", () => setCmd({ yaw: 0, pitch: 0 })); } function renderControls() { // Pad dot: x right = turn right (negative yaw), y down = pitch down. const x = (-cmd.yaw * YAW_SIGN) / LIMITS.yaw; const y = (cmd.pitch * PITCH_SIGN) / LIMITS.pitch; ui.dot.style.left = `${50 + x * 50}%`; ui.dot.style.top = `${50 + y * 50}%`; ui.pad.setAttribute("aria-valuetext", `Turn ${Math.round(Math.abs(cmd.yaw))}° ${cmd.yaw * YAW_SIGN >= 0 ? "left" : "right"}, ` + `${Math.round(Math.abs(cmd.pitch))}° ${cmd.pitch * PITCH_SIGN >= 0 ? "down" : "up"}`); for (const key of ["roll", "body", "antL", "antR"]) { const s = ui[`slider_${key}`]; const v = Math.round(s.invert ? -cmd[key] : cmd[key]); s.input.value = v; s.out.textContent = `${v}°`; } } function renderToggles() { ui.trackBtn.classList.toggle("on", tracking); ui.trackBtn.setAttribute("aria-pressed", String(tracking)); ui.puppetBtn.classList.toggle("on", puppet); ui.puppetBtn.setAttribute("aria-pressed", String(puppet)); for (const b of Object.values(ui.emoteButtons)) b.disabled = puppet; } function setBusy(id) { for (const [eid, b] of Object.entries(ui.emoteButtons)) b.classList.toggle("playing", eid === id); ui.wakeBtn.disabled = id === "wake" || id === "sleep"; ui.sleepBtn.disabled = id === "wake" || id === "sleep"; } function log(text, level = "info") { const time = new Date().toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit" }); ui.log.prepend(el("li", { class: level }, el("time", {}, time), text)); while (ui.log.children.length > 8) ui.log.lastChild.remove(); } // ---- Robot events ----------------------------------------------------------------- const MODE_LABELS = { enabled: "Motors on", gravity_compensation: "Puppet mode", disabled: "Motors off" }; const onState = (e) => { const s = e.detail || robot.robotState; if (!s) return; if (s.motor_mode) { ui.mode.textContent = MODE_LABELS[s.motor_mode] || s.motor_mode; const wasPuppet = puppet; puppet = s.motor_mode === "gravity_compensation"; if (wasPuppet !== puppet) renderToggles(); } if (!touched) syncFromState(); if (demo && s.head) ui.avatar.update(s, math); }; robot.addEventListener("state", onState); cleanups.push(() => robot.removeEventListener("state", onState)); const onStopped = () => { ui.status.textContent = "Disconnected"; ui.status.classList.add("off"); log("Session ended", "warn"); }; robot.addEventListener("sessionStopped", onStopped); cleanups.push(() => robot.removeEventListener("sessionStopped", onStopped)); const onError = (e) => log(`Robot error: ${e.detail?.error?.message || e.detail?.error || "unknown"}`, "warn"); robot.addEventListener("error", onError); cleanups.push(() => robot.removeEventListener("error", onError)); // Smooth live feedback while the app is open (no-op on older daemons). try { robot.subscribePose?.(); cleanups.push(() => robot.unsubscribePose?.()); } catch {} // Camera if (!demo) { try { const detach = robot.attachVideo(ui.video); if (typeof detach === "function") cleanups.push(detach); ui.video.addEventListener("playing", () => ui.videoNote.remove(), { once: true }); setTimeout(() => { if (ui.videoNote.isConnected) ui.videoNote.textContent = "No camera feed yet. Motion controls still work."; }, 6000); } catch (err) { ui.videoNote.textContent = "Camera unavailable"; } } // Volume Promise.resolve(robot.getVolume?.()).then((v) => { if (typeof v === "number") { ui.volume.value = v; ui.volumeOut.textContent = String(v); } }).catch(() => {}); // Resync when the tab comes back to the foreground. const onVisible = () => { if (!document.hidden) robot.requestState(); }; document.addEventListener("visibilitychange", onVisible); cleanups.push(() => document.removeEventListener("visibilitychange", onVisible)); // Keyboard shortcuts const onKey = (e) => { if (e.target.closest?.("input, textarea, select") || e.metaKey || e.ctrlKey || e.altKey) return; const k = e.key.toLowerCase(); const step = e.shiftKey ? KEY_STEP * 3 : KEY_STEP; const actions = { arrowleft: () => setCmd({ yaw: cmd.yaw + step * YAW_SIGN }), arrowright: () => setCmd({ yaw: cmd.yaw - step * YAW_SIGN }), arrowup: () => setCmd({ pitch: cmd.pitch - step * PITCH_SIGN }), arrowdown: () => setCmd({ pitch: cmd.pitch + step * PITCH_SIGN }), q: () => setCmd({ roll: cmd.roll - step }), e: () => setCmd({ roll: cmd.roll + step }), a: () => setCmd({ body: cmd.body + step }), d: () => setCmd({ body: cmd.body - step }), h: () => goHome(), t: () => toggleTracking(), }; const emote = EMOTES.find((x) => x.key === k); const action = actions[k] || (emote && (() => playEmote(emote))); if (action) { e.preventDefault(); action(); } }; window.addEventListener("keydown", onKey); cleanups.push(() => window.removeEventListener("keydown", onKey)); // ---- Start ------------------------------------------------------------------ renderControls(); renderToggles(); setBusy(null); robot.requestState(); log(demo ? "Demo robot ready" : `Connected to ${robotName}`); return { /** Stop everything and send Reachy back to rest. Safe to call more than once. */ shutdown() { emoteToken++; clearTimeout(sendTimer); try { if (tracking) robot.stopHeadTracking(); } catch {} try { robot.setMotorMode("enabled"); robot.gotoTarget({ ...toWire(HOME), duration: 1.0 }); } catch {} for (const fn of cleanups.splice(0)) { try { fn(); } catch {} } }, }; } // ---- Demo avatar -------------------------------------------------------------- // A simple front view of Reachy that mirrors the robot state in demo mode. function buildAvatar() { const NS = "http://www.w3.org/2000/svg"; const svg = document.createElementNS(NS, "svg"); svg.setAttribute("viewBox", "0 0 320 240"); svg.setAttribute("class", "avatar"); svg.setAttribute("role", "img"); svg.setAttribute("aria-label", "Simulated Reachy Mini"); svg.innerHTML = ` `; const headG = svg.querySelector(".av-head-g"); const bodyMark = svg.querySelector(".av-body-mark"); const eyes = svg.querySelector(".av-eyes"); const antR = svg.querySelector(".av-ant-r"); const antL = svg.querySelector(".av-ant-l"); function update(s, math) { const rpy = math.matrixToRpy(s.head); const body = s.body_yaw ? (s.body_yaw * 180) / Math.PI : 0; // Viewer faces the robot: robot's left (positive yaw) appears on our right. const dx = Math.sin((rpy.yaw * Math.PI) / 180) * 50; const dy = Math.sin((rpy.pitch * Math.PI) / 180) * 40; headG.setAttribute("transform", `translate(${160 + dx * 0.5} ${115 + dy * 0.4}) rotate(${rpy.roll})`); eyes.setAttribute("transform", `translate(${dx * 0.35} ${dy * 0.35})`); bodyMark.setAttribute("cx", String(160 + Math.sin((body * Math.PI) / 180) * 50)); const [rRad, lRad] = s.antennas || [0, 0]; // Robot's right antenna is on the viewer's left. antR.setAttribute("transform", `translate(-40 -40) rotate(${(rRad * 180) / Math.PI})`); antL.setAttribute("transform", `translate(40 -40) rotate(${(lRad * 180) / Math.PI})`); } return { svg, update }; }