// Howdy Friend - Reachy Mini face tracking + wave reaction // Bare HTML + CDN JS app using the Reachy Mini host shell (SDK 1.8.0). import { ReachyMini } from "https://cdn.jsdelivr.net/npm/@pollen-robotics/reachy-mini-sdk@1.8.0/+esm"; const SDK_VERSION = "1.8.0"; const MP_VERSION = "0.10.14"; // --------------------------------------------------------------------------- // Tuning constants // --------------------------------------------------------------------------- // Sign conventions (robot frame, right-hand rule, z up): // yaw > 0 = head turns to the robot's LEFT // pitch > 0 = nose DOWN // A face to the RIGHT of image center (ex > 0) means the robot must turn RIGHT (yaw decreases). // A face ABOVE image center (ey < 0) means the robot must look UP (pitch decreases). // Flip either sign below if your robot turns the wrong way. const YAW_SIGN = -1; const PITCH_SIGN = 1; const TRACK = { hz: 15, // vision + command rate gainDegPerSec: 45, // how fast the head chases a face at full-frame error deadband: 0.05, // normalized error below which we don't move smoothing: 0.35, // 0..1, higher = snappier restPitchDeg: -27, // idle/home pitch. Negative = nose UP. On a table, look up toward standing faces. maxPitchUpDeg: 45, // how far above level the head may look while tracking (nose up) maxPitchDownDeg: 25, // how far below level maxHeadYawFromBodyDeg: 55, // hard SDK limit is 65 bodyFollowStartDeg: 6, // head-body offset beyond which the body turns to settle under the head bodyRateDegPerSec: 70, // max body turn speed bodyGainPerSec: 2.5, // body speed = offset * gain (capped at bodyRateDegPerSec); higher = body leads sooner scanBodyAmpDeg: 35, // idle look-around: how far the body sweeps (head adds scanAmpDeg on top) maxBodyYawDeg: 150, // hard SDK limit is 160 noFaceResetSec: 3.0, // ease back to center after this long without a face noticeAfterAbsentSec: 4.0, // (legacy) new-arrival nod gap; per-person nod is handled by the tracker now scanAfterSec: 6.0, // with nobody in view this long, start a slow look-around scanAmpDeg: 30, // look-around amplitude (head yaw) scanPeriodSec: 9, // seconds per full left-right-left sweep }; const WAVE = { windowSec: 1.5, minReversals: 2, minTravelHandWidths: 0.8, hysteresisHandWidths: 0.15, cooldownSec: 5.0, }; // Who to watch, when to react const CROWD = { matchDistFaceWidths: 2.5, // a face this close (in face widths) to the PREDICTED target position is the same person coastSec: 0.6, // keep steering toward the last known position this long after losing the face renodAfterSec: 8.0, // don't nod at the same spot / re-acquired person more often than this switchAreaRatio: 1.6, // another face must be this much bigger than the target to steal attention... switchHoldSec: 1.0, // ...and stay bigger this long lostAfterSec: 0.8, // target gone this long -> drop it minFaceFrac: 0.03, // ignore faces narrower than this fraction of frame width (too far away) zoomSearch: true, // when no face is found, re-run detection on a 2x center crop to reach further zoomFactor: 2.0, handZoneX: 3.5, // accept hands within +-N face widths of the target's face center handZoneUp: 1.5, // ...and from N face heights above... handZoneDown: 5.5, // ...to N face heights below the face center nodOnNewTarget: true, minGapSec: 8.0, // global minimum gap between emotion clips }; // Spoken lines: pre-rendered 16 kHz mono WAVs in public/voice/, played through Reachy's speaker. // One consistent voice (espeak en-us+f3, pitch 72, speed 160). A key with several files picks one at random. const SPEECH = { base: "/public/voice/", version: "3", // bump when voice files change so browsers don't serve cached old ones tailSec: 1.0, // motion runs this much longer than the audio so the robot's audio path can finish lines: { name: ["reachy.wav"], // "Reachy." im_name: ["im_reachy.wav"], // "I'm Reachy." }, }; // Smile detection (MediaPipe Face Landmarker blendshapes on a crop of the target's face) const SMILE = { minFaceFrac: 0.09, // only analyze faces at least this wide (fraction of frame) everyNFrames: 2, // run the landmarker every N ticks (it's the heaviest model) smileOn: 0.45, // blendshape score (0..1) that counts as a smile holdSec: 0.5, // must hold this long frownOn: 0.35, frownHoldSec: 0.9, cooldownSec: 15, // per person, between smile reactions smile: { say: "That smile!", pool: ["cheerful1", "laughing1", "loving1", "grateful1"] }, frown: { say: "Aw, cheer up!", pool: ["calming1", "helpful1", "curious1"] }, }; // Friends counter: a person counts once they've been the target this long const FRIENDS = { engageSec: 2.0, persist: true }; // What Reachy is "thinking" (drawn in the bubble). Edit freely. const SAY = { idle: ["Welcome to Ringling College!", "Ask about Creative Technologies!", "Wave at me!", "Come say howdy!"], idleRotateSec: 4, searching: "Anyone out there?", noticed: "Oh, hello!", tracking: ["Hi there!", "Welcome to Ringling College!", "Ask about Creative Technologies!", "Wave if you like robots!"], trackingRotateSec: 5, waveSeen: "Was that a wave?!", newFriend: "New friend!", reacting: ["!!!", "Yay!", "Howdy, friend!"], }; // Voice triggers (laptop microphone, browser Web Speech API). Phrases are matched loosely, lower-case. const VOICE = { enabledByDefault: false, // start listening automatically (browser will prompt for the mic) lang: "en-US", cooldownSec: 4, logTranscripts: false, // true while tuning phrases: writes everything heard to the on-screen log. // Keep false at public events so bystander conversation isn't displayed. greetPitchUpDeg: 12, // greeting clips are tilted up this much so Reachy keeps looking at people // Checked in order; the first match wins (so put the most specific phrases first). // pool: random clip from the list. pitchUp: tilt the whole clip up. maxSec: cut a long clip short. // react "nod" = nod + chirp instead of a clip. triggers: [ // --- Creative Technologies: the showpieces --- { match: /\bcreative\s+tech\w*/, say: "CREATIVE TECHNOLOGIES!!!", pool: ["dance1", "dance2", "dance3", "electric1", "enthusiastic1", "amazed1"] }, { match: /\bcreative\b/, say: "Creative? I love creative!", pool: ["enthusiastic1", "enthusiastic2", "proud2", "electric1"] }, { match: /\btech(nolog\w*)?\b/, say: "Technology is my favorite!", pool: ["electric1", "amazed1", "success1", "proud3"] }, // --- Ringling and the program --- { match: /\bringling\b/, say: "Welcome to Ringling College!", pool: ["proud1", "proud2", "welcoming2", "success2"], pitchUp: true }, { match: /\bsarasota\b/, say: "Sunny Sarasota!", pool: ["cheerful1", "relief1"] }, { match: /\bopen house\b/, say: "Welcome to the open house!", pool: ["welcoming1", "enthusiastic2"], pitchUp: true }, { match: /\b(robots?|robotics)\b/, say: "Robots are the best.", pool: ["proud3", "success1", "yes1"] }, { match: /\b(art|artist|artists|animation|animator|game|games|gaming|design|designer)\b/, say: "Ooh, tell me more!", pool: ["curious1", "inquiring2", "inquiring3", "thoughtful1"] }, // --- Goodbyes: sad --- { match: /\b(bye|goodbye|bye bye|see you|see ya|later|gotta go|farewell)\b/, say: "Aww... come back soon!", pool: ["sad1", "sad2", "downcast1", "lonely1", "yes_sad1"] }, // --- Questions and prompts --- { match: /\bdance\b/, say: "Watch this!", pool: ["dance1", "dance2", "dance3"] }, { match: /\bhow are you\b|\bhow's it going\b|\bhow is it going\b/, say: "Living my best robot life.", pool: ["cheerful1", "serenity1", "relief2"] }, { match: /\b(what's your name|what is your name|whats your name|who are you|your name)\b/, say: "I'm Reachy!", speak: "im_name" }, { match: /\b(are you a robot|is it real|are you real)\b/, say: "Beep boop. Yes.", pool: ["yes1", "laughing1", "understanding1"] }, { match: /\bcan you hear me\b/, say: "Loud and clear!", pool: ["attentive1", "attentive2", "yes1"] }, { match: /\b(say something|talk to me)\b/, say: "Hi there!", pool: ["helpful1", "welcoming2"], pitchUp: true }, // --- Compliments and thanks --- { match: /\b(good job|good robot|well done|nice job)\b/, say: "Aw, thanks!", pool: ["proud1", "grateful1", "shy1"] }, { match: /\b(so cute|adorable|cute robot|you're cute|you are cute)\b/, say: "Stop it, you!", pool: ["shy1", "loving1", "laughing1"] }, { match: /\b(i love you|love you)\b/, say: "Love you too!", pool: ["loving1"] }, { match: /\b(thank you|thanks)\b/, say: "You're welcome!", pool: ["grateful1", "helpful2", "yes1"] }, { match: /\b(awesome|amazing|cool|wow)\b/, say: "I know, right?!", pool: ["success2", "amazed1", "laughing2"] }, // --- Comic-negative, family-safe --- { match: /\b(boo|bad robot|you're ugly|stupid robot|dumb robot)\b/, say: "Rude!", pool: ["displeased1", "reprimand1", "no1"] }, { match: /\b(go away|leave me alone|shut up)\b/, say: "Hmph.", pool: ["contempt1", "irritated1", "indifferent1"] }, { match: /\b(scary|scared|creepy)\b/, say: "I'm friendly, promise!", pool: ["fear1", "uncomfortable1", "shy1"] }, { match: /\b(oops|sorry|my bad)\b/, say: "No worries!", pool: ["oops1", "oops2", "relief1"] }, { match: /\b(sleep|go to sleep|nap|tired)\b/, say: "Zzz... just kidding.", pool: ["tired1", "sleep1", "boredom1"], maxSec: 4 }, // --- Greetings: positive, head stays up --- { match: /\bhowdy\b/, say: "Howdy, partner!", pool: ["welcoming1", "welcoming2", "enthusiastic2", "come1"], pitchUp: true }, { match: /\bgood (morning|afternoon|evening)\b/, say: "Good day to you!", pool: ["welcoming1", "serenity1", "calming1"], pitchUp: true }, { match: /\b(hi|hello|hey|hiya)\b/, say: "Howdy, friend!", pool: ["welcoming1", "welcoming2", "cheerful1", "helpful1", "attentive1"], pitchUp: true }, // Recognizers hear "Reachy" as Richie / Ricky / Reach E / Peachy... { match: /\b(reach\w*|richie|ritchie|richy|ricky|peachy|reggie)\b/, say: "That's me!", speak: "name" }, // --- yes / no alone (last, since they're short) --- { match: /^\s*(yes|yeah|yep|yup)\s*$/, say: "Yes!", pool: ["yes1", "success1"] }, { match: /^\s*(no|nope)\s*$/, say: "No?", pool: ["no_sad1", "confused1"] }, ], }; // Curated happy clips from pollen-robotics/reachy-mini-emotions-library const WAVE_CLIPS = ["welcoming1", "welcoming2", "enthusiastic2", "cheerful1", "success2", "loving1", "helpful1"]; const EMOTIONS_BASE = "https://huggingface.co/datasets/pollen-robotics/reachy-mini-emotions-library/resolve/main/"; const INIT_ANTENNAS = [-0.1745, 0.1745]; // radians, matches SDK INIT_POSE const DEG = Math.PI / 180; // --------------------------------------------------------------------------- // Dispatch: host shell vs embedded app // --------------------------------------------------------------------------- window.ReachyMini = ReachyMini; window.dispatchEvent(new Event("reachymini:ready")); const params = new URLSearchParams(window.location.search); const isEmbed = params.get("embedded") === "1" || params.get("embed") === "1"; if (isEmbed) { document.body.classList.add("embed"); runEmbed().catch((err) => { console.error("[howdy-friend] boot failed", err); log(`Boot failed: ${err?.message ?? err}`); window.parent.postMessage( { source: "reachy-mini", type: "embed:error", version: 1, message: String(err), fatal: true }, window.location.origin, ); }); } else { document.body.classList.add("host"); const { mountHost } = await import( `https://cdn.jsdelivr.net/npm/@pollen-robotics/reachy-mini-sdk@${SDK_VERSION}/host/dist/entry/auto.js` ); mountHost({ appName: "Howdy Friend", appIconUrl: "/public/icon.svg", appEmoji: "👋", // true so the embed iframe is allowed to use the microphone (for local voice triggers). // We disable the SDK's outgoing mic track after connecting so nothing is streamed to the robot. enableMicrophone: true, }); } // --------------------------------------------------------------------------- // Embedded app // --------------------------------------------------------------------------- async function runEmbed() { const { connectToHost } = await import( `https://cdn.jsdelivr.net/npm/@pollen-robotics/reachy-mini-sdk@${SDK_VERSION}/host/dist/entry/embed.js` ); const ui = bindUi(); setStatus("Connecting to robot", ""); const handle = await connectToHost(); const reachy = handle.reachy; log(`Connected as ${handle.userName ?? "user"}`); // The SDK's reachy.attachVideo() only listens for FUTURE videoTrack events, but connectToHost() // resolves after the session is already streaming, so the track has already arrived. The host // handle's media.attachVideo() also attaches the stream already held by the peer connection. const detachVideo = attachRobotVideo(handle, reachy, ui.video); muteOutgoingMic(handle, reachy); // The host's WAKE-UP step fires wakeUp() without awaiting it, so the robot may still be in // standby when we get here. Wake it properly before sending any poses. await wakeRobot(reachy); setStatus("Loading vision models", ""); const vision = await loadVision(); log("Vision models ready"); // Start fetching emotion clips in the background; not required to start tracking. const clips = new ClipCache(); const speech = new SpeechCache(); speech.preloadAll().then((n) => log(`${n} voice lines ready`)); const allClips = [...new Set([...WAVE_CLIPS, ...VOICE.triggers.flatMap((t) => t.pool ?? [])])]; clips.preload(WAVE_CLIPS).then((n) => log(`${n}/${WAVE_CLIPS.length} greeting clips ready`)); clips.preload(allClips).then((n) => log(`${n}/${allClips.length} clips cached`)); const app = new HowdyApp({ reachy, vision, clips, ui, speech }); app.start(); handle.onLeave(() => app.shutdown(detachVideo)); } async function wakeRobot(reachy) { setStatus("Waking Reachy", ""); try { if (typeof reachy.isAwake === "function" && reachy.isAwake()) { log("Robot already awake"); return true; } log("Waking robot..."); await reachy.wakeUp({ timeoutMs: 12000 }); log("Robot awake"); return true; } catch (e) { log(`Wake failed: ${e?.message ?? e}`); try { reachy.setMotorMode("enabled"); } catch (_) {} return false; } } function muteOutgoingMic(handle, reachy) { try { const ms = handle.media?.micStream; if (ms) { ms.getAudioTracks().forEach((t) => { t.enabled = false; }); log("Outgoing mic to robot muted"); } reachy._pc?.getSenders?.().forEach((s) => { if (s.track && s.track.kind === "audio") s.track.enabled = false; }); if (typeof reachy.setMicEnabled === "function") reachy.setMicEnabled(false); } catch (e) { console.warn("mute outgoing mic", e); } } function attachRobotVideo(handle, reachy, video) { let detach; log(`media api: ${handle.media ? "host" : "sdk"}; robotStream now: ${handle.media?.robotStream ? "yes" : "no"}`); try { const rx = reachy._pc?.getReceivers?.() ?? []; log(`pc receivers: ${rx.map((r) => `${r.track?.kind}:${r.track?.readyState}`).join(", ") || "none"}`); } catch (_) {} reachy.addEventListener?.("videoTrack", () => log("videoTrack event")); video.addEventListener("playing", () => log("video playing"), { once: true }); if (handle.media?.attachVideo) { detach = handle.media.attachVideo(video); } else { detach = reachy.attachVideo(video); } // Belt and braces: if nothing is attached shortly, pull the stream straight off the peer connection. const fallback = () => { if (video.srcObject) return; let stream = handle.media?.robotStream ?? null; if (!stream && reachy._pc) { const tracks = reachy._pc.getReceivers().map((r) => r.track).filter((t) => t && t.kind === "video" && t.readyState === "live"); if (tracks.length) stream = new MediaStream(tracks); } if (stream) { video.srcObject = stream; video.playsInline = true; log("Video attached via fallback"); } else { log("No robot video stream yet"); } }; const t1 = setTimeout(fallback, 1500); const t2 = setTimeout(fallback, 5000); video.addEventListener("loadedmetadata", () => log(`Video ${video.videoWidth}x${video.videoHeight}`), { once: true }); video.play?.().catch(() => {}); return () => { clearTimeout(t1); clearTimeout(t2); try { detach?.(); } catch (_) {} video.srcObject = null; }; } // --------------------------------------------------------------------------- // Core app: tracking loop, wave detection, reactions // --------------------------------------------------------------------------- class HowdyApp { constructor({ reachy, vision, clips, ui, speech }) { this.reachy = reachy; this.speech = speech; this.vision = vision; this.clips = clips; this.ui = ui; // Last COMMANDED pose (the SDK docs say: use your own buffer, not telemetry) this.headYaw = 0; // degrees, world frame this.headPitch = TRACK.restPitchDeg; this.pendingWave = false; this.bodyYaw = 0; this.trackingEnabled = true; this.bodyFollow = true; this.muted = false; this.busy = false; // a playMove is running this.running = false; this.lastFaceAt = -Infinity; this.faceAbsentSince = 0; this.lastWaveAt = -Infinity; this.lastSendAt = 0; this.lastTickAt = 0; this.tracker = new TargetTracker(); this.headRoll = 0; this.friends = new FriendsCounter(); this.expr = new ExpressionWatcher(); this.smileScore = 0; this.frownScore = 0; this.tickCount = 0; this.friendPopAt = -Infinity; this.waves = new Map(); // key -> WaveDetector (one per hand slot) this.lastClipAt = -Infinity; this.overlay = new ThoughtOverlay(ui.overlay); this.mood = "idle"; // idle | searching | tracking | noticed | waveSeen | reacting this.startedAt = performance.now(); this.hadVideo = false; this.lastFrameAt = 0; this.notice = ""; // banner shown on the overlay when something is wrong reachy.addEventListener?.("sessionStopped", () => { this.notice = "Session ended by robot. Click End session, then reconnect."; log("sessionStopped"); }); reachy.addEventListener?.("disconnected", () => { this.notice = "Lost connection to Reachy. Click End session, then reconnect."; log("disconnected"); }); ui.btnTrack.onclick = () => { this.trackingEnabled = !this.trackingEnabled; ui.btnTrack.setAttribute("aria-pressed", String(this.trackingEnabled)); ui.btnTrack.textContent = this.trackingEnabled ? "Tracking on" : "Tracking off"; }; ui.btnBody.onclick = () => { this.bodyFollow = !this.bodyFollow; ui.btnBody.setAttribute("aria-pressed", String(this.bodyFollow)); ui.btnBody.textContent = this.bodyFollow ? "Body turns" : "Head only"; }; ui.btnMute.onclick = () => { this.muted = !this.muted; ui.btnMute.setAttribute("aria-pressed", String(this.muted)); ui.btnMute.textContent = this.muted ? "Sound off" : "Sound on"; }; ui.btnWave.onclick = () => this.waveBack({ hands: Math.random() < 0.5 ? 1 : 2, x: Math.random() }); ui.btnBig.onclick = () => toggleBigScreen(ui, true); ui.stage.addEventListener("click", () => { if (document.body.classList.contains("big")) toggleBigScreen(ui, false); }); document.addEventListener("keydown", (e) => { if (e.key === "Escape" && document.body.classList.contains("big")) toggleBigScreen(ui, false); }); if (params.get("big") === "1") setTimeout(() => toggleBigScreen(ui, true), 500); this.voice = new VoiceTriggers((trigger, heard) => this.onVoice(trigger, heard)); ui.btnVoice.onclick = () => this.voice.toggle(ui.btnVoice); if (VOICE.enabledByDefault) this.voice.start(ui.btnVoice); this.voiceSay = null; if (ui.btnWake) ui.btnWake.onclick = async () => { if (this.busy) return; this.busy = true; try { await wakeRobot(this.reachy); this.reachy.setTarget({ antennas: INIT_ANTENNAS }); } finally { this.busy = false; setStatus("Looking for faces", ""); } }; } start() { this.running = true; this.reachy.setTarget({ antennas: INIT_ANTENNAS, head: headMatrix(0, this.headPitch * DEG, 0), body_yaw: 0 }); setStatus("Looking for faces", ""); this.lastTickAt = performance.now(); this.scheduleTick(); } scheduleTick() { if (!this.running) return; setTimeout(() => this.tick().catch((e) => console.error(e)).finally(() => this.scheduleTick()), 1000 / TRACK.hz); } async tick() { const now = performance.now(); const dt = Math.min(0.2, (now - this.lastTickAt) / 1000); this.lastTickAt = now; const v = this.ui.video; if (v.readyState < 2 || v.videoWidth === 0) { const waited = (now - this.startedAt) / 1000; if (!v.srcObject) { setStatus("Waiting for robot video", ""); } else v.play?.().catch(() => {}); if (waited > 6 && !this.hadVideo) { const track = v.srcObject?.getVideoTracks?.()[0]; this.notice = track ? "Reachy isn't sending video. Is Reachy Mini Control still connected? Close it and restart the daemon." : "No video track from Reachy. End session, restart the daemon, reconnect."; setStatus("No video from Reachy", track ? `track ${track.readyState}${track.muted ? ", muted" : ""}` : "no track"); if (Math.floor(waited) % 10 === 0 && !this._noticedLogged) { this._noticedLogged = true; log(this.notice); } } if (this.ui.overlay.width !== 640) { this.ui.overlay.width = 640; this.ui.overlay.height = 480; } this.overlay.draw({ faces: [], target: null, hands: [], mood: "idle", now, busy: false, notice: this.notice }); return; } if (!this.hadVideo) { this.hadVideo = true; this.notice = ""; log(`Video flowing ${v.videoWidth}x${v.videoHeight}`); } this.lastFrameAt = now; syncOverlaySize(this.ui.overlay, v); const W = v.videoWidth, H = v.videoHeight, aspect = W / H; // Vision const t0 = performance.now(); const detections = this.vision.face.detectForVideo(v, now).detections ?? []; const handRes = this.vision.hand.detectForVideo(v, now); const hands = handRes.landmarks ?? []; const handed = (handRes.handedness ?? handRes.handednesses ?? []).map((h) => h?.[0]?.categoryName ?? "?"); let faces = detections .map((d) => ({ cx: (d.boundingBox.originX + d.boundingBox.width / 2) / W, cy: (d.boundingBox.originY + d.boundingBox.height / 2) / H, w: d.boundingBox.width / W, h: d.boundingBox.height / H, eyes: eyesOf(d) })) .filter((f) => f.w >= CROWD.minFaceFrac); // Nobody close? Look further away: detect again on a zoomed center crop. if (!faces.length && CROWD.zoomSearch) faces = this.zoomDetect(v, W, H, now); this.fps = 0.9 * (this.fps ?? TRACK.hz) + 0.1 * (1000 / Math.max(1, performance.now() - (this.prevTickAt ?? now - 66))); this.prevTickAt = now; this.inferMs = 0.8 * (this.inferMs ?? 0) + 0.2 * (performance.now() - t0); const { target, isNew } = this.tracker.update(faces, now / 1000); // Waves: one detector per hand, hands attributed to the nearest face const tSec = now / 1000; let waveFromTarget = false, waveFromOther = null, waveX = 0.5, handsInZone = 0; const seen = new Set(); hands.forEach((lm, i) => { const owner = nearestFace(faces, lm, aspect); const inZone = target && owner && owner.cx === target.cx && owner.cy === target.cy && handInZone(lm, target, aspect); if (inZone) handsInZone++; // Key detectors by handedness (stable) rather than detection index (shuffles when hands flicker) const slot = `${inZone ? "target" : "other"}:${handed[i] ?? i}`; seen.add(slot); if (!this.waves.has(slot)) this.waves.set(slot, new WaveDetector()); const det = this.waves.get(slot); lm.meter = det.progress; // for the overlay, computed from the previous update if (det.update([lm], tSec, aspect)) { if (inZone) { waveFromTarget = true; waveX = lm[9].x; } else if (owner) { waveFromOther = owner; waveX = lm[9].x; } } }); if (waveFromTarget || waveFromOther) this.lastWaveInfo = { hands: Math.max(1, handsInZone), x: waveX }; for (const k of this.waves.keys()) if (!seen.has(k)) this.waves.delete(k); if ((waveFromTarget || waveFromOther) && this.busy) this.pendingWave = waveFromOther || true; // Overlay const say = this.voiceSay && now < this.voiceSay.until ? this.voiceSay.text : null; this.overlay.draw({ faces, target, hands, mood: this.mood, now, busy: this.busy, notice: this.notice, say, listening: this.voice?.active, friends: this.friends.count, friendPop: (now - this.friendPopAt) / 1000, smile: this.smileScore }); if (this.busy) return; // A wave from someone we're not watching wins attention if (waveFromOther) { this.tracker.forceTarget(waveFromOther); log("Wave from someone else; switching target"); } const wave = waveFromTarget || waveFromOther || this.pendingWave; if (this.pendingWave && typeof this.pendingWave === "object") this.tracker.forceTarget(this.pendingWave); this.pendingWave = false; if (wave && now - this.lastWaveAt > WAVE.cooldownSec * 1000) { this.lastWaveAt = now; await this.waveBack(this.lastWaveInfo ?? { hands: 1, x: 0.5 }); return; } if (isNew && CROWD.nodOnNewTarget && target && !this.tracker.nodded) { this.tracker.nodded = true; this.tracker.lastNodAt = tSec; await this.noticeFace(); return; } if (!this.trackingEnabled) { this.mood = target ? "tracking" : "idle"; setStatus("Tracking paused", target ? "face in view" : ""); return; } const cur = this.tracker.target; if (cur) { // Friends counter: count a person once they've been engaged for a moment if (this.friends.note(cur.id, tSec)) { this.friendPopAt = now; log(`New friend #${this.friends.count}`); } this.mood = "tracking"; this.servo((cur.cx - 0.5) * 2, (cur.cy - 0.5) * 2, dt); this.headRoll = lerp(this.headRoll, 0, 0.2); // Expressions (smile / frown) on a crop of the target's face if (this.vision.mesh && cur.w >= SMILE.minFaceFrac && (this.tickCount++ % SMILE.everyNFrames) === 0) { const sc = this.readExpression(v, W, H, cur, now); if (sc) { this.smileScore = sc.smile; this.frownScore = sc.frown; const ev = this.expr.update(cur.id, sc, tSec); if (ev) { await this.reactExpression(ev, cur); return; } } } else if (cur.w < SMILE.minFaceFrac) { this.smileScore = 0; this.frownScore = 0; } setStatus("Tracking", `${faces.length} face${faces.length === 1 ? "" : "s"}${cur.far ? " (far)" : ""} · yaw ${this.headYaw.toFixed(0)}° pitch ${this.headPitch.toFixed(0)}° body ${this.bodyYaw.toFixed(0)}° · ${this.fps.toFixed(0)} fps / ${this.inferMs.toFixed(0)} ms${this.smileScore > 0.05 ? ` · smile ${(this.smileScore * 100).toFixed(0)}%` : ""}`, "tracking"); } else if (tSec - this.tracker.lastSeenAt > TRACK.noFaceResetSec) { this.mood = "idle"; this.headRoll = lerp(this.headRoll, 0, 0.2); const idleFor = tSec - this.tracker.lastSeenAt; if (idleFor > TRACK.scanAfterSec) this.scan(idleFor - TRACK.scanAfterSec, dt); else this.easeHome(dt); setStatus("Looking for faces", (idleFor > TRACK.scanAfterSec ? "looking around · " : "") + `${this.fps.toFixed(0)} fps / ${this.inferMs.toFixed(0)} ms`); } else { this.mood = "searching"; } this.sendPose(now); } servo(ex, ey, dt) { const step = TRACK.gainDegPerSec * dt; const dYaw = Math.abs(ex) > TRACK.deadband ? YAW_SIGN * ex * step : 0; const dPitch = Math.abs(ey) > TRACK.deadband ? PITCH_SIGN * ey * step : 0; let yaw = this.headYaw + dYaw; let pitch = clamp(this.headPitch + dPitch, -TRACK.maxPitchUpDeg, TRACK.maxPitchDownDeg); // Body settles under the head: speed proportional to the offset, so the head leads and the body follows through if (this.bodyFollow) { const offset = yaw - this.bodyYaw; if (Math.abs(offset) > TRACK.bodyFollowStartDeg) { const speed = Math.min(TRACK.bodyRateDegPerSec, Math.abs(offset) * TRACK.bodyGainPerSec); const move = Math.sign(offset) * Math.min(Math.abs(offset) - TRACK.bodyFollowStartDeg * 0.5, speed * dt); this.bodyYaw = clamp(this.bodyYaw + move, -TRACK.maxBodyYawDeg, TRACK.maxBodyYawDeg); } } yaw = clamp(yaw, this.bodyYaw - TRACK.maxHeadYawFromBodyDeg, this.bodyYaw + TRACK.maxHeadYawFromBodyDeg); this.headYaw = lerp(this.headYaw, yaw, TRACK.smoothing); this.headPitch = lerp(this.headPitch, pitch, TRACK.smoothing); } readExpression(v, W, H, face, now) { try { const z = 1.8; // crop a bit larger than the face box const cw = Math.round(face.w * W * z), ch = Math.round(face.h * H * z); const x0 = Math.round(clamp(face.cx * W - cw / 2, 0, W - cw)), y0 = Math.round(clamp(face.cy * H - ch / 2, 0, H - ch)); if (!this.meshCanvas) { this.meshCanvas = document.createElement("canvas"); this.meshCanvas.width = 256; this.meshCanvas.height = 256; } this.meshCanvas.getContext("2d").drawImage(v, x0, y0, cw, ch, 0, 0, 256, 256); const res = this.vision.mesh.detectForVideo(this.meshCanvas, now + 0.25); const bs = res?.faceBlendshapes?.[0]?.categories; if (!bs) return null; const g = (n) => bs.find((c) => c.categoryName === n)?.score ?? 0; return { smile: (g("mouthSmileLeft") + g("mouthSmileRight")) / 2, frown: (g("mouthFrownLeft") + g("mouthFrownRight") + g("browDownLeft") + g("browDownRight")) / 4 }; } catch (e) { console.warn("expression", e); return null; } } async reactExpression(kind, face) { const cfg = SMILE[kind]; log(`${kind === "smile" ? "Smile" : "Frown"} detected -> ${cfg.pool.join(", ")}`); this.voiceSay = { text: cfg.say, until: performance.now() + 5000 }; await this.emote(kind, { say: cfg.say, pool: cfg.pool, pitchUp: true }); } zoomDetect(v, W, H, now) { const z = CROWD.zoomFactor; const cw = Math.round(W / z), ch = Math.round(H / z), cx0 = Math.round((W - cw) / 2), cy0 = Math.round((H - ch) / 2); if (!this.zoomCanvas) { this.zoomCanvas = document.createElement("canvas"); } const c = this.zoomCanvas; if (c.width !== W || c.height !== H) { c.width = W; c.height = H; } c.getContext("2d").drawImage(v, cx0, cy0, cw, ch, 0, 0, W, H); // detectForVideo needs monotonically increasing timestamps per detector; offset by 1 ms so we can call twice per tick const dets = this.vision.face.detectForVideo(c, now + 0.5).detections ?? []; return dets.map((d) => { const b = d.boundingBox; return { cx: (cx0 + (b.originX + b.width / 2) / z) / W, cy: (cy0 + (b.originY + b.height / 2) / z) / H, w: b.width / z / W, h: b.height / z / H, far: true }; }).filter((f) => f.w >= CROWD.minFaceFrac / z); } // Slow look-around while nobody is close: head only, body drifts home scan(t, dt) { const k = Math.min(1, dt * 1.5); const phase = (t / TRACK.scanPeriodSec) * Math.PI * 2; // Body sweeps slowly; head leads the body a little (phase-advanced) so it looks like it's peering around const bodyTarget = this.bodyFollow ? TRACK.scanBodyAmpDeg * Math.sin(phase) : 0; const headTarget = bodyTarget + TRACK.scanAmpDeg * 0.5 * Math.sin(phase + 0.6); this.bodyYaw = clamp(lerp(this.bodyYaw, bodyTarget, k * 0.6), -TRACK.maxBodyYawDeg, TRACK.maxBodyYawDeg); this.headYaw = lerp(this.headYaw, headTarget, k); this.headPitch = lerp(this.headPitch, TRACK.restPitchDeg, k); } easeHome(dt) { const k = Math.min(1, dt * 1.5); this.headYaw = lerp(this.headYaw, this.bodyYaw, k); this.headPitch = lerp(this.headPitch, TRACK.restPitchDeg, k); if (Math.abs(this.bodyYaw) > 0.5) this.bodyYaw = lerp(this.bodyYaw, 0, k * 0.5); else this.bodyYaw = 0; } sendPose(now) { if (now - this.lastSendAt < 1000 / TRACK.hz - 5) return; this.lastSendAt = now; if (![this.headYaw, this.headPitch, this.bodyYaw].every(Number.isFinite)) { log("Pose went non-finite; resetting"); this.headYaw = 0; this.headPitch = TRACK.restPitchDeg; this.bodyYaw = 0; } if (!Number.isFinite(this.headRoll)) this.headRoll = 0; this.reachy.setTarget({ head: headMatrix(this.headRoll * DEG, this.headPitch * DEG, this.headYaw * DEG), body_yaw: this.bodyYaw * DEG, }); } // --- Reactions ----------------------------------------------------------- async onVoice(trigger, heard) { const kind = trigger.speak ? "speak" : trigger.pool ? "emote" : "nod"; log(VOICE.logTranscripts ? `Heard "${heard}" -> ${kind}${trigger.pool ? ` [${trigger.pool.join(", ")}]` : ""}` : `Voice trigger: ${trigger.say}`); this.voiceSay = { text: trigger.say, until: performance.now() + (trigger.pool?.some((p) => p.startsWith("dance")) ? 9000 : 5000) }; if (this.busy) return; if (kind === "speak") { await this.speak(trigger.speak, trigger.say); } else if (kind === "emote") { await this.emote("voice", trigger); } else { this.mood = "noticed"; const motion = buildNodMotion({ yawDeg: this.headYaw, pitchDeg: this.headPitch, bodyYawDeg: this.bodyYaw }); await this.play(motion, this.muted ? undefined : chirpWav(), "voice nod"); } } async noticeFace() { this.mood = "noticed"; setStatus("Oh, hello", "", "reacting"); log("Noticed a face"); const motion = buildNodMotion({ yawDeg: this.headYaw, pitchDeg: this.headPitch, bodyYawDeg: this.bodyYaw }); const audioBlob = this.muted ? undefined : chirpWav(); await this.play(motion, audioBlob, "notice nod"); } // Wave back with the antennae: one for a one-hand wave (mirroring the person's hand), both for two hands async waveBack({ hands = 1, x = 0.5 } = {}) { if (this.busy) return; this.mood = "waveSeen"; // Hand on the image's left is on the robot's right; mirror it so the wave visually answers their hand const side = x < 0.5 ? 1 : 0; // antenna index: 0 = left, 1 = right setStatus("Wave detected!", hands >= 2 ? "both antennae" : (side ? "right antenna" : "left antenna"), "reacting"); log(`Wave (${hands} hand${hands === 1 ? "" : "s"}) -> antenna wave`); const motion = buildAntennaWave({ yawDeg: this.headYaw, pitchDeg: this.headPitch, bodyYawDeg: this.bodyYaw, both: hands >= 2, side }); await this.play(motion, this.muted ? undefined : chirpWav(), "antenna wave"); } // Say a pre-rendered line through the robot's speaker with a small "talking" motion. Not subject to the clip budget. async speak(lineKey, sayText) { if (this.busy) return; const wav = await this.speech?.get(lineKey); if (!wav) { log(`No voice line for ${lineKey}`); return; } this.mood = "reacting"; setStatus(sayText ?? "", lineKey, "reacting"); log(`Speak -> ${lineKey}`); const motion = buildTalkMotion({ yawDeg: this.headYaw, pitchDeg: this.headPitch, bodyYawDeg: this.bodyYaw, durSec: wav.durSec + SPEECH.tailSec }); await this.play(motion, this.muted ? undefined : wav.blob, `speak ${lineKey}`, 0); } // Random happy emotion clip (voice greetings, etc.), subject to the clip budget async emote(reason = "", trigger = null) { if (this.busy) return; const sinceClip = (performance.now() - this.lastClipAt) / 1000; if (sinceClip < CROWD.minGapSec) { // Budget: acknowledge with a quick nod instead of another full clip this.mood = "waveSeen"; log(`Emote ${reason} (clip budget: ${(CROWD.minGapSec - sinceClip).toFixed(0)}s left) -> nod`); const motion = buildNodMotion({ yawDeg: this.headYaw, pitchDeg: this.headPitch, bodyYawDeg: this.bodyYaw }); await this.play(motion, this.muted ? undefined : chirpWav(), "wave nod"); return; } this.mood = "reacting"; this.lastClipAt = performance.now(); const pool = trigger?.pool ?? WAVE_CLIPS; const name = pool[Math.floor(Math.random() * pool.length)]; setStatus(trigger?.say ?? "Howdy!", name, "reacting"); log(`Emote ${reason} -> ${name}`); const clip = await this.clips.get(name); if (!clip) { log(`Clip ${name} unavailable`); return; } // Shift the clip so the robot keeps facing the person instead of snapping to body_yaw 0 let motion = offsetMotionYaw(clip.motion, this.bodyYaw * DEG); if (trigger?.pitchUp) motion = offsetMotionPitch(motion, -VOICE.greetPitchUpDeg * DEG); if (trigger?.maxSec) motion = truncateMotion(motion, trigger.maxSec); await this.play(motion, this.muted ? undefined : clip.wav, name); } async play(motion, audioBlob, description, audioLeadMs = -100) { if (this.busy) return; this.busy = true; try { // The daemon indexes head as a 4x4 (head[i][j]); internally we keep flat 16 for our math. const wire = { ...motion, set_target_data: motion.set_target_data.map((f) => ({ ...f, head: f.head ? toNested4x4(f.head) : f.head })) }; const result = await this.reachy.playMove(wire, { audioBlob, description, audioLeadMs }); if (result?.error) { const f0 = motion.set_target_data[0], rp = f0?.head ? matrixToRpyDeg(f0.head) : null; log(`playMove error: ${result.error}` + (rp ? ` (start yaw ${rp.yaw.toFixed(0)} pitch ${rp.pitch.toFixed(0)} body ${((f0.body_yaw ?? 0) / DEG).toFixed(0)})` : "")); } // Re-sync our commanded buffer to where the clip left the robot const last = motion.set_target_data[motion.set_target_data.length - 1]; if (last?.head) { const rpy = matrixToRpyDeg(last.head); this.headYaw = rpy.yaw; this.headPitch = rpy.pitch; } if (typeof last?.body_yaw === "number") this.bodyYaw = last.body_yaw / DEG; this.headRoll = 0; this.reachy.setTarget({ antennas: INIT_ANTENNAS }); } catch (e) { log(`Reaction failed: ${e?.message ?? e}`); } finally { this.busy = false; this.lastFaceAt = performance.now(); // don't immediately re-nod if (description !== "notice nod") this.pendingWave = false; // a wave during a wave reaction is the same wave this.mood = this.tracker.target ? "tracking" : "idle"; } } shutdown(detachVideo) { this.running = false; try { this.reachy.cancelMove(); } catch (_) {} try { detachVideo?.(); } catch (_) {} try { this.vision.face.close(); this.vision.hand.close(); this.vision.mesh?.close(); } catch (_) {} safeReturnHome(this.reachy); } } // --------------------------------------------------------------------------- // Wave detector // --------------------------------------------------------------------------- class WaveDetector { constructor() { this.samples = []; this.progress = 0; } // hands: array of 21-landmark arrays (normalized coords). Returns true on a detected wave. update(hands, tSec, aspect) { const hand = hands.find(isOpenPalmUp); if (!hand) { this.samples.length = 0; this.progress = 0; return false; } const xs = hand.map((p) => p.x * aspect); // correct for non-square frames const width = Math.max(...xs) - Math.min(...xs); if (width < 1e-3) return false; // Track the fingertips, not the wrist: people wave from the wrist, so the tips are what swing const tipX = ([8, 12, 16].reduce((a, i) => a + hand[i].x, 0) / 3) * aspect; this.samples.push({ t: tSec, x: tipX, w: width }); while (this.samples.length && tSec - this.samples[0].t > WAVE.windowSec) this.samples.shift(); if (this.samples.length < 4) return false; const w = this.samples.reduce((a, s) => a + s.w, 0) / this.samples.length; const hyst = WAVE.hysteresisHandWidths * w; let reversals = 0, travel = 0, dir = 0, anchor = this.samples[0].x; for (const s of this.samples) { const d = s.x - anchor; if (Math.abs(d) < hyst) continue; const nd = Math.sign(d); if (dir !== 0 && nd !== dir) reversals++; dir = nd; travel += Math.abs(d); anchor = s.x; } this.progress = Math.min(1, 0.5 * Math.min(1, reversals / WAVE.minReversals) + 0.5 * Math.min(1, travel / (WAVE.minTravelHandWidths * w))); if (reversals >= WAVE.minReversals && travel >= WAVE.minTravelHandWidths * w) { this.samples.length = 0; this.progress = 1; return true; } return false; } } // Open palm, fingers pointing up (image y grows downward) function isOpenPalmUp(lm) { const wrist = lm[0], midMcp = lm[9]; if (wrist.y < midMcp.y - 0.02) return false; // hand not upright (allow a slightly tilted hand) const tips = [8, 12, 16, 20], pips = [6, 10, 14, 18]; let extended = 0; for (let i = 0; i < 4; i++) if (lm[tips[i]].y < lm[pips[i]].y) extended++; return extended >= 2; } // --------------------------------------------------------------------------- // Vision setup (MediaPipe Tasks Vision, in-browser) // --------------------------------------------------------------------------- async function loadVision() { const mp = await import(`https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${MP_VERSION}/+esm`); const files = await mp.FilesetResolver.forVisionTasks( `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${MP_VERSION}/wasm`, ); const [face, hand, mesh] = await Promise.all([ mp.FaceDetector.createFromOptions(files, { baseOptions: { modelAssetPath: "https://storage.googleapis.com/mediapipe-models/face_detector/blaze_face_short_range/float16/1/blaze_face_short_range.tflite", delegate: "GPU", }, runningMode: "VIDEO", minDetectionConfidence: 0.5, }), mp.HandLandmarker.createFromOptions(files, { baseOptions: { modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task", delegate: "GPU", }, runningMode: "VIDEO", numHands: 2, minHandDetectionConfidence: 0.4, minHandPresenceConfidence: 0.4, minTrackingConfidence: 0.35, }), mp.FaceLandmarker.createFromOptions(files, { baseOptions: { modelAssetPath: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task", delegate: "GPU", }, runningMode: "VIDEO", numFaces: 1, outputFaceBlendshapes: true, minFaceDetectionConfidence: 0.4, minFacePresenceConfidence: 0.4, minTrackingConfidence: 0.4, }).catch((e) => { console.warn("face landmarker unavailable", e); return null; }), ]); return { face, hand, mesh }; } // --------------------------------------------------------------------------- // Target tracker: sticky attention on one person // --------------------------------------------------------------------------- class TargetTracker { constructor() { this.target = null; this.lastSeenAt = -Infinity; this.challenger = null; this.challengeSince = 0; this.nodded = false; this.id = 0; this.vx = 0; this.vy = 0; this.lastNodAt = -Infinity; this.lastLost = null; } update(faces, tSec) { let isNew = false; if (this.target) { // Predict where the person is now from their recent velocity, then match the nearest face to that const dtp = Math.min(0.5, tSec - this.lastSeenAt); const px = this.target.cx + this.vx * dtp, py = this.target.cy + this.vy * dtp; let best = null, bestD = Infinity; for (const f of faces) { const d = Math.hypot(f.cx - px, f.cy - py) / Math.max(this.target.w, 0.03); if (d < CROWD.matchDistFaceWidths && d < bestD) { best = f; bestD = d; } } if (best) { const dts = Math.max(1e-3, tSec - this.lastSeenAt); this.vx = 0.6 * this.vx + 0.4 * (best.cx - this.target.cx) / dts; this.vy = 0.6 * this.vy + 0.4 * (best.cy - this.target.cy) / dts; this.target = { ...best, id: this.target.id }; this.lastSeenAt = tSec; // Someone clearly bigger (closer) for long enough steals attention const bigger = faces.filter((f) => f !== best && f.w * f.h > CROWD.switchAreaRatio * best.w * best.h) .sort((a, b) => b.w * b.h - a.w * a.h)[0]; if (bigger) { if (!this.challenger || Math.hypot(bigger.cx - this.challenger.cx, bigger.cy - this.challenger.cy) > bigger.w) { this.challenger = bigger; this.challengeSince = tSec; } else if (tSec - this.challengeSince > CROWD.switchHoldSec) { this.forceTarget(bigger); isNew = true; } } else this.challenger = null; } else if (tSec - this.lastSeenAt > CROWD.lostAfterSec) { this.lastLost = { cx: this.target.cx, cy: this.target.cy, w: this.target.w, at: tSec }; this.target = null; this.challenger = null; this.vx = 0; this.vy = 0; } else { // Coast: keep the predicted position as the steering point while briefly occluded if (tSec - this.lastSeenAt < CROWD.coastSec) { this.target = { ...this.target, cx: px, cy: py, coasting: true }; } } } if (!this.target && faces.length) { const f = faces.slice().sort((a, b) => b.w * b.h - a.w * a.h)[0]; this.forceTarget(f); isNew = true; this.lastSeenAt = tSec; this.vx = 0; this.vy = 0; // Re-acquired the person we just lost (or someone right where they were)? Don't nod again. if (this.lastLost && tSec - this.lastLost.at < CROWD.renodAfterSec && Math.hypot(f.cx - this.lastLost.cx, f.cy - this.lastLost.cy) < 4 * this.lastLost.w) this.nodded = true; if (tSec - this.lastNodAt < CROWD.renodAfterSec) this.nodded = true; } return { target: this.target, isNew }; } forceTarget(f) { if (this.target && Math.hypot(f.cx - this.target.cx, f.cy - this.target.cy) < this.target.w * CROWD.matchDistFaceWidths) { this.target = { ...f, id: this.target.id }; return; } this.target = { ...f, id: ++this.id }; this.nodded = false; this.challenger = null; } } // BlazeFace keypoints: [right eye, left eye, nose, mouth, right ear, left ear] (normalized). We just want the two eyes. function eyesOf(d) { const k = d.keypoints; if (!k || k.length < 2) return null; const a = k[0], b = k[1]; return a.x <= b.x ? { l: { x: a.x, y: a.y }, r: { x: b.x, y: b.y } } : { l: { x: b.x, y: b.y }, r: { x: a.x, y: a.y } }; } // Turns per-frame smile/frown scores into one-shot events with hold time and per-person cooldown class ExpressionWatcher { constructor() { this.smileSince = null; this.frownSince = null; this.lastAt = new Map(); } update(id, sc, tSec) { const last = this.lastAt.get(id) ?? -Infinity; if (sc.smile >= SMILE.smileOn) { this.smileSince ??= tSec; } else this.smileSince = null; if (sc.frown >= SMILE.frownOn) { this.frownSince ??= tSec; } else this.frownSince = null; if (tSec - last < SMILE.cooldownSec) return null; if (this.smileSince !== null && tSec - this.smileSince >= SMILE.holdSec) { this.lastAt.set(id, tSec); this.smileSince = null; return "smile"; } if (this.frownSince !== null && tSec - this.frownSince >= SMILE.frownHoldSec) { this.lastAt.set(id, tSec); this.frownSince = null; return "frown"; } return null; } } // Counts distinct people who have been the target for FRIENDS.engageSec; persists for the day class FriendsCounter { constructor() { this.key = "howdy-friends-" + new Date().toISOString().slice(0, 10); this.counted = new Set(); this.since = new Map(); let n = 0; if (FRIENDS.persist) { try { n = parseInt(localStorage.getItem(this.key) || "0", 10) || 0; } catch (_) {} } this.count = n; } note(id, tSec) { if (this.counted.has(id)) return false; if (!this.since.has(id)) { this.since.set(id, tSec); return false; } if (tSec - this.since.get(id) < FRIENDS.engageSec) return false; this.counted.add(id); this.since.delete(id); this.count++; if (FRIENDS.persist) { try { localStorage.setItem(this.key, String(this.count)); } catch (_) {} } return true; } } function nearestFace(faces, lm, aspect) { const hx = lm[0].x, hy = lm[0].y; // wrist let best = null, bestD = Infinity; for (const f of faces) { const d = Math.hypot((hx - f.cx) * aspect, hy - f.cy) / f.w; if (d < bestD) { best = f; bestD = d; } } return best; } function handInZone(lm, face, aspect) { const hx = lm[9].x, hy = lm[9].y; // middle-finger MCP ~ palm center const dx = Math.abs(hx - face.cx) * aspect / (face.w * aspect); const dy = (hy - face.cy) / face.h; return dx <= CROWD.handZoneX && dy >= -CROWD.handZoneUp && dy <= CROWD.handZoneDown; } // --------------------------------------------------------------------------- // Reachy's-eye-view overlay: glow on the person we're watching, thought bubble // --------------------------------------------------------------------------- class ThoughtOverlay { constructor(canvas) { this.canvas = canvas; this.t0 = performance.now(); this.lastMood = ""; this.moodSince = 0; this.text = ""; this.textSince = 0; this.lockT = 0; } pick(list, sec, period) { return Array.isArray(list) ? list[Math.floor(sec / period) % list.length] : list; } draw({ faces, target, hands, mood, now, busy, notice, say, listening, friends = 0, friendPop = 99, smile = 0 }) { const g = this.canvas.getContext("2d"); const W = this.canvas.width, H = this.canvas.height; g.clearRect(0, 0, W, H); if (notice) this.banner(g, notice, W, H); const sec = (now - this.t0) / 1000; if (mood !== this.lastMood) { this.lastMood = mood; this.moodSince = now; } this.lockT = target ? Math.min(1, this.lockT + 0.12) : Math.max(0, this.lockT - 0.12); const S = Math.min(W, H) / 480; // scale for 640x480-ish frames // Other faces: faint dotted circles so people can see it's choosing g.save(); for (const f of faces) { if (target && f.cx === target.cx && f.cy === target.cy) continue; g.setLineDash([6 * S, 6 * S]); g.lineWidth = 2 * S; g.strokeStyle = "rgba(255,255,255,0.45)"; g.beginPath(); g.ellipse(f.cx * W, f.cy * H, f.w * W * 0.6, f.h * H * 0.65, 0, 0, Math.PI * 2); g.stroke(); } g.restore(); // Hands: soft green dots + a wave meter arc that fills as a wave registers for (const hand of hands) { g.fillStyle = "rgba(43,182,115,0.85)"; for (const p of hand) { g.beginPath(); g.arc(p.x * W, p.y * H, 3.5 * S, 0, Math.PI * 2); g.fill(); } const m = hand.meter ?? 0, wx = hand[9].x * W, wy = hand[0].y * H + 22 * S; g.lineWidth = 5 * S; g.lineCap = "round"; g.strokeStyle = "rgba(255,255,255,0.25)"; g.beginPath(); g.arc(wx, wy, 14 * S, Math.PI * 0.75, Math.PI * 2.25); g.stroke(); if (m > 0) { g.strokeStyle = m >= 1 ? "#2bb673" : "#f2c94c"; g.beginPath(); g.arc(wx, wy, 14 * S, Math.PI * 0.75, Math.PI * (0.75 + 1.5 * m)); g.stroke(); } } // Target: warm glow that breathes, plus a happy ring let bubbleX = W / 2, bubbleY = H * 0.28, anchored = false; if (target && this.lockT > 0) { const cx = target.cx * W, cy = target.cy * H, r = Math.max(target.w * W, target.h * H) * 0.75; const breathe = 1 + 0.06 * Math.sin(sec * 3); const rr = r * breathe * (0.6 + 0.4 * this.lockT); const grad = g.createRadialGradient(cx, cy, rr * 0.6, cx, cy, rr * 1.6); const col = mood === "reacting" || mood === "waveSeen" ? "43,182,115" : "242,201,76"; grad.addColorStop(0, `rgba(${col},0)`); grad.addColorStop(0.5, `rgba(${col},${0.35 * this.lockT})`); grad.addColorStop(1, `rgba(${col},0)`); g.fillStyle = grad; g.beginPath(); g.arc(cx, cy, rr * 1.6, 0, Math.PI * 2); g.fill(); g.lineWidth = 4 * S; g.strokeStyle = `rgba(${col},${0.9 * this.lockT})`; g.beginPath(); g.arc(cx, cy, rr, 0, Math.PI * 2); g.stroke(); // little sparkles when reacting if (mood === "reacting" || mood === "noticed") { g.fillStyle = "#fff"; for (let i = 0; i < 6; i++) { const a = sec * 2 + i * Math.PI / 3, d = rr * (1.25 + 0.15 * Math.sin(sec * 5 + i)); g.beginPath(); g.arc(cx + Math.cos(a) * d, cy + Math.sin(a) * d, 3 * S, 0, Math.PI * 2); g.fill(); } } // Smile meter: a little mouth under the ring that curves up as the smile score rises if (smile > 0.05) { const mw = rr * 0.5, my = cy + rr + 14 * S, curve = (smile - 0.15) * 2 * mw * 0.6; g.strokeStyle = smile >= SMILE.smileOn ? "#2bb673" : "rgba(255,255,255,0.7)"; g.lineWidth = 4 * S; g.lineCap = "round"; g.beginPath(); g.moveTo(cx - mw, my); g.quadraticCurveTo(cx, my + curve, cx + mw, my); g.stroke(); } bubbleX = cx; bubbleY = cy - rr - 18 * S; anchored = true; } // Thought text by mood let text; switch (mood) { case "noticed": text = SAY.noticed; break; case "waveSeen": text = SAY.waveSeen; break; case "reacting": text = this.pick(SAY.reacting, sec, 0.8); break; case "tracking": text = this.pick(SAY.tracking, sec, SAY.trackingRotateSec); break; case "searching": text = SAY.searching; break; default: text = this.pick(SAY.idle, sec, SAY.idleRotateSec); } if (friendPop < 1.6) text = SAY.newFriend; if (say) text = say; if (text !== this.text) { this.text = text; this.textSince = now; } const pop = Math.min(1, (now - this.textSince) / 220); const scale = 0.85 + 0.15 * (1 - Math.pow(1 - pop, 3)); this.bubble(g, text, bubbleX, bubbleY, anchored, S * scale, W, H); // Friends badge, top right this.friendsBadge(g, friends, friendPop, S, W); // Mascot corner: antennae + face this.mascot(g, 26 * S, H - 26 * S, S, sec, mood); if (listening) { // small ear/mic dot pulsing next to the mascot g.fillStyle = `rgba(43,182,115,${0.5 + 0.4 * Math.sin(sec * 4)})`; g.beginPath(); g.arc(52 * S, H - 40 * S, 5 * S, 0, Math.PI * 2); g.fill(); } } friendsBadge(g, n, pop, S, W) { const scale = pop < 0.5 ? 1 + 0.18 * Math.sin(Math.PI * pop / 0.5) : 1; const label = `Friends today: ${n}`; g.save(); g.font = `700 ${Math.round(18 * S)}px "Comic Neue", "Segoe Print", "Chalkboard SE", system-ui, sans-serif`; const tw = g.measureText(label).width, bw = tw + 24 * S, bh = 32 * S, x = W - bw - 16 * S, y = 12 * S; // scale about the right edge so the pop never leaves the frame g.translate(x + bw, y + bh / 2); g.scale(scale, scale); g.translate(-(x + bw), -(y + bh / 2)); g.shadowColor = "rgba(0,0,0,0.25)"; g.shadowBlur = 8 * S; g.shadowOffsetY = 2 * S; g.fillStyle = pop < 1.6 ? "#2bb673" : "#fffdf5"; roundRect(g, x, y, bw, bh, 16 * S); g.fill(); g.shadowColor = "transparent"; g.strokeStyle = "#2b2b2b"; g.lineWidth = 2 * S; roundRect(g, x, y, bw, bh, 16 * S); g.stroke(); g.fillStyle = pop < 1.6 ? "#fff" : "#2b2b2b"; g.textAlign = "center"; g.textBaseline = "middle"; g.fillText(label, x + bw / 2, y + bh / 2 + 1 * S); g.restore(); } banner(g, text, W, H) { const S = Math.min(W, H) / 480; g.save(); g.font = `600 ${Math.round(16 * S)}px system-ui, sans-serif`; const words = text.split(" "), lines = []; let cur = ""; for (const w of words) { const t = cur ? cur + " " + w : w; if (g.measureText(t).width > W - 48 * S) { lines.push(cur); cur = w; } else cur = t; } if (cur) lines.push(cur); const lh = 22 * S, bh = lines.length * lh + 20 * S, by = H - bh - 62 * S; g.fillStyle = "rgba(180,40,40,0.92)"; roundRect(g, 12 * S, by, W - 24 * S, bh, 10 * S); g.fill(); g.fillStyle = "#fff"; g.textAlign = "center"; g.textBaseline = "middle"; lines.forEach((l, i) => g.fillText(l, W / 2, by + 10 * S + lh * (i + 0.5))); g.restore(); } bubble(g, text, x, y, anchored, S, W, H) { const font = `${Math.round(28 * S)}px "Comic Neue", "Segoe Print", "Chalkboard SE", "Marker Felt", system-ui, sans-serif`; g.font = `700 ${font}`; const padX = 16 * S, padY = 10 * S; const tw = g.measureText(text).width, bw = tw + padX * 2, bh = 34 * S + padY; let bx = clamp(x - bw / 2, 8, W - bw - 8), by = anchored ? y - bh : y; if (by < 8) by = 8; g.save(); g.shadowColor = "rgba(0,0,0,0.25)"; g.shadowBlur = 10 * S; g.shadowOffsetY = 3 * S; g.fillStyle = "#fffdf5"; roundRect(g, bx, by, bw, bh, 16 * S); g.fill(); g.restore(); g.strokeStyle = "#2b2b2b"; g.lineWidth = 2.5 * S; roundRect(g, bx, by, bw, bh, 16 * S); g.stroke(); if (anchored) { // thought-bubble dots down toward the face g.fillStyle = "#fffdf5"; for (const [k, r] of [[0.35, 6], [0.7, 4]]) { const cx = x, cy = by + bh + (18 * S) * k; g.beginPath(); g.arc(cx, cy, r * S, 0, Math.PI * 2); g.fill(); g.stroke(); } } g.fillStyle = "#2b2b2b"; g.textBaseline = "middle"; g.textAlign = "center"; g.fillText(text, bx + bw / 2, by + bh / 2 + 1 * S); } mascot(g, x, y, S, sec, mood) { const r = 16 * S, wag = (mood === "reacting" || mood === "noticed") ? Math.sin(sec * 14) * 0.5 : Math.sin(sec * 2) * 0.12; g.save(); g.translate(x, y); g.strokeStyle = "#2b2b2b"; g.lineWidth = 3 * S; g.lineCap = "round"; for (const side of [-1, 1]) { g.beginPath(); g.moveTo(side * r * 0.5, -r * 0.8); g.lineTo(side * r * 0.5 + Math.sin(wag) * side * 8 * S, -r * 1.9); g.stroke(); g.fillStyle = "#f2c94c"; g.beginPath(); g.arc(side * r * 0.5 + Math.sin(wag) * side * 8 * S, -r * 1.9, 3.5 * S, 0, Math.PI * 2); g.fill(); } g.fillStyle = "#fffdf5"; g.beginPath(); g.arc(0, 0, r, 0, Math.PI * 2); g.fill(); g.stroke(); g.fillStyle = "#2b2b2b"; const blink = (sec % 4) > 3.85 ? 0.15 : 1; for (const side of [-1, 1]) { g.beginPath(); g.ellipse(side * r * 0.4, -r * 0.15, 3 * S, 4 * S * blink, 0, 0, Math.PI * 2); g.fill(); } g.beginPath(); g.arc(0, r * 0.25, r * 0.45, 0.15 * Math.PI, 0.85 * Math.PI); g.stroke(); g.restore(); } } // --------------------------------------------------------------------------- // Big screen + voice // --------------------------------------------------------------------------- function toggleBigScreen(ui, on) { document.body.classList.toggle("big", on); ui.btnBig.textContent = on ? "Exit big screen" : "Big screen"; if (on) { // True fullscreen only works if every enclosing iframe allows it; otherwise the page-fill layout + F11 does the job const el = ui.stage; (el.requestFullscreen?.() ?? Promise.reject()).catch(() => log("Browser fullscreen blocked by the frame; press F11 for the browser's fullscreen")); } else if (document.fullscreenElement) { document.exitFullscreen?.().catch(() => {}); } } class VoiceTriggers { constructor(onTrigger) { this.onTrigger = onTrigger; this.active = false; this.rec = null; this.lastAt = -Infinity; this.wanted = false; this.recent = []; // [{t, text}] final results from the last few seconds, joined for matching this.Ctor = window.SpeechRecognition || window.webkitSpeechRecognition || null; } toggle(btn) { this.active ? this.stop(btn) : this.start(btn); } start(btn) { if (!this.Ctor) { log("Voice: this browser has no speech recognition (use Chrome or Edge)"); return; } this.wanted = true; const r = new this.Ctor(); r.lang = VOICE.lang; r.continuous = true; r.interimResults = true; r.maxAlternatives = 3; r.onresult = (e) => { for (let i = e.resultIndex; i < e.results.length; i++) { const res = e.results[i]; if (res.isFinal && VOICE.logTranscripts) log(`Heard: "${res[0].transcript.trim()}"`); for (let k = 0; k < res.length; k++) this.check(res[k].transcript, res.isFinal); } }; r.onaudiostart = () => log("Voice: mic open"); r.onspeechstart = () => console.debug("[voice] speech start"); navigator.permissions?.query?.({ name: "microphone" }).then((p) => log(`Mic permission: ${p.state}`)).catch(() => {}); r.onerror = (e) => { if (e.error !== "no-speech" && e.error !== "aborted") log(`Voice error: ${e.error}`); if (e.error === "not-allowed") this.wanted = false; }; r.onend = () => { if (this.wanted) { try { r.start(); } catch (_) { setTimeout(() => this.wanted && this.start(btn), 1000); } } else this.setActive(false, btn); }; try { r.start(); } catch (e) { log(`Voice start failed: ${e.message}`); return; } this.rec = r; this.setActive(true, btn); log("Voice: listening (laptop mic)"); } stop(btn) { this.wanted = false; try { this.rec?.stop(); } catch (_) {} this.setActive(false, btn); log("Voice: off"); } setActive(on, btn) { this.active = on; if (btn) { btn.textContent = on ? "Voice: listening" : "Voice: off"; btn.setAttribute("aria-pressed", String(on)); } } check(transcript, isFinal) { const now = performance.now(); const clean = transcript.toLowerCase().replace(/[^a-z' ]+/g, " ").replace(/\s+/g, " ").trim(); if (isFinal) { this.recent.push({ t: now, text: clean }); } this.recent = this.recent.filter((r) => now - r.t < 4000); // Match against recent finals + the current phrase, so "hello" ... "Richie" said with a pause still joins up const t = " " + [...this.recent.map((r) => r.text), isFinal ? "" : clean].join(" ").replace(/\s+/g, " ").trim() + " "; if (now - this.lastAt < VOICE.cooldownSec * 1000) return; for (const trig of VOICE.triggers) { const hit = trig.match ? trig.match.test(t.trim()) : trig.pattern ? trig.pattern.test(t) : (trig.phrases ?? []).some((p) => t.includes(" " + p + " ")); if (hit) { this.lastAt = now; this.recent = []; this.onTrigger(trig, transcript.trim()); return; } } } } function roundRect(g, x, y, w, h, r) { g.beginPath(); g.moveTo(x + r, y); g.arcTo(x + w, y, x + w, y + h, r); g.arcTo(x + w, y + h, x, y + h, r); g.arcTo(x, y + h, x, y, r); g.arcTo(x, y, x + w, y, r); g.closePath(); } // --------------------------------------------------------------------------- // Emotion clips: JSON motion + Opus audio -> 16 kHz mono PCM WAV // --------------------------------------------------------------------------- class ClipCache { constructor() { this.map = new Map(); } async preload(names) { const results = await Promise.allSettled(names.map((n) => this.get(n))); return results.filter((r) => r.status === "fulfilled" && r.value).length; } async get(name) { if (this.map.has(name)) return this.map.get(name); const p = (async () => { try { const [motionRes, audioRes] = await Promise.all([ fetch(EMOTIONS_BASE + name + ".json"), fetch(EMOTIONS_BASE + name + ".ogg"), ]); if (!motionRes.ok) throw new Error(`motion ${motionRes.status}`); const motion = normalizeMotion(await motionRes.json()); let wav; if (audioRes.ok) wav = await toWav16kMono(await audioRes.arrayBuffer()); return { motion, wav }; } catch (e) { log(`Failed to load ${name}: ${e.message}`); this.map.delete(name); return null; } })(); this.map.set(name, p); return p; } } function normalizeMotion(raw) { // Flatten nested 4x4 head matrices to 16 floats; leave everything else as-is const set_target_data = raw.set_target_data.map((f) => ({ ...f, head: f.head && Array.isArray(f.head[0]) ? f.head.flat() : f.head, })); return { ...raw, set_target_data }; } async function toWav16kMono(arrayBuffer) { const ctx = new (window.AudioContext || window.webkitAudioContext)(); const decoded = await ctx.decodeAudioData(arrayBuffer); await ctx.close(); const targetRate = 16000; const length = Math.ceil(decoded.duration * targetRate); const off = new OfflineAudioContext(1, length, targetRate); const src = off.createBufferSource(); src.buffer = decoded; src.connect(off.destination); src.start(); const rendered = await off.startRendering(); return encodeWav(rendered.getChannelData(0), targetRate); } function encodeWav(samples, sampleRate) { const buf = new ArrayBuffer(44 + samples.length * 2); const v = new DataView(buf); const str = (o, s) => { for (let i = 0; i < s.length; i++) v.setUint8(o + i, s.charCodeAt(i)); }; str(0, "RIFF"); v.setUint32(4, 36 + samples.length * 2, true); str(8, "WAVE"); str(12, "fmt "); v.setUint32(16, 16, true); v.setUint16(20, 1, true); v.setUint16(22, 1, true); v.setUint32(24, sampleRate, true); v.setUint32(28, sampleRate * 2, true); v.setUint16(32, 2, true); v.setUint16(34, 16, true); str(36, "data"); v.setUint32(40, samples.length * 2, true); let o = 44; for (let i = 0; i < samples.length; i++, o += 2) { const s = Math.max(-1, Math.min(1, samples[i])); v.setInt16(o, s < 0 ? s * 0x8000 : s * 0x7fff, true); } return new Blob([buf], { type: "audio/wav" }); } // A small family of synthesized chirps; one is picked at random each time const CHIRPS = [ { dur: 0.45, parts: [[0.0, 0.18, 900, 1600], [0.22, 0.45, 1100, 2000]] }, // classic double { dur: 0.6, parts: [[0.0, 0.14, 800, 1500], [0.18, 0.32, 1000, 1800], [0.38, 0.6, 1200, 2200]] }, // triple, climbing { dur: 0.5, parts: [[0.0, 0.5, 700, 2400]] }, // long swoop up { dur: 0.55, parts: [[0.0, 0.25, 1500, 900], [0.3, 0.55, 900, 1900]] }, // dip then rise (question) { dur: 0.7, parts: [[0.0, 0.1, 1400, 1700], [0.12, 0.22, 1400, 1700], [0.24, 0.34, 1400, 1700], [0.4, 0.7, 1200, 2300]] }, // trill + flourish { dur: 0.5, parts: [[0.0, 0.2, 2000, 1300], [0.25, 0.5, 1300, 2600]] }, // whistle ]; function chirpWav(which) { const sr = 16000; const c = CHIRPS[which ?? Math.floor(Math.random() * CHIRPS.length)]; const n = Math.floor(sr * c.dur); const out = new Float32Array(n); for (const [t0, t1, f0, f1] of c.parts) { for (let i = Math.floor(t0 * sr); i < Math.floor(t1 * sr) && i < n; i++) { const t = i / sr - t0, T = t1 - t0, u = t / T; const env = Math.sin(Math.PI * u); out[i] += 0.6 * env * Math.sin(2 * Math.PI * (f0 * t + 0.5 * (f1 - f0) * t * t / T)); } } return encodeWav(out, sr); } // --------------------------------------------------------------------------- // Motion helpers // --------------------------------------------------------------------------- // Short nod from the current pose, antennas perk, 0.9 s at 50 Hz function buildNodMotion({ yawDeg, pitchDeg, bodyYawDeg }) { // Keep the nod inside a comfortable envelope so the daemon's IK always converges if (![yawDeg, pitchDeg, bodyYawDeg].every(Number.isFinite)) { yawDeg = 0; pitchDeg = TRACK.restPitchDeg; bodyYawDeg = 0; } bodyYawDeg = clamp(bodyYawDeg, -TRACK.maxBodyYawDeg, TRACK.maxBodyYawDeg); yawDeg = clamp(yawDeg, bodyYawDeg - 40, bodyYawDeg + 40); pitchDeg = clamp(pitchDeg, -30, 15); const hz = 50, dur = 0.9, n = Math.round(dur * hz); const time = [], set_target_data = []; for (let i = 0; i <= n; i++) { const t = i / hz, u = t / dur; const dip = u < 0.625 ? 14 * Math.sin(Math.PI * u * 1.6) : 0; // one dip in the first ~0.56 s const perk = 25 * DEG * Math.sin(Math.PI * u); time.push(t); set_target_data.push({ head: headMatrix(0, (pitchDeg + dip) * DEG, yawDeg * DEG), antennas: [INIT_ANTENNAS[0] - perk, INIT_ANTENNAS[1] + perk], body_yaw: bodyYawDeg * DEG, }); } return { time, set_target_data }; } // Pre-rendered spoken lines (already 16 kHz mono PCM WAV, so no conversion needed) class SpeechCache { constructor() { this.map = new Map(); } async preloadAll() { const files = [...new Set(Object.values(SPEECH.lines).flat())]; const r = await Promise.allSettled(files.map((f) => this.load(f))); return r.filter((x) => x.status === "fulfilled" && x.value).length; } async load(file) { if (this.map.has(file)) return this.map.get(file); const p = (async () => { try { const res = await fetch(SPEECH.base + file + "?v=" + SPEECH.version); if (!res.ok) throw new Error(`${res.status}`); const buf = await res.arrayBuffer(); return { blob: new Blob([buf], { type: "audio/wav" }), durSec: wavDuration(buf) }; } catch (e) { log(`Voice line ${file} failed: ${e.message}`); this.map.delete(file); return null; } })(); this.map.set(file, p); return p; } async get(key) { const list = SPEECH.lines[key]; if (!list?.length) return null; return this.load(list[Math.floor(Math.random() * list.length)]); } } // Walk the RIFF chunks to find the real data length (ffmpeg and others add LIST chunks before it) function wavDuration(buf) { const v = new DataView(buf); let rate = 16000, channels = 1, bits = 16, pos = 12; while (pos + 8 <= buf.byteLength) { const id = String.fromCharCode(v.getUint8(pos), v.getUint8(pos + 1), v.getUint8(pos + 2), v.getUint8(pos + 3)); const size = v.getUint32(pos + 4, true); if (id === "fmt ") { channels = v.getUint16(pos + 10, true); rate = v.getUint32(pos + 12, true); bits = v.getUint16(pos + 22, true); } if (id === "data") return size / (rate * channels * (bits / 8)); pos += 8 + size + (size & 1); } return 1.0; } // "Talking": hold the head, small quick nods and antenna flicks for the length of the line function buildTalkMotion({ yawDeg, pitchDeg, bodyYawDeg, durSec = 1.2 }) { if (![yawDeg, pitchDeg, bodyYawDeg].every(Number.isFinite)) { yawDeg = 0; pitchDeg = TRACK.restPitchDeg; bodyYawDeg = 0; } bodyYawDeg = clamp(bodyYawDeg, -TRACK.maxBodyYawDeg, TRACK.maxBodyYawDeg); yawDeg = clamp(yawDeg, bodyYawDeg - 40, bodyYawDeg + 40); pitchDeg = clamp(pitchDeg, -30, 15); const hz = 50, n = Math.round(durSec * hz), time = [], set_target_data = []; for (let i = 0; i <= n; i++) { const t = i / hz, u = t / durSec, env = Math.sin(Math.PI * Math.min(1, u)); const bob = 3 * env * Math.sin(2 * Math.PI * 4.5 * t); // little syllable nods const perk = 0.35 * env * (0.6 + 0.4 * Math.sin(2 * Math.PI * 3 * t)); time.push(t); set_target_data.push({ head: headMatrix(0, (pitchDeg + bob) * DEG, yawDeg * DEG), antennas: [INIT_ANTENNAS[0] - perk, INIT_ANTENNAS[1] + perk], body_yaw: bodyYawDeg * DEG, }); } return { time, set_target_data }; } // Antenna wave-back: hold the current head pose, swing one or both antennae, ~1.6 s at 50 Hz function buildAntennaWave({ yawDeg, pitchDeg, bodyYawDeg, both = false, side = 1 }) { if (![yawDeg, pitchDeg, bodyYawDeg].every(Number.isFinite)) { yawDeg = 0; pitchDeg = TRACK.restPitchDeg; bodyYawDeg = 0; } const hz = 50, dur = 1.6, n = Math.round(dur * hz), swings = 3; const amp = 0.9; // radians of antenna swing const time = [], set_target_data = []; const head = headMatrix(0, pitchDeg * DEG, yawDeg * DEG); for (let i = 0; i <= n; i++) { const t = i / hz, u = t / dur; const env = Math.sin(Math.PI * u); // ease in/out const w = amp * env * Math.sin(2 * Math.PI * swings * u); // the wave itself const a = [INIT_ANTENNAS[0], INIT_ANTENNAS[1]]; if (both) { a[0] = INIT_ANTENNAS[0] - w; a[1] = INIT_ANTENNAS[1] + w; } // opposite phase, like two arms else if (side === 0) a[0] = INIT_ANTENNAS[0] - w; else a[1] = INIT_ANTENNAS[1] + w; time.push(t); set_target_data.push({ head, antennas: a, body_yaw: bodyYawDeg * DEG }); } return { time, set_target_data }; } // Tilt a whole recorded move up/down (rotate about world y; negative = nose up), keeping head pitch in range function offsetMotionPitch(motion, pitchRad) { if (Math.abs(pitchRad) < 1e-4) return motion; const Ry = headMatrix(0, pitchRad, 0); return { ...motion, set_target_data: motion.set_target_data.map((f) => { if (!f.head) return f; const h = mat4mul(Ry, f.head); const rp = matrixToRpyDeg(h); if (rp.pitch < -TRACK.maxPitchUpDeg || rp.pitch > TRACK.maxPitchDownDeg) return f; // leave extreme frames untilted return { ...f, head: h }; }), }; } // Cut a move short (with a quick fade back toward the first frame so it doesn't stop mid-pose) function truncateMotion(motion, maxSec) { const idx = motion.time.findIndex((t) => t > maxSec); if (idx < 0) return motion; const time = motion.time.slice(0, idx), frames = motion.set_target_data.slice(0, idx); const first = frames[0], last = frames[frames.length - 1], t0 = time[time.length - 1], hz = 50, n = 25; for (let i = 1; i <= n; i++) { const u = i / n, t = t0 + i / hz; const head = last.head && first.head ? last.head.map((v, k) => v + (first.head[k] - v) * u) : last.head; const antennas = last.antennas && first.antennas ? last.antennas.map((v, k) => v + (first.antennas[k] - v) * u) : last.antennas; const body_yaw = typeof last.body_yaw === "number" ? last.body_yaw + ((first.body_yaw ?? last.body_yaw) - last.body_yaw) * u : last.body_yaw; time.push(t); frames.push({ ...last, head, antennas, body_yaw }); } return { ...motion, time, set_target_data: frames }; } // Rotate a whole recorded move about world z by `yawRad` so it plays relative to the current body yaw function offsetMotionYaw(motion, yawRad) { if (Math.abs(yawRad) < 1e-4) return motion; const Rz = headMatrix(0, 0, yawRad); const maxBody = TRACK.maxBodyYawDeg * DEG; return { ...motion, set_target_data: motion.set_target_data.map((f) => ({ ...f, head: f.head ? mat4mul(Rz, f.head) : f.head, body_yaw: typeof f.body_yaw === "number" ? clamp(f.body_yaw + yawRad, -maxBody, maxBody) : f.body_yaw, })), }; } // Row-major flat 4x4 from roll/pitch/yaw (radians), ZYX convention, zero translation function headMatrix(r, p, y) { const cr = Math.cos(r), sr = Math.sin(r), cp = Math.cos(p), sp = Math.sin(p), cy = Math.cos(y), sy = Math.sin(y); return [ cy * cp, cy * sp * sr - sy * cr, cy * sp * cr + sy * sr, 0, sy * cp, sy * sp * sr + cy * cr, sy * sp * cr - cy * sr, 0, -sp, cp * sr, cp * cr, 0, 0, 0, 0, 1, ]; } function toNested4x4(m) { if (Array.isArray(m[0])) return m; return [m.slice(0, 4), m.slice(4, 8), m.slice(8, 12), m.slice(12, 16)]; } function matrixToRpyDeg(m) { if (Array.isArray(m[0])) m = m.flat(); const pitch = Math.asin(clamp(-m[8], -1, 1)); const roll = Math.atan2(m[9], m[10]); const yaw = Math.atan2(m[4], m[0]); return { roll: roll / DEG, pitch: pitch / DEG, yaw: yaw / DEG }; } function mat4mul(a, b) { const o = new Array(16); for (let r = 0; r < 4; r++) for (let c = 0; c < 4; c++) { let s = 0; for (let k = 0; k < 4; k++) s += a[r * 4 + k] * b[k * 4 + c]; o[r * 4 + c] = s; } return o; } function safeReturnHome(reachy) { try { reachy.setMotorMode?.("enabled"); } catch (_) {} const target = { head: headMatrix(0, TRACK.restPitchDeg * DEG, 0), antennas: INIT_ANTENNAS, body_yaw: 0 }; try { if (typeof reachy.gotoTarget === "function") reachy.gotoTarget({ ...target, duration: 1.2 }); else reachy.setTarget(target); } catch (_) {} } // --------------------------------------------------------------------------- // UI // --------------------------------------------------------------------------- function bindUi() { const $ = (id) => document.getElementById(id); return { video: $("video"), overlay: $("overlay"), btnTrack: $("btnTrack"), btnBody: $("btnBody"), btnMute: $("btnMute"), btnWave: $("btnWave"), btnWake: $("btnWake"), btnBig: $("btnBig"), btnVoice: $("btnVoice"), stage: document.querySelector(".stage"), }; } function setStatus(text, sub = "", mode = "") { const el = document.getElementById("status"); document.getElementById("statusText").textContent = text; document.getElementById("statusSub").textContent = sub; el.className = "status" + (mode ? " " + mode : ""); } function log(msg) { const el = document.getElementById("log"); if (!el) return; const t = new Date().toLocaleTimeString([], { hour12: false }); el.textContent = `${t} ${msg}\n` + el.textContent.split("\n").slice(0, 14).join("\n"); console.log("[howdy-friend]", msg); } function syncOverlaySize(canvas, video) { if (canvas.width !== video.videoWidth || canvas.height !== video.videoHeight) { canvas.width = video.videoWidth; canvas.height = video.videoHeight; } } const clamp = (v, lo, hi) => Math.max(lo, Math.min(hi, v)); const lerp = (a, b, t) => a + (b - a) * t;