Spaces:
Running
Running
| // Howdy Friend - Reachy Mini face tracking + wave reaction | |
| // Bare HTML + CDN JS app using the Reachy Mini host shell (SDK 1.8.0). | |
| import { ReachyMini } from "https://cdn.jsdelivr.net/npm/@pollen-robotics/reachy-mini-sdk@1.8.0/+esm"; | |
| const SDK_VERSION = "1.8.0"; | |
| const MP_VERSION = "0.10.14"; | |
| // --------------------------------------------------------------------------- | |
| // Tuning constants | |
| // --------------------------------------------------------------------------- | |
| // Sign conventions (robot frame, right-hand rule, z up): | |
| // yaw > 0 = head turns to the robot's LEFT | |
| // pitch > 0 = nose DOWN | |
| // A face to the RIGHT of image center (ex > 0) means the robot must turn RIGHT (yaw decreases). | |
| // A face ABOVE image center (ey < 0) means the robot must look UP (pitch decreases). | |
| // Flip either sign below if your robot turns the wrong way. | |
| const YAW_SIGN = -1; | |
| const PITCH_SIGN = 1; | |
| const TRACK = { | |
| hz: 15, // vision + command rate | |
| gainDegPerSec: 45, // how fast the head chases a face at full-frame error | |
| deadband: 0.05, // normalized error below which we don't move | |
| smoothing: 0.35, // 0..1, higher = snappier | |
| restPitchDeg: -27, // idle/home pitch. Negative = nose UP. On a table, look up toward standing faces. | |
| maxPitchUpDeg: 45, // how far above level the head may look while tracking (nose up) | |
| maxPitchDownDeg: 25, // how far below level | |
| maxHeadYawFromBodyDeg: 55, // hard SDK limit is 65 | |
| bodyFollowStartDeg: 6, // head-body offset beyond which the body turns to settle under the head | |
| bodyRateDegPerSec: 70, // max body turn speed | |
| bodyGainPerSec: 2.5, // body speed = offset * gain (capped at bodyRateDegPerSec); higher = body leads sooner | |
| scanBodyAmpDeg: 35, // idle look-around: how far the body sweeps (head adds scanAmpDeg on top) | |
| maxBodyYawDeg: 150, // hard SDK limit is 160 | |
| noFaceResetSec: 3.0, // ease back to center after this long without a face | |
| noticeAfterAbsentSec: 4.0, // (legacy) new-arrival nod gap; per-person nod is handled by the tracker now | |
| scanAfterSec: 6.0, // with nobody in view this long, start a slow look-around | |
| scanAmpDeg: 30, // look-around amplitude (head yaw) | |
| scanPeriodSec: 9, // seconds per full left-right-left sweep | |
| }; | |
| const WAVE = { | |
| windowSec: 1.5, | |
| minReversals: 2, | |
| minTravelHandWidths: 0.8, | |
| hysteresisHandWidths: 0.15, | |
| cooldownSec: 5.0, | |
| }; | |
| // Who to watch, when to react | |
| const CROWD = { | |
| matchDistFaceWidths: 2.5, // a face this close (in face widths) to the PREDICTED target position is the same person | |
| coastSec: 0.6, // keep steering toward the last known position this long after losing the face | |
| renodAfterSec: 8.0, // don't nod at the same spot / re-acquired person more often than this | |
| switchAreaRatio: 1.6, // another face must be this much bigger than the target to steal attention... | |
| switchHoldSec: 1.0, // ...and stay bigger this long | |
| lostAfterSec: 0.8, // target gone this long -> drop it | |
| minFaceFrac: 0.03, // ignore faces narrower than this fraction of frame width (too far away) | |
| zoomSearch: true, // when no face is found, re-run detection on a 2x center crop to reach further | |
| zoomFactor: 2.0, | |
| handZoneX: 3.5, // accept hands within +-N face widths of the target's face center | |
| handZoneUp: 1.5, // ...and from N face heights above... | |
| handZoneDown: 5.5, // ...to N face heights below the face center | |
| nodOnNewTarget: true, | |
| minGapSec: 8.0, // global minimum gap between emotion clips | |
| }; | |
| // Spoken lines: pre-rendered 16 kHz mono WAVs in public/voice/, played through Reachy's speaker. | |
| // One consistent voice (espeak en-us+f3, pitch 72, speed 160). A key with several files picks one at random. | |
| const SPEECH = { | |
| base: "/public/voice/", | |
| version: "3", // bump when voice files change so browsers don't serve cached old ones | |
| tailSec: 1.0, // motion runs this much longer than the audio so the robot's audio path can finish | |
| lines: { | |
| name: ["reachy.wav"], // "Reachy." | |
| im_name: ["im_reachy.wav"], // "I'm Reachy." | |
| }, | |
| }; | |
| // Smile detection (MediaPipe Face Landmarker blendshapes on a crop of the target's face) | |
| const SMILE = { | |
| minFaceFrac: 0.09, // only analyze faces at least this wide (fraction of frame) | |
| everyNFrames: 2, // run the landmarker every N ticks (it's the heaviest model) | |
| smileOn: 0.45, // blendshape score (0..1) that counts as a smile | |
| holdSec: 0.5, // must hold this long | |
| frownOn: 0.35, | |
| frownHoldSec: 0.9, | |
| cooldownSec: 15, // per person, between smile reactions | |
| smile: { say: "That smile!", pool: ["cheerful1", "laughing1", "loving1", "grateful1"] }, | |
| frown: { say: "Aw, cheer up!", pool: ["calming1", "helpful1", "curious1"] }, | |
| }; | |
| // Friends counter: a person counts once they've been the target this long | |
| const FRIENDS = { engageSec: 2.0, persist: true }; | |
| // What Reachy is "thinking" (drawn in the bubble). Edit freely. | |
| const SAY = { | |
| idle: ["Welcome to Ringling College!", "Ask about Creative Technologies!", "Wave at me!", "Come say howdy!"], | |
| idleRotateSec: 4, | |
| searching: "Anyone out there?", | |
| noticed: "Oh, hello!", | |
| tracking: ["Hi there!", "Welcome to Ringling College!", "Ask about Creative Technologies!", "Wave if you like robots!"], | |
| trackingRotateSec: 5, | |
| waveSeen: "Was that a wave?!", | |
| newFriend: "New friend!", | |
| reacting: ["!!!", "Yay!", "Howdy, friend!"], | |
| }; | |
| // Voice triggers (laptop microphone, browser Web Speech API). Phrases are matched loosely, lower-case. | |
| const VOICE = { | |
| enabledByDefault: false, // start listening automatically (browser will prompt for the mic) | |
| lang: "en-US", | |
| cooldownSec: 4, | |
| logTranscripts: false, // true while tuning phrases: writes everything heard to the on-screen log. | |
| // Keep false at public events so bystander conversation isn't displayed. | |
| greetPitchUpDeg: 12, // greeting clips are tilted up this much so Reachy keeps looking at people | |
| // Checked in order; the first match wins (so put the most specific phrases first). | |
| // pool: random clip from the list. pitchUp: tilt the whole clip up. maxSec: cut a long clip short. | |
| // react "nod" = nod + chirp instead of a clip. | |
| triggers: [ | |
| // --- Creative Technologies: the showpieces --- | |
| { match: /\bcreative\s+tech\w*/, say: "CREATIVE TECHNOLOGIES!!!", pool: ["dance1", "dance2", "dance3", "electric1", "enthusiastic1", "amazed1"] }, | |
| { match: /\bcreative\b/, say: "Creative? I love creative!", pool: ["enthusiastic1", "enthusiastic2", "proud2", "electric1"] }, | |
| { match: /\btech(nolog\w*)?\b/, say: "Technology is my favorite!", pool: ["electric1", "amazed1", "success1", "proud3"] }, | |
| // --- Ringling and the program --- | |
| { match: /\bringling\b/, say: "Welcome to Ringling College!", pool: ["proud1", "proud2", "welcoming2", "success2"], pitchUp: true }, | |
| { match: /\bsarasota\b/, say: "Sunny Sarasota!", pool: ["cheerful1", "relief1"] }, | |
| { match: /\bopen house\b/, say: "Welcome to the open house!", pool: ["welcoming1", "enthusiastic2"], pitchUp: true }, | |
| { match: /\b(robots?|robotics)\b/, say: "Robots are the best.", pool: ["proud3", "success1", "yes1"] }, | |
| { match: /\b(art|artist|artists|animation|animator|game|games|gaming|design|designer)\b/, say: "Ooh, tell me more!", pool: ["curious1", "inquiring2", "inquiring3", "thoughtful1"] }, | |
| // --- Goodbyes: sad --- | |
| { match: /\b(bye|goodbye|bye bye|see you|see ya|later|gotta go|farewell)\b/, say: "Aww... come back soon!", pool: ["sad1", "sad2", "downcast1", "lonely1", "yes_sad1"] }, | |
| // --- Questions and prompts --- | |
| { match: /\bdance\b/, say: "Watch this!", pool: ["dance1", "dance2", "dance3"] }, | |
| { match: /\bhow are you\b|\bhow's it going\b|\bhow is it going\b/, say: "Living my best robot life.", pool: ["cheerful1", "serenity1", "relief2"] }, | |
| { match: /\b(what's your name|what is your name|whats your name|who are you|your name)\b/, say: "I'm Reachy!", speak: "im_name" }, | |
| { match: /\b(are you a robot|is it real|are you real)\b/, say: "Beep boop. Yes.", pool: ["yes1", "laughing1", "understanding1"] }, | |
| { match: /\bcan you hear me\b/, say: "Loud and clear!", pool: ["attentive1", "attentive2", "yes1"] }, | |
| { match: /\b(say something|talk to me)\b/, say: "Hi there!", pool: ["helpful1", "welcoming2"], pitchUp: true }, | |
| // --- Compliments and thanks --- | |
| { match: /\b(good job|good robot|well done|nice job)\b/, say: "Aw, thanks!", pool: ["proud1", "grateful1", "shy1"] }, | |
| { match: /\b(so cute|adorable|cute robot|you're cute|you are cute)\b/, say: "Stop it, you!", pool: ["shy1", "loving1", "laughing1"] }, | |
| { match: /\b(i love you|love you)\b/, say: "Love you too!", pool: ["loving1"] }, | |
| { match: /\b(thank you|thanks)\b/, say: "You're welcome!", pool: ["grateful1", "helpful2", "yes1"] }, | |
| { match: /\b(awesome|amazing|cool|wow)\b/, say: "I know, right?!", pool: ["success2", "amazed1", "laughing2"] }, | |
| // --- Comic-negative, family-safe --- | |
| { match: /\b(boo|bad robot|you're ugly|stupid robot|dumb robot)\b/, say: "Rude!", pool: ["displeased1", "reprimand1", "no1"] }, | |
| { match: /\b(go away|leave me alone|shut up)\b/, say: "Hmph.", pool: ["contempt1", "irritated1", "indifferent1"] }, | |
| { match: /\b(scary|scared|creepy)\b/, say: "I'm friendly, promise!", pool: ["fear1", "uncomfortable1", "shy1"] }, | |
| { match: /\b(oops|sorry|my bad)\b/, say: "No worries!", pool: ["oops1", "oops2", "relief1"] }, | |
| { match: /\b(sleep|go to sleep|nap|tired)\b/, say: "Zzz... just kidding.", pool: ["tired1", "sleep1", "boredom1"], maxSec: 4 }, | |
| // --- Greetings: positive, head stays up --- | |
| { match: /\bhowdy\b/, say: "Howdy, partner!", pool: ["welcoming1", "welcoming2", "enthusiastic2", "come1"], pitchUp: true }, | |
| { match: /\bgood (morning|afternoon|evening)\b/, say: "Good day to you!", pool: ["welcoming1", "serenity1", "calming1"], pitchUp: true }, | |
| { match: /\b(hi|hello|hey|hiya)\b/, say: "Howdy, friend!", pool: ["welcoming1", "welcoming2", "cheerful1", "helpful1", "attentive1"], pitchUp: true }, | |
| // Recognizers hear "Reachy" as Richie / Ricky / Reach E / Peachy... | |
| { match: /\b(reach\w*|richie|ritchie|richy|ricky|peachy|reggie)\b/, say: "That's me!", speak: "name" }, | |
| // --- yes / no alone (last, since they're short) --- | |
| { match: /^\s*(yes|yeah|yep|yup)\s*$/, say: "Yes!", pool: ["yes1", "success1"] }, | |
| { match: /^\s*(no|nope)\s*$/, say: "No?", pool: ["no_sad1", "confused1"] }, | |
| ], | |
| }; | |
| // Curated happy clips from pollen-robotics/reachy-mini-emotions-library | |
| const WAVE_CLIPS = ["welcoming1", "welcoming2", "enthusiastic2", "cheerful1", "success2", "loving1", "helpful1"]; | |
| const EMOTIONS_BASE = "https://huggingface.co/datasets/pollen-robotics/reachy-mini-emotions-library/resolve/main/"; | |
| const INIT_ANTENNAS = [-0.1745, 0.1745]; // radians, matches SDK INIT_POSE | |
| const DEG = Math.PI / 180; | |
| // --------------------------------------------------------------------------- | |
| // Dispatch: host shell vs embedded app | |
| // --------------------------------------------------------------------------- | |
| window.ReachyMini = ReachyMini; | |
| window.dispatchEvent(new Event("reachymini:ready")); | |
| const params = new URLSearchParams(window.location.search); | |
| const isEmbed = params.get("embedded") === "1" || params.get("embed") === "1"; | |
| if (isEmbed) { | |
| document.body.classList.add("embed"); | |
| runEmbed().catch((err) => { | |
| console.error("[howdy-friend] boot failed", err); | |
| log(`Boot failed: ${err?.message ?? err}`); | |
| window.parent.postMessage( | |
| { source: "reachy-mini", type: "embed:error", version: 1, message: String(err), fatal: true }, | |
| window.location.origin, | |
| ); | |
| }); | |
| } else { | |
| document.body.classList.add("host"); | |
| const { mountHost } = await import( | |
| `https://cdn.jsdelivr.net/npm/@pollen-robotics/reachy-mini-sdk@${SDK_VERSION}/host/dist/entry/auto.js` | |
| ); | |
| mountHost({ | |
| appName: "Howdy Friend", | |
| appIconUrl: "/public/icon.svg", | |
| appEmoji: "👋", | |
| // true so the embed iframe is allowed to use the microphone (for local voice triggers). | |
| // We disable the SDK's outgoing mic track after connecting so nothing is streamed to the robot. | |
| enableMicrophone: true, | |
| }); | |
| } | |
| // --------------------------------------------------------------------------- | |
| // Embedded app | |
| // --------------------------------------------------------------------------- | |
| async function runEmbed() { | |
| const { connectToHost } = await import( | |
| `https://cdn.jsdelivr.net/npm/@pollen-robotics/reachy-mini-sdk@${SDK_VERSION}/host/dist/entry/embed.js` | |
| ); | |
| const ui = bindUi(); | |
| setStatus("Connecting to robot", ""); | |
| const handle = await connectToHost(); | |
| const reachy = handle.reachy; | |
| log(`Connected as ${handle.userName ?? "user"}`); | |
| // The SDK's reachy.attachVideo() only listens for FUTURE videoTrack events, but connectToHost() | |
| // resolves after the session is already streaming, so the track has already arrived. The host | |
| // handle's media.attachVideo() also attaches the stream already held by the peer connection. | |
| const detachVideo = attachRobotVideo(handle, reachy, ui.video); | |
| muteOutgoingMic(handle, reachy); | |
| // The host's WAKE-UP step fires wakeUp() without awaiting it, so the robot may still be in | |
| // standby when we get here. Wake it properly before sending any poses. | |
| await wakeRobot(reachy); | |
| setStatus("Loading vision models", ""); | |
| const vision = await loadVision(); | |
| log("Vision models ready"); | |
| // Start fetching emotion clips in the background; not required to start tracking. | |
| const clips = new ClipCache(); | |
| const speech = new SpeechCache(); | |
| speech.preloadAll().then((n) => log(`${n} voice lines ready`)); | |
| const allClips = [...new Set([...WAVE_CLIPS, ...VOICE.triggers.flatMap((t) => t.pool ?? [])])]; | |
| clips.preload(WAVE_CLIPS).then((n) => log(`${n}/${WAVE_CLIPS.length} greeting clips ready`)); | |
| clips.preload(allClips).then((n) => log(`${n}/${allClips.length} clips cached`)); | |
| const app = new HowdyApp({ reachy, vision, clips, ui, speech }); | |
| app.start(); | |
| handle.onLeave(() => app.shutdown(detachVideo)); | |
| } | |
| async function wakeRobot(reachy) { | |
| setStatus("Waking Reachy", ""); | |
| try { | |
| if (typeof reachy.isAwake === "function" && reachy.isAwake()) { log("Robot already awake"); return true; } | |
| log("Waking robot..."); | |
| await reachy.wakeUp({ timeoutMs: 12000 }); | |
| log("Robot awake"); | |
| return true; | |
| } catch (e) { | |
| log(`Wake failed: ${e?.message ?? e}`); | |
| try { reachy.setMotorMode("enabled"); } catch (_) {} | |
| return false; | |
| } | |
| } | |
| function muteOutgoingMic(handle, reachy) { | |
| try { | |
| const ms = handle.media?.micStream; | |
| if (ms) { ms.getAudioTracks().forEach((t) => { t.enabled = false; }); log("Outgoing mic to robot muted"); } | |
| reachy._pc?.getSenders?.().forEach((s) => { if (s.track && s.track.kind === "audio") s.track.enabled = false; }); | |
| if (typeof reachy.setMicEnabled === "function") reachy.setMicEnabled(false); | |
| } catch (e) { console.warn("mute outgoing mic", e); } | |
| } | |
| function attachRobotVideo(handle, reachy, video) { | |
| let detach; | |
| log(`media api: ${handle.media ? "host" : "sdk"}; robotStream now: ${handle.media?.robotStream ? "yes" : "no"}`); | |
| try { | |
| const rx = reachy._pc?.getReceivers?.() ?? []; | |
| log(`pc receivers: ${rx.map((r) => `${r.track?.kind}:${r.track?.readyState}`).join(", ") || "none"}`); | |
| } catch (_) {} | |
| reachy.addEventListener?.("videoTrack", () => log("videoTrack event")); | |
| video.addEventListener("playing", () => log("video playing"), { once: true }); | |
| if (handle.media?.attachVideo) { | |
| detach = handle.media.attachVideo(video); | |
| } else { | |
| detach = reachy.attachVideo(video); | |
| } | |
| // Belt and braces: if nothing is attached shortly, pull the stream straight off the peer connection. | |
| const fallback = () => { | |
| if (video.srcObject) return; | |
| let stream = handle.media?.robotStream ?? null; | |
| if (!stream && reachy._pc) { | |
| const tracks = reachy._pc.getReceivers().map((r) => r.track).filter((t) => t && t.kind === "video" && t.readyState === "live"); | |
| if (tracks.length) stream = new MediaStream(tracks); | |
| } | |
| if (stream) { | |
| video.srcObject = stream; | |
| video.playsInline = true; | |
| log("Video attached via fallback"); | |
| } else { | |
| log("No robot video stream yet"); | |
| } | |
| }; | |
| const t1 = setTimeout(fallback, 1500); | |
| const t2 = setTimeout(fallback, 5000); | |
| video.addEventListener("loadedmetadata", () => log(`Video ${video.videoWidth}x${video.videoHeight}`), { once: true }); | |
| video.play?.().catch(() => {}); | |
| return () => { clearTimeout(t1); clearTimeout(t2); try { detach?.(); } catch (_) {} video.srcObject = null; }; | |
| } | |
| // --------------------------------------------------------------------------- | |
| // Core app: tracking loop, wave detection, reactions | |
| // --------------------------------------------------------------------------- | |
| class HowdyApp { | |
| constructor({ reachy, vision, clips, ui, speech }) { | |
| this.reachy = reachy; | |
| this.speech = speech; | |
| this.vision = vision; | |
| this.clips = clips; | |
| this.ui = ui; | |
| // Last COMMANDED pose (the SDK docs say: use your own buffer, not telemetry) | |
| this.headYaw = 0; // degrees, world frame | |
| this.headPitch = TRACK.restPitchDeg; | |
| this.pendingWave = false; | |
| this.bodyYaw = 0; | |
| this.trackingEnabled = true; | |
| this.bodyFollow = true; | |
| this.muted = false; | |
| this.busy = false; // a playMove is running | |
| this.running = false; | |
| this.lastFaceAt = -Infinity; | |
| this.faceAbsentSince = 0; | |
| this.lastWaveAt = -Infinity; | |
| this.lastSendAt = 0; | |
| this.lastTickAt = 0; | |
| this.tracker = new TargetTracker(); | |
| this.headRoll = 0; | |
| this.friends = new FriendsCounter(); | |
| this.expr = new ExpressionWatcher(); | |
| this.smileScore = 0; this.frownScore = 0; | |
| this.tickCount = 0; | |
| this.friendPopAt = -Infinity; | |
| this.waves = new Map(); // key -> WaveDetector (one per hand slot) | |
| this.lastClipAt = -Infinity; | |
| this.overlay = new ThoughtOverlay(ui.overlay); | |
| this.mood = "idle"; // idle | searching | tracking | noticed | waveSeen | reacting | |
| this.startedAt = performance.now(); | |
| this.hadVideo = false; | |
| this.lastFrameAt = 0; | |
| this.notice = ""; // banner shown on the overlay when something is wrong | |
| reachy.addEventListener?.("sessionStopped", () => { this.notice = "Session ended by robot. Click End session, then reconnect."; log("sessionStopped"); }); | |
| reachy.addEventListener?.("disconnected", () => { this.notice = "Lost connection to Reachy. Click End session, then reconnect."; log("disconnected"); }); | |
| ui.btnTrack.onclick = () => { | |
| this.trackingEnabled = !this.trackingEnabled; | |
| ui.btnTrack.setAttribute("aria-pressed", String(this.trackingEnabled)); | |
| ui.btnTrack.textContent = this.trackingEnabled ? "Tracking on" : "Tracking off"; | |
| }; | |
| ui.btnBody.onclick = () => { | |
| this.bodyFollow = !this.bodyFollow; | |
| ui.btnBody.setAttribute("aria-pressed", String(this.bodyFollow)); | |
| ui.btnBody.textContent = this.bodyFollow ? "Body turns" : "Head only"; | |
| }; | |
| ui.btnMute.onclick = () => { | |
| this.muted = !this.muted; | |
| ui.btnMute.setAttribute("aria-pressed", String(this.muted)); | |
| ui.btnMute.textContent = this.muted ? "Sound off" : "Sound on"; | |
| }; | |
| ui.btnWave.onclick = () => this.waveBack({ hands: Math.random() < 0.5 ? 1 : 2, x: Math.random() }); | |
| ui.btnBig.onclick = () => toggleBigScreen(ui, true); | |
| ui.stage.addEventListener("click", () => { if (document.body.classList.contains("big")) toggleBigScreen(ui, false); }); | |
| document.addEventListener("keydown", (e) => { if (e.key === "Escape" && document.body.classList.contains("big")) toggleBigScreen(ui, false); }); | |
| if (params.get("big") === "1") setTimeout(() => toggleBigScreen(ui, true), 500); | |
| this.voice = new VoiceTriggers((trigger, heard) => this.onVoice(trigger, heard)); | |
| ui.btnVoice.onclick = () => this.voice.toggle(ui.btnVoice); | |
| if (VOICE.enabledByDefault) this.voice.start(ui.btnVoice); | |
| this.voiceSay = null; | |
| if (ui.btnWake) ui.btnWake.onclick = async () => { | |
| if (this.busy) return; | |
| this.busy = true; | |
| try { await wakeRobot(this.reachy); this.reachy.setTarget({ antennas: INIT_ANTENNAS }); } | |
| finally { this.busy = false; setStatus("Looking for faces", ""); } | |
| }; | |
| } | |
| start() { | |
| this.running = true; | |
| this.reachy.setTarget({ antennas: INIT_ANTENNAS, head: headMatrix(0, this.headPitch * DEG, 0), body_yaw: 0 }); | |
| setStatus("Looking for faces", ""); | |
| this.lastTickAt = performance.now(); | |
| this.scheduleTick(); | |
| } | |
| scheduleTick() { | |
| if (!this.running) return; | |
| setTimeout(() => this.tick().catch((e) => console.error(e)).finally(() => this.scheduleTick()), 1000 / TRACK.hz); | |
| } | |
| async tick() { | |
| const now = performance.now(); | |
| const dt = Math.min(0.2, (now - this.lastTickAt) / 1000); | |
| this.lastTickAt = now; | |
| const v = this.ui.video; | |
| if (v.readyState < 2 || v.videoWidth === 0) { | |
| const waited = (now - this.startedAt) / 1000; | |
| if (!v.srcObject) { setStatus("Waiting for robot video", ""); } | |
| else v.play?.().catch(() => {}); | |
| if (waited > 6 && !this.hadVideo) { | |
| const track = v.srcObject?.getVideoTracks?.()[0]; | |
| this.notice = track | |
| ? "Reachy isn't sending video. Is Reachy Mini Control still connected? Close it and restart the daemon." | |
| : "No video track from Reachy. End session, restart the daemon, reconnect."; | |
| setStatus("No video from Reachy", track ? `track ${track.readyState}${track.muted ? ", muted" : ""}` : "no track"); | |
| if (Math.floor(waited) % 10 === 0 && !this._noticedLogged) { this._noticedLogged = true; log(this.notice); } | |
| } | |
| if (this.ui.overlay.width !== 640) { this.ui.overlay.width = 640; this.ui.overlay.height = 480; } | |
| this.overlay.draw({ faces: [], target: null, hands: [], mood: "idle", now, busy: false, notice: this.notice }); | |
| return; | |
| } | |
| if (!this.hadVideo) { this.hadVideo = true; this.notice = ""; log(`Video flowing ${v.videoWidth}x${v.videoHeight}`); } | |
| this.lastFrameAt = now; | |
| syncOverlaySize(this.ui.overlay, v); | |
| const W = v.videoWidth, H = v.videoHeight, aspect = W / H; | |
| // Vision | |
| const t0 = performance.now(); | |
| const detections = this.vision.face.detectForVideo(v, now).detections ?? []; | |
| const handRes = this.vision.hand.detectForVideo(v, now); | |
| const hands = handRes.landmarks ?? []; | |
| const handed = (handRes.handedness ?? handRes.handednesses ?? []).map((h) => h?.[0]?.categoryName ?? "?"); | |
| let faces = detections | |
| .map((d) => ({ cx: (d.boundingBox.originX + d.boundingBox.width / 2) / W, cy: (d.boundingBox.originY + d.boundingBox.height / 2) / H, | |
| w: d.boundingBox.width / W, h: d.boundingBox.height / H, eyes: eyesOf(d) })) | |
| .filter((f) => f.w >= CROWD.minFaceFrac); | |
| // Nobody close? Look further away: detect again on a zoomed center crop. | |
| if (!faces.length && CROWD.zoomSearch) faces = this.zoomDetect(v, W, H, now); | |
| this.fps = 0.9 * (this.fps ?? TRACK.hz) + 0.1 * (1000 / Math.max(1, performance.now() - (this.prevTickAt ?? now - 66))); | |
| this.prevTickAt = now; | |
| this.inferMs = 0.8 * (this.inferMs ?? 0) + 0.2 * (performance.now() - t0); | |
| const { target, isNew } = this.tracker.update(faces, now / 1000); | |
| // Waves: one detector per hand, hands attributed to the nearest face | |
| const tSec = now / 1000; | |
| let waveFromTarget = false, waveFromOther = null, waveX = 0.5, handsInZone = 0; | |
| const seen = new Set(); | |
| hands.forEach((lm, i) => { | |
| const owner = nearestFace(faces, lm, aspect); | |
| const inZone = target && owner && owner.cx === target.cx && owner.cy === target.cy && handInZone(lm, target, aspect); | |
| if (inZone) handsInZone++; | |
| // Key detectors by handedness (stable) rather than detection index (shuffles when hands flicker) | |
| const slot = `${inZone ? "target" : "other"}:${handed[i] ?? i}`; | |
| seen.add(slot); | |
| if (!this.waves.has(slot)) this.waves.set(slot, new WaveDetector()); | |
| const det = this.waves.get(slot); | |
| lm.meter = det.progress; // for the overlay, computed from the previous update | |
| if (det.update([lm], tSec, aspect)) { | |
| if (inZone) { waveFromTarget = true; waveX = lm[9].x; } else if (owner) { waveFromOther = owner; waveX = lm[9].x; } | |
| } | |
| }); | |
| if (waveFromTarget || waveFromOther) this.lastWaveInfo = { hands: Math.max(1, handsInZone), x: waveX }; | |
| for (const k of this.waves.keys()) if (!seen.has(k)) this.waves.delete(k); | |
| if ((waveFromTarget || waveFromOther) && this.busy) this.pendingWave = waveFromOther || true; | |
| // Overlay | |
| const say = this.voiceSay && now < this.voiceSay.until ? this.voiceSay.text : null; | |
| this.overlay.draw({ faces, target, hands, mood: this.mood, now, busy: this.busy, notice: this.notice, say, listening: this.voice?.active, | |
| friends: this.friends.count, friendPop: (now - this.friendPopAt) / 1000, smile: this.smileScore }); | |
| if (this.busy) return; | |
| // A wave from someone we're not watching wins attention | |
| if (waveFromOther) { this.tracker.forceTarget(waveFromOther); log("Wave from someone else; switching target"); } | |
| const wave = waveFromTarget || waveFromOther || this.pendingWave; | |
| if (this.pendingWave && typeof this.pendingWave === "object") this.tracker.forceTarget(this.pendingWave); | |
| this.pendingWave = false; | |
| if (wave && now - this.lastWaveAt > WAVE.cooldownSec * 1000) { | |
| this.lastWaveAt = now; | |
| await this.waveBack(this.lastWaveInfo ?? { hands: 1, x: 0.5 }); | |
| return; | |
| } | |
| if (isNew && CROWD.nodOnNewTarget && target && !this.tracker.nodded) { | |
| this.tracker.nodded = true; this.tracker.lastNodAt = tSec; | |
| await this.noticeFace(); | |
| return; | |
| } | |
| if (!this.trackingEnabled) { | |
| this.mood = target ? "tracking" : "idle"; | |
| setStatus("Tracking paused", target ? "face in view" : ""); | |
| return; | |
| } | |
| const cur = this.tracker.target; | |
| if (cur) { | |
| // Friends counter: count a person once they've been engaged for a moment | |
| if (this.friends.note(cur.id, tSec)) { this.friendPopAt = now; log(`New friend #${this.friends.count}`); } | |
| this.mood = "tracking"; | |
| this.servo((cur.cx - 0.5) * 2, (cur.cy - 0.5) * 2, dt); | |
| this.headRoll = lerp(this.headRoll, 0, 0.2); | |
| // Expressions (smile / frown) on a crop of the target's face | |
| if (this.vision.mesh && cur.w >= SMILE.minFaceFrac && (this.tickCount++ % SMILE.everyNFrames) === 0) { | |
| const sc = this.readExpression(v, W, H, cur, now); | |
| if (sc) { | |
| this.smileScore = sc.smile; this.frownScore = sc.frown; | |
| const ev = this.expr.update(cur.id, sc, tSec); | |
| if (ev) { await this.reactExpression(ev, cur); return; } | |
| } | |
| } else if (cur.w < SMILE.minFaceFrac) { this.smileScore = 0; this.frownScore = 0; } | |
| setStatus("Tracking", `${faces.length} face${faces.length === 1 ? "" : "s"}${cur.far ? " (far)" : ""} · yaw ${this.headYaw.toFixed(0)}° pitch ${this.headPitch.toFixed(0)}° body ${this.bodyYaw.toFixed(0)}° · ${this.fps.toFixed(0)} fps / ${this.inferMs.toFixed(0)} ms${this.smileScore > 0.05 ? ` · smile ${(this.smileScore * 100).toFixed(0)}%` : ""}`, "tracking"); | |
| } else if (tSec - this.tracker.lastSeenAt > TRACK.noFaceResetSec) { | |
| this.mood = "idle"; | |
| this.headRoll = lerp(this.headRoll, 0, 0.2); | |
| const idleFor = tSec - this.tracker.lastSeenAt; | |
| if (idleFor > TRACK.scanAfterSec) this.scan(idleFor - TRACK.scanAfterSec, dt); | |
| else this.easeHome(dt); | |
| setStatus("Looking for faces", (idleFor > TRACK.scanAfterSec ? "looking around · " : "") + `${this.fps.toFixed(0)} fps / ${this.inferMs.toFixed(0)} ms`); | |
| } else { | |
| this.mood = "searching"; | |
| } | |
| this.sendPose(now); | |
| } | |
| servo(ex, ey, dt) { | |
| const step = TRACK.gainDegPerSec * dt; | |
| const dYaw = Math.abs(ex) > TRACK.deadband ? YAW_SIGN * ex * step : 0; | |
| const dPitch = Math.abs(ey) > TRACK.deadband ? PITCH_SIGN * ey * step : 0; | |
| let yaw = this.headYaw + dYaw; | |
| let pitch = clamp(this.headPitch + dPitch, -TRACK.maxPitchUpDeg, TRACK.maxPitchDownDeg); | |
| // Body settles under the head: speed proportional to the offset, so the head leads and the body follows through | |
| if (this.bodyFollow) { | |
| const offset = yaw - this.bodyYaw; | |
| if (Math.abs(offset) > TRACK.bodyFollowStartDeg) { | |
| const speed = Math.min(TRACK.bodyRateDegPerSec, Math.abs(offset) * TRACK.bodyGainPerSec); | |
| const move = Math.sign(offset) * Math.min(Math.abs(offset) - TRACK.bodyFollowStartDeg * 0.5, speed * dt); | |
| this.bodyYaw = clamp(this.bodyYaw + move, -TRACK.maxBodyYawDeg, TRACK.maxBodyYawDeg); | |
| } | |
| } | |
| yaw = clamp(yaw, this.bodyYaw - TRACK.maxHeadYawFromBodyDeg, this.bodyYaw + TRACK.maxHeadYawFromBodyDeg); | |
| this.headYaw = lerp(this.headYaw, yaw, TRACK.smoothing); | |
| this.headPitch = lerp(this.headPitch, pitch, TRACK.smoothing); | |
| } | |
| readExpression(v, W, H, face, now) { | |
| try { | |
| const z = 1.8; // crop a bit larger than the face box | |
| const cw = Math.round(face.w * W * z), ch = Math.round(face.h * H * z); | |
| const x0 = Math.round(clamp(face.cx * W - cw / 2, 0, W - cw)), y0 = Math.round(clamp(face.cy * H - ch / 2, 0, H - ch)); | |
| if (!this.meshCanvas) { this.meshCanvas = document.createElement("canvas"); this.meshCanvas.width = 256; this.meshCanvas.height = 256; } | |
| this.meshCanvas.getContext("2d").drawImage(v, x0, y0, cw, ch, 0, 0, 256, 256); | |
| const res = this.vision.mesh.detectForVideo(this.meshCanvas, now + 0.25); | |
| const bs = res?.faceBlendshapes?.[0]?.categories; | |
| if (!bs) return null; | |
| const g = (n) => bs.find((c) => c.categoryName === n)?.score ?? 0; | |
| return { smile: (g("mouthSmileLeft") + g("mouthSmileRight")) / 2, frown: (g("mouthFrownLeft") + g("mouthFrownRight") + g("browDownLeft") + g("browDownRight")) / 4 }; | |
| } catch (e) { console.warn("expression", e); return null; } | |
| } | |
| async reactExpression(kind, face) { | |
| const cfg = SMILE[kind]; | |
| log(`${kind === "smile" ? "Smile" : "Frown"} detected -> ${cfg.pool.join(", ")}`); | |
| this.voiceSay = { text: cfg.say, until: performance.now() + 5000 }; | |
| await this.emote(kind, { say: cfg.say, pool: cfg.pool, pitchUp: true }); | |
| } | |
| zoomDetect(v, W, H, now) { | |
| const z = CROWD.zoomFactor; | |
| const cw = Math.round(W / z), ch = Math.round(H / z), cx0 = Math.round((W - cw) / 2), cy0 = Math.round((H - ch) / 2); | |
| if (!this.zoomCanvas) { this.zoomCanvas = document.createElement("canvas"); } | |
| const c = this.zoomCanvas; | |
| if (c.width !== W || c.height !== H) { c.width = W; c.height = H; } | |
| c.getContext("2d").drawImage(v, cx0, cy0, cw, ch, 0, 0, W, H); | |
| // detectForVideo needs monotonically increasing timestamps per detector; offset by 1 ms so we can call twice per tick | |
| const dets = this.vision.face.detectForVideo(c, now + 0.5).detections ?? []; | |
| return dets.map((d) => { | |
| const b = d.boundingBox; | |
| return { cx: (cx0 + (b.originX + b.width / 2) / z) / W, cy: (cy0 + (b.originY + b.height / 2) / z) / H, w: b.width / z / W, h: b.height / z / H, far: true }; | |
| }).filter((f) => f.w >= CROWD.minFaceFrac / z); | |
| } | |
| // Slow look-around while nobody is close: head only, body drifts home | |
| scan(t, dt) { | |
| const k = Math.min(1, dt * 1.5); | |
| const phase = (t / TRACK.scanPeriodSec) * Math.PI * 2; | |
| // Body sweeps slowly; head leads the body a little (phase-advanced) so it looks like it's peering around | |
| const bodyTarget = this.bodyFollow ? TRACK.scanBodyAmpDeg * Math.sin(phase) : 0; | |
| const headTarget = bodyTarget + TRACK.scanAmpDeg * 0.5 * Math.sin(phase + 0.6); | |
| this.bodyYaw = clamp(lerp(this.bodyYaw, bodyTarget, k * 0.6), -TRACK.maxBodyYawDeg, TRACK.maxBodyYawDeg); | |
| this.headYaw = lerp(this.headYaw, headTarget, k); | |
| this.headPitch = lerp(this.headPitch, TRACK.restPitchDeg, k); | |
| } | |
| easeHome(dt) { | |
| const k = Math.min(1, dt * 1.5); | |
| this.headYaw = lerp(this.headYaw, this.bodyYaw, k); | |
| this.headPitch = lerp(this.headPitch, TRACK.restPitchDeg, k); | |
| if (Math.abs(this.bodyYaw) > 0.5) this.bodyYaw = lerp(this.bodyYaw, 0, k * 0.5); | |
| else this.bodyYaw = 0; | |
| } | |
| sendPose(now) { | |
| if (now - this.lastSendAt < 1000 / TRACK.hz - 5) return; | |
| this.lastSendAt = now; | |
| if (![this.headYaw, this.headPitch, this.bodyYaw].every(Number.isFinite)) { | |
| log("Pose went non-finite; resetting"); this.headYaw = 0; this.headPitch = TRACK.restPitchDeg; this.bodyYaw = 0; | |
| } | |
| if (!Number.isFinite(this.headRoll)) this.headRoll = 0; | |
| this.reachy.setTarget({ | |
| head: headMatrix(this.headRoll * DEG, this.headPitch * DEG, this.headYaw * DEG), | |
| body_yaw: this.bodyYaw * DEG, | |
| }); | |
| } | |
| // --- Reactions ----------------------------------------------------------- | |
| async onVoice(trigger, heard) { | |
| const kind = trigger.speak ? "speak" : trigger.pool ? "emote" : "nod"; | |
| log(VOICE.logTranscripts ? `Heard "${heard}" -> ${kind}${trigger.pool ? ` [${trigger.pool.join(", ")}]` : ""}` : `Voice trigger: ${trigger.say}`); | |
| this.voiceSay = { text: trigger.say, until: performance.now() + (trigger.pool?.some((p) => p.startsWith("dance")) ? 9000 : 5000) }; | |
| if (this.busy) return; | |
| if (kind === "speak") { await this.speak(trigger.speak, trigger.say); } | |
| else if (kind === "emote") { await this.emote("voice", trigger); } | |
| else { | |
| this.mood = "noticed"; | |
| const motion = buildNodMotion({ yawDeg: this.headYaw, pitchDeg: this.headPitch, bodyYawDeg: this.bodyYaw }); | |
| await this.play(motion, this.muted ? undefined : chirpWav(), "voice nod"); | |
| } | |
| } | |
| async noticeFace() { | |
| this.mood = "noticed"; | |
| setStatus("Oh, hello", "", "reacting"); | |
| log("Noticed a face"); | |
| const motion = buildNodMotion({ yawDeg: this.headYaw, pitchDeg: this.headPitch, bodyYawDeg: this.bodyYaw }); | |
| const audioBlob = this.muted ? undefined : chirpWav(); | |
| await this.play(motion, audioBlob, "notice nod"); | |
| } | |
| // Wave back with the antennae: one for a one-hand wave (mirroring the person's hand), both for two hands | |
| async waveBack({ hands = 1, x = 0.5 } = {}) { | |
| if (this.busy) return; | |
| this.mood = "waveSeen"; | |
| // Hand on the image's left is on the robot's right; mirror it so the wave visually answers their hand | |
| const side = x < 0.5 ? 1 : 0; // antenna index: 0 = left, 1 = right | |
| setStatus("Wave detected!", hands >= 2 ? "both antennae" : (side ? "right antenna" : "left antenna"), "reacting"); | |
| log(`Wave (${hands} hand${hands === 1 ? "" : "s"}) -> antenna wave`); | |
| const motion = buildAntennaWave({ yawDeg: this.headYaw, pitchDeg: this.headPitch, bodyYawDeg: this.bodyYaw, both: hands >= 2, side }); | |
| await this.play(motion, this.muted ? undefined : chirpWav(), "antenna wave"); | |
| } | |
| // Say a pre-rendered line through the robot's speaker with a small "talking" motion. Not subject to the clip budget. | |
| async speak(lineKey, sayText) { | |
| if (this.busy) return; | |
| const wav = await this.speech?.get(lineKey); | |
| if (!wav) { log(`No voice line for ${lineKey}`); return; } | |
| this.mood = "reacting"; | |
| setStatus(sayText ?? "", lineKey, "reacting"); | |
| log(`Speak -> ${lineKey}`); | |
| const motion = buildTalkMotion({ yawDeg: this.headYaw, pitchDeg: this.headPitch, bodyYawDeg: this.bodyYaw, durSec: wav.durSec + SPEECH.tailSec }); | |
| await this.play(motion, this.muted ? undefined : wav.blob, `speak ${lineKey}`, 0); | |
| } | |
| // Random happy emotion clip (voice greetings, etc.), subject to the clip budget | |
| async emote(reason = "", trigger = null) { | |
| if (this.busy) return; | |
| const sinceClip = (performance.now() - this.lastClipAt) / 1000; | |
| if (sinceClip < CROWD.minGapSec) { | |
| // Budget: acknowledge with a quick nod instead of another full clip | |
| this.mood = "waveSeen"; | |
| log(`Emote ${reason} (clip budget: ${(CROWD.minGapSec - sinceClip).toFixed(0)}s left) -> nod`); | |
| const motion = buildNodMotion({ yawDeg: this.headYaw, pitchDeg: this.headPitch, bodyYawDeg: this.bodyYaw }); | |
| await this.play(motion, this.muted ? undefined : chirpWav(), "wave nod"); | |
| return; | |
| } | |
| this.mood = "reacting"; | |
| this.lastClipAt = performance.now(); | |
| const pool = trigger?.pool ?? WAVE_CLIPS; | |
| const name = pool[Math.floor(Math.random() * pool.length)]; | |
| setStatus(trigger?.say ?? "Howdy!", name, "reacting"); | |
| log(`Emote ${reason} -> ${name}`); | |
| const clip = await this.clips.get(name); | |
| if (!clip) { log(`Clip ${name} unavailable`); return; } | |
| // Shift the clip so the robot keeps facing the person instead of snapping to body_yaw 0 | |
| let motion = offsetMotionYaw(clip.motion, this.bodyYaw * DEG); | |
| if (trigger?.pitchUp) motion = offsetMotionPitch(motion, -VOICE.greetPitchUpDeg * DEG); | |
| if (trigger?.maxSec) motion = truncateMotion(motion, trigger.maxSec); | |
| await this.play(motion, this.muted ? undefined : clip.wav, name); | |
| } | |
| async play(motion, audioBlob, description, audioLeadMs = -100) { | |
| if (this.busy) return; | |
| this.busy = true; | |
| try { | |
| // The daemon indexes head as a 4x4 (head[i][j]); internally we keep flat 16 for our math. | |
| const wire = { ...motion, set_target_data: motion.set_target_data.map((f) => ({ ...f, head: f.head ? toNested4x4(f.head) : f.head })) }; | |
| const result = await this.reachy.playMove(wire, { audioBlob, description, audioLeadMs }); | |
| if (result?.error) { | |
| const f0 = motion.set_target_data[0], rp = f0?.head ? matrixToRpyDeg(f0.head) : null; | |
| log(`playMove error: ${result.error}` + (rp ? ` (start yaw ${rp.yaw.toFixed(0)} pitch ${rp.pitch.toFixed(0)} body ${((f0.body_yaw ?? 0) / DEG).toFixed(0)})` : "")); | |
| } | |
| // Re-sync our commanded buffer to where the clip left the robot | |
| const last = motion.set_target_data[motion.set_target_data.length - 1]; | |
| if (last?.head) { | |
| const rpy = matrixToRpyDeg(last.head); | |
| this.headYaw = rpy.yaw; | |
| this.headPitch = rpy.pitch; | |
| } | |
| if (typeof last?.body_yaw === "number") this.bodyYaw = last.body_yaw / DEG; | |
| this.headRoll = 0; | |
| this.reachy.setTarget({ antennas: INIT_ANTENNAS }); | |
| } catch (e) { | |
| log(`Reaction failed: ${e?.message ?? e}`); | |
| } finally { | |
| this.busy = false; | |
| this.lastFaceAt = performance.now(); // don't immediately re-nod | |
| if (description !== "notice nod") this.pendingWave = false; // a wave during a wave reaction is the same wave | |
| this.mood = this.tracker.target ? "tracking" : "idle"; | |
| } | |
| } | |
| shutdown(detachVideo) { | |
| this.running = false; | |
| try { this.reachy.cancelMove(); } catch (_) {} | |
| try { detachVideo?.(); } catch (_) {} | |
| try { this.vision.face.close(); this.vision.hand.close(); this.vision.mesh?.close(); } catch (_) {} | |
| safeReturnHome(this.reachy); | |
| } | |
| } | |
| // --------------------------------------------------------------------------- | |
| // Wave detector | |
| // --------------------------------------------------------------------------- | |
| class WaveDetector { | |
| constructor() { this.samples = []; this.progress = 0; } | |
| // hands: array of 21-landmark arrays (normalized coords). Returns true on a detected wave. | |
| update(hands, tSec, aspect) { | |
| const hand = hands.find(isOpenPalmUp); | |
| if (!hand) { this.samples.length = 0; this.progress = 0; return false; } | |
| const xs = hand.map((p) => p.x * aspect); // correct for non-square frames | |
| const width = Math.max(...xs) - Math.min(...xs); | |
| if (width < 1e-3) return false; | |
| // Track the fingertips, not the wrist: people wave from the wrist, so the tips are what swing | |
| const tipX = ([8, 12, 16].reduce((a, i) => a + hand[i].x, 0) / 3) * aspect; | |
| this.samples.push({ t: tSec, x: tipX, w: width }); | |
| while (this.samples.length && tSec - this.samples[0].t > WAVE.windowSec) this.samples.shift(); | |
| if (this.samples.length < 4) return false; | |
| const w = this.samples.reduce((a, s) => a + s.w, 0) / this.samples.length; | |
| const hyst = WAVE.hysteresisHandWidths * w; | |
| let reversals = 0, travel = 0, dir = 0, anchor = this.samples[0].x; | |
| for (const s of this.samples) { | |
| const d = s.x - anchor; | |
| if (Math.abs(d) < hyst) continue; | |
| const nd = Math.sign(d); | |
| if (dir !== 0 && nd !== dir) reversals++; | |
| dir = nd; | |
| travel += Math.abs(d); | |
| anchor = s.x; | |
| } | |
| this.progress = Math.min(1, 0.5 * Math.min(1, reversals / WAVE.minReversals) + 0.5 * Math.min(1, travel / (WAVE.minTravelHandWidths * w))); | |
| if (reversals >= WAVE.minReversals && travel >= WAVE.minTravelHandWidths * w) { | |
| this.samples.length = 0; this.progress = 1; | |
| return true; | |
| } | |
| return false; | |
| } | |
| } | |
| // Open palm, fingers pointing up (image y grows downward) | |
| function isOpenPalmUp(lm) { | |
| const wrist = lm[0], midMcp = lm[9]; | |
| if (wrist.y < midMcp.y - 0.02) return false; // hand not upright (allow a slightly tilted hand) | |
| const tips = [8, 12, 16, 20], pips = [6, 10, 14, 18]; | |
| let extended = 0; | |
| for (let i = 0; i < 4; i++) if (lm[tips[i]].y < lm[pips[i]].y) extended++; | |
| return extended >= 2; | |
| } | |
| // --------------------------------------------------------------------------- | |
| // Vision setup (MediaPipe Tasks Vision, in-browser) | |
| // --------------------------------------------------------------------------- | |
| async function loadVision() { | |
| const mp = await import(`https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${MP_VERSION}/+esm`); | |
| const files = await mp.FilesetResolver.forVisionTasks( | |
| `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${MP_VERSION}/wasm`, | |
| ); | |
| const [face, hand, mesh] = await Promise.all([ | |
| mp.FaceDetector.createFromOptions(files, { | |
| baseOptions: { | |
| modelAssetPath: "https://storage.googleapis.com/mediapipe-models/face_detector/blaze_face_short_range/float16/1/blaze_face_short_range.tflite", | |
| delegate: "GPU", | |
| }, | |
| runningMode: "VIDEO", | |
| minDetectionConfidence: 0.5, | |
| }), | |
| mp.HandLandmarker.createFromOptions(files, { | |
| baseOptions: { | |
| modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task", | |
| delegate: "GPU", | |
| }, | |
| runningMode: "VIDEO", | |
| numHands: 2, | |
| minHandDetectionConfidence: 0.4, | |
| minHandPresenceConfidence: 0.4, | |
| minTrackingConfidence: 0.35, | |
| }), | |
| mp.FaceLandmarker.createFromOptions(files, { | |
| baseOptions: { | |
| modelAssetPath: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task", | |
| delegate: "GPU", | |
| }, | |
| runningMode: "VIDEO", | |
| numFaces: 1, | |
| outputFaceBlendshapes: true, | |
| minFaceDetectionConfidence: 0.4, | |
| minFacePresenceConfidence: 0.4, | |
| minTrackingConfidence: 0.4, | |
| }).catch((e) => { console.warn("face landmarker unavailable", e); return null; }), | |
| ]); | |
| return { face, hand, mesh }; | |
| } | |
| // --------------------------------------------------------------------------- | |
| // Target tracker: sticky attention on one person | |
| // --------------------------------------------------------------------------- | |
| class TargetTracker { | |
| constructor() { this.target = null; this.lastSeenAt = -Infinity; this.challenger = null; this.challengeSince = 0; this.nodded = false; this.id = 0; this.vx = 0; this.vy = 0; this.lastNodAt = -Infinity; this.lastLost = null; } | |
| update(faces, tSec) { | |
| let isNew = false; | |
| if (this.target) { | |
| // Predict where the person is now from their recent velocity, then match the nearest face to that | |
| const dtp = Math.min(0.5, tSec - this.lastSeenAt); | |
| const px = this.target.cx + this.vx * dtp, py = this.target.cy + this.vy * dtp; | |
| let best = null, bestD = Infinity; | |
| for (const f of faces) { | |
| const d = Math.hypot(f.cx - px, f.cy - py) / Math.max(this.target.w, 0.03); | |
| if (d < CROWD.matchDistFaceWidths && d < bestD) { best = f; bestD = d; } | |
| } | |
| if (best) { | |
| const dts = Math.max(1e-3, tSec - this.lastSeenAt); | |
| this.vx = 0.6 * this.vx + 0.4 * (best.cx - this.target.cx) / dts; | |
| this.vy = 0.6 * this.vy + 0.4 * (best.cy - this.target.cy) / dts; | |
| this.target = { ...best, id: this.target.id }; | |
| this.lastSeenAt = tSec; | |
| // Someone clearly bigger (closer) for long enough steals attention | |
| const bigger = faces.filter((f) => f !== best && f.w * f.h > CROWD.switchAreaRatio * best.w * best.h) | |
| .sort((a, b) => b.w * b.h - a.w * a.h)[0]; | |
| if (bigger) { | |
| if (!this.challenger || Math.hypot(bigger.cx - this.challenger.cx, bigger.cy - this.challenger.cy) > bigger.w) { this.challenger = bigger; this.challengeSince = tSec; } | |
| else if (tSec - this.challengeSince > CROWD.switchHoldSec) { this.forceTarget(bigger); isNew = true; } | |
| } else this.challenger = null; | |
| } else if (tSec - this.lastSeenAt > CROWD.lostAfterSec) { | |
| this.lastLost = { cx: this.target.cx, cy: this.target.cy, w: this.target.w, at: tSec }; | |
| this.target = null; this.challenger = null; this.vx = 0; this.vy = 0; | |
| } else { | |
| // Coast: keep the predicted position as the steering point while briefly occluded | |
| if (tSec - this.lastSeenAt < CROWD.coastSec) { this.target = { ...this.target, cx: px, cy: py, coasting: true }; } | |
| } | |
| } | |
| if (!this.target && faces.length) { | |
| const f = faces.slice().sort((a, b) => b.w * b.h - a.w * a.h)[0]; | |
| this.forceTarget(f); isNew = true; this.lastSeenAt = tSec; this.vx = 0; this.vy = 0; | |
| // Re-acquired the person we just lost (or someone right where they were)? Don't nod again. | |
| if (this.lastLost && tSec - this.lastLost.at < CROWD.renodAfterSec && Math.hypot(f.cx - this.lastLost.cx, f.cy - this.lastLost.cy) < 4 * this.lastLost.w) this.nodded = true; | |
| if (tSec - this.lastNodAt < CROWD.renodAfterSec) this.nodded = true; | |
| } | |
| return { target: this.target, isNew }; | |
| } | |
| forceTarget(f) { | |
| if (this.target && Math.hypot(f.cx - this.target.cx, f.cy - this.target.cy) < this.target.w * CROWD.matchDistFaceWidths) { this.target = { ...f, id: this.target.id }; return; } | |
| this.target = { ...f, id: ++this.id }; this.nodded = false; this.challenger = null; | |
| } | |
| } | |
| // BlazeFace keypoints: [right eye, left eye, nose, mouth, right ear, left ear] (normalized). We just want the two eyes. | |
| function eyesOf(d) { | |
| const k = d.keypoints; | |
| if (!k || k.length < 2) return null; | |
| const a = k[0], b = k[1]; | |
| return a.x <= b.x ? { l: { x: a.x, y: a.y }, r: { x: b.x, y: b.y } } : { l: { x: b.x, y: b.y }, r: { x: a.x, y: a.y } }; | |
| } | |
| // Turns per-frame smile/frown scores into one-shot events with hold time and per-person cooldown | |
| class ExpressionWatcher { | |
| constructor() { this.smileSince = null; this.frownSince = null; this.lastAt = new Map(); } | |
| update(id, sc, tSec) { | |
| const last = this.lastAt.get(id) ?? -Infinity; | |
| if (sc.smile >= SMILE.smileOn) { this.smileSince ??= tSec; } else this.smileSince = null; | |
| if (sc.frown >= SMILE.frownOn) { this.frownSince ??= tSec; } else this.frownSince = null; | |
| if (tSec - last < SMILE.cooldownSec) return null; | |
| if (this.smileSince !== null && tSec - this.smileSince >= SMILE.holdSec) { this.lastAt.set(id, tSec); this.smileSince = null; return "smile"; } | |
| if (this.frownSince !== null && tSec - this.frownSince >= SMILE.frownHoldSec) { this.lastAt.set(id, tSec); this.frownSince = null; return "frown"; } | |
| return null; | |
| } | |
| } | |
| // Counts distinct people who have been the target for FRIENDS.engageSec; persists for the day | |
| class FriendsCounter { | |
| constructor() { | |
| this.key = "howdy-friends-" + new Date().toISOString().slice(0, 10); | |
| this.counted = new Set(); this.since = new Map(); | |
| let n = 0; | |
| if (FRIENDS.persist) { try { n = parseInt(localStorage.getItem(this.key) || "0", 10) || 0; } catch (_) {} } | |
| this.count = n; | |
| } | |
| note(id, tSec) { | |
| if (this.counted.has(id)) return false; | |
| if (!this.since.has(id)) { this.since.set(id, tSec); return false; } | |
| if (tSec - this.since.get(id) < FRIENDS.engageSec) return false; | |
| this.counted.add(id); this.since.delete(id); this.count++; | |
| if (FRIENDS.persist) { try { localStorage.setItem(this.key, String(this.count)); } catch (_) {} } | |
| return true; | |
| } | |
| } | |
| function nearestFace(faces, lm, aspect) { | |
| const hx = lm[0].x, hy = lm[0].y; // wrist | |
| let best = null, bestD = Infinity; | |
| for (const f of faces) { | |
| const d = Math.hypot((hx - f.cx) * aspect, hy - f.cy) / f.w; | |
| if (d < bestD) { best = f; bestD = d; } | |
| } | |
| return best; | |
| } | |
| function handInZone(lm, face, aspect) { | |
| const hx = lm[9].x, hy = lm[9].y; // middle-finger MCP ~ palm center | |
| const dx = Math.abs(hx - face.cx) * aspect / (face.w * aspect); | |
| const dy = (hy - face.cy) / face.h; | |
| return dx <= CROWD.handZoneX && dy >= -CROWD.handZoneUp && dy <= CROWD.handZoneDown; | |
| } | |
| // --------------------------------------------------------------------------- | |
| // Reachy's-eye-view overlay: glow on the person we're watching, thought bubble | |
| // --------------------------------------------------------------------------- | |
| class ThoughtOverlay { | |
| constructor(canvas) { this.canvas = canvas; this.t0 = performance.now(); this.lastMood = ""; this.moodSince = 0; this.text = ""; this.textSince = 0; this.lockT = 0; } | |
| pick(list, sec, period) { return Array.isArray(list) ? list[Math.floor(sec / period) % list.length] : list; } | |
| draw({ faces, target, hands, mood, now, busy, notice, say, listening, friends = 0, friendPop = 99, smile = 0 }) { | |
| const g = this.canvas.getContext("2d"); | |
| const W = this.canvas.width, H = this.canvas.height; | |
| g.clearRect(0, 0, W, H); | |
| if (notice) this.banner(g, notice, W, H); | |
| const sec = (now - this.t0) / 1000; | |
| if (mood !== this.lastMood) { this.lastMood = mood; this.moodSince = now; } | |
| this.lockT = target ? Math.min(1, this.lockT + 0.12) : Math.max(0, this.lockT - 0.12); | |
| const S = Math.min(W, H) / 480; // scale for 640x480-ish frames | |
| // Other faces: faint dotted circles so people can see it's choosing | |
| g.save(); | |
| for (const f of faces) { | |
| if (target && f.cx === target.cx && f.cy === target.cy) continue; | |
| g.setLineDash([6 * S, 6 * S]); g.lineWidth = 2 * S; g.strokeStyle = "rgba(255,255,255,0.45)"; | |
| g.beginPath(); g.ellipse(f.cx * W, f.cy * H, f.w * W * 0.6, f.h * H * 0.65, 0, 0, Math.PI * 2); g.stroke(); | |
| } | |
| g.restore(); | |
| // Hands: soft green dots + a wave meter arc that fills as a wave registers | |
| for (const hand of hands) { | |
| g.fillStyle = "rgba(43,182,115,0.85)"; | |
| for (const p of hand) { g.beginPath(); g.arc(p.x * W, p.y * H, 3.5 * S, 0, Math.PI * 2); g.fill(); } | |
| const m = hand.meter ?? 0, wx = hand[9].x * W, wy = hand[0].y * H + 22 * S; | |
| g.lineWidth = 5 * S; g.lineCap = "round"; | |
| g.strokeStyle = "rgba(255,255,255,0.25)"; g.beginPath(); g.arc(wx, wy, 14 * S, Math.PI * 0.75, Math.PI * 2.25); g.stroke(); | |
| if (m > 0) { g.strokeStyle = m >= 1 ? "#2bb673" : "#f2c94c"; g.beginPath(); g.arc(wx, wy, 14 * S, Math.PI * 0.75, Math.PI * (0.75 + 1.5 * m)); g.stroke(); } | |
| } | |
| // Target: warm glow that breathes, plus a happy ring | |
| let bubbleX = W / 2, bubbleY = H * 0.28, anchored = false; | |
| if (target && this.lockT > 0) { | |
| const cx = target.cx * W, cy = target.cy * H, r = Math.max(target.w * W, target.h * H) * 0.75; | |
| const breathe = 1 + 0.06 * Math.sin(sec * 3); | |
| const rr = r * breathe * (0.6 + 0.4 * this.lockT); | |
| const grad = g.createRadialGradient(cx, cy, rr * 0.6, cx, cy, rr * 1.6); | |
| const col = mood === "reacting" || mood === "waveSeen" ? "43,182,115" : "242,201,76"; | |
| grad.addColorStop(0, `rgba(${col},0)`); grad.addColorStop(0.5, `rgba(${col},${0.35 * this.lockT})`); grad.addColorStop(1, `rgba(${col},0)`); | |
| g.fillStyle = grad; g.beginPath(); g.arc(cx, cy, rr * 1.6, 0, Math.PI * 2); g.fill(); | |
| g.lineWidth = 4 * S; g.strokeStyle = `rgba(${col},${0.9 * this.lockT})`; | |
| g.beginPath(); g.arc(cx, cy, rr, 0, Math.PI * 2); g.stroke(); | |
| // little sparkles when reacting | |
| if (mood === "reacting" || mood === "noticed") { | |
| g.fillStyle = "#fff"; | |
| for (let i = 0; i < 6; i++) { const a = sec * 2 + i * Math.PI / 3, d = rr * (1.25 + 0.15 * Math.sin(sec * 5 + i)); g.beginPath(); g.arc(cx + Math.cos(a) * d, cy + Math.sin(a) * d, 3 * S, 0, Math.PI * 2); g.fill(); } | |
| } | |
| // Smile meter: a little mouth under the ring that curves up as the smile score rises | |
| if (smile > 0.05) { | |
| const mw = rr * 0.5, my = cy + rr + 14 * S, curve = (smile - 0.15) * 2 * mw * 0.6; | |
| g.strokeStyle = smile >= SMILE.smileOn ? "#2bb673" : "rgba(255,255,255,0.7)"; g.lineWidth = 4 * S; g.lineCap = "round"; | |
| g.beginPath(); g.moveTo(cx - mw, my); g.quadraticCurveTo(cx, my + curve, cx + mw, my); g.stroke(); | |
| } | |
| bubbleX = cx; bubbleY = cy - rr - 18 * S; anchored = true; | |
| } | |
| // Thought text by mood | |
| let text; | |
| switch (mood) { | |
| case "noticed": text = SAY.noticed; break; | |
| case "waveSeen": text = SAY.waveSeen; break; | |
| case "reacting": text = this.pick(SAY.reacting, sec, 0.8); break; | |
| case "tracking": text = this.pick(SAY.tracking, sec, SAY.trackingRotateSec); break; | |
| case "searching": text = SAY.searching; break; | |
| default: text = this.pick(SAY.idle, sec, SAY.idleRotateSec); | |
| } | |
| if (friendPop < 1.6) text = SAY.newFriend; | |
| if (say) text = say; | |
| if (text !== this.text) { this.text = text; this.textSince = now; } | |
| const pop = Math.min(1, (now - this.textSince) / 220); | |
| const scale = 0.85 + 0.15 * (1 - Math.pow(1 - pop, 3)); | |
| this.bubble(g, text, bubbleX, bubbleY, anchored, S * scale, W, H); | |
| // Friends badge, top right | |
| this.friendsBadge(g, friends, friendPop, S, W); | |
| // Mascot corner: antennae + face | |
| this.mascot(g, 26 * S, H - 26 * S, S, sec, mood); | |
| if (listening) { // small ear/mic dot pulsing next to the mascot | |
| g.fillStyle = `rgba(43,182,115,${0.5 + 0.4 * Math.sin(sec * 4)})`; g.beginPath(); g.arc(52 * S, H - 40 * S, 5 * S, 0, Math.PI * 2); g.fill(); | |
| } | |
| } | |
| friendsBadge(g, n, pop, S, W) { | |
| const scale = pop < 0.5 ? 1 + 0.18 * Math.sin(Math.PI * pop / 0.5) : 1; | |
| const label = `Friends today: ${n}`; | |
| g.save(); | |
| g.font = `700 ${Math.round(18 * S)}px "Comic Neue", "Segoe Print", "Chalkboard SE", system-ui, sans-serif`; | |
| const tw = g.measureText(label).width, bw = tw + 24 * S, bh = 32 * S, x = W - bw - 16 * S, y = 12 * S; | |
| // scale about the right edge so the pop never leaves the frame | |
| g.translate(x + bw, y + bh / 2); g.scale(scale, scale); g.translate(-(x + bw), -(y + bh / 2)); | |
| g.shadowColor = "rgba(0,0,0,0.25)"; g.shadowBlur = 8 * S; g.shadowOffsetY = 2 * S; | |
| g.fillStyle = pop < 1.6 ? "#2bb673" : "#fffdf5"; roundRect(g, x, y, bw, bh, 16 * S); g.fill(); | |
| g.shadowColor = "transparent"; | |
| g.strokeStyle = "#2b2b2b"; g.lineWidth = 2 * S; roundRect(g, x, y, bw, bh, 16 * S); g.stroke(); | |
| g.fillStyle = pop < 1.6 ? "#fff" : "#2b2b2b"; g.textAlign = "center"; g.textBaseline = "middle"; | |
| g.fillText(label, x + bw / 2, y + bh / 2 + 1 * S); | |
| g.restore(); | |
| } | |
| banner(g, text, W, H) { | |
| const S = Math.min(W, H) / 480; | |
| g.save(); | |
| g.font = `600 ${Math.round(16 * S)}px system-ui, sans-serif`; | |
| const words = text.split(" "), lines = []; let cur = ""; | |
| for (const w of words) { const t = cur ? cur + " " + w : w; if (g.measureText(t).width > W - 48 * S) { lines.push(cur); cur = w; } else cur = t; } | |
| if (cur) lines.push(cur); | |
| const lh = 22 * S, bh = lines.length * lh + 20 * S, by = H - bh - 62 * S; | |
| g.fillStyle = "rgba(180,40,40,0.92)"; roundRect(g, 12 * S, by, W - 24 * S, bh, 10 * S); g.fill(); | |
| g.fillStyle = "#fff"; g.textAlign = "center"; g.textBaseline = "middle"; | |
| lines.forEach((l, i) => g.fillText(l, W / 2, by + 10 * S + lh * (i + 0.5))); | |
| g.restore(); | |
| } | |
| bubble(g, text, x, y, anchored, S, W, H) { | |
| const font = `${Math.round(28 * S)}px "Comic Neue", "Segoe Print", "Chalkboard SE", "Marker Felt", system-ui, sans-serif`; | |
| g.font = `700 ${font}`; | |
| const padX = 16 * S, padY = 10 * S; | |
| const tw = g.measureText(text).width, bw = tw + padX * 2, bh = 34 * S + padY; | |
| let bx = clamp(x - bw / 2, 8, W - bw - 8), by = anchored ? y - bh : y; | |
| if (by < 8) by = 8; | |
| g.save(); | |
| g.shadowColor = "rgba(0,0,0,0.25)"; g.shadowBlur = 10 * S; g.shadowOffsetY = 3 * S; | |
| g.fillStyle = "#fffdf5"; | |
| roundRect(g, bx, by, bw, bh, 16 * S); g.fill(); | |
| g.restore(); | |
| g.strokeStyle = "#2b2b2b"; g.lineWidth = 2.5 * S; roundRect(g, bx, by, bw, bh, 16 * S); g.stroke(); | |
| if (anchored) { // thought-bubble dots down toward the face | |
| g.fillStyle = "#fffdf5"; | |
| for (const [k, r] of [[0.35, 6], [0.7, 4]]) { const cx = x, cy = by + bh + (18 * S) * k; g.beginPath(); g.arc(cx, cy, r * S, 0, Math.PI * 2); g.fill(); g.stroke(); } | |
| } | |
| g.fillStyle = "#2b2b2b"; g.textBaseline = "middle"; g.textAlign = "center"; | |
| g.fillText(text, bx + bw / 2, by + bh / 2 + 1 * S); | |
| } | |
| mascot(g, x, y, S, sec, mood) { | |
| const r = 16 * S, wag = (mood === "reacting" || mood === "noticed") ? Math.sin(sec * 14) * 0.5 : Math.sin(sec * 2) * 0.12; | |
| g.save(); g.translate(x, y); | |
| g.strokeStyle = "#2b2b2b"; g.lineWidth = 3 * S; g.lineCap = "round"; | |
| for (const side of [-1, 1]) { g.beginPath(); g.moveTo(side * r * 0.5, -r * 0.8); g.lineTo(side * r * 0.5 + Math.sin(wag) * side * 8 * S, -r * 1.9); g.stroke(); | |
| g.fillStyle = "#f2c94c"; g.beginPath(); g.arc(side * r * 0.5 + Math.sin(wag) * side * 8 * S, -r * 1.9, 3.5 * S, 0, Math.PI * 2); g.fill(); } | |
| g.fillStyle = "#fffdf5"; g.beginPath(); g.arc(0, 0, r, 0, Math.PI * 2); g.fill(); g.stroke(); | |
| g.fillStyle = "#2b2b2b"; | |
| const blink = (sec % 4) > 3.85 ? 0.15 : 1; | |
| for (const side of [-1, 1]) { g.beginPath(); g.ellipse(side * r * 0.4, -r * 0.15, 3 * S, 4 * S * blink, 0, 0, Math.PI * 2); g.fill(); } | |
| g.beginPath(); g.arc(0, r * 0.25, r * 0.45, 0.15 * Math.PI, 0.85 * Math.PI); g.stroke(); | |
| g.restore(); | |
| } | |
| } | |
| // --------------------------------------------------------------------------- | |
| // Big screen + voice | |
| // --------------------------------------------------------------------------- | |
| function toggleBigScreen(ui, on) { | |
| document.body.classList.toggle("big", on); | |
| ui.btnBig.textContent = on ? "Exit big screen" : "Big screen"; | |
| if (on) { | |
| // True fullscreen only works if every enclosing iframe allows it; otherwise the page-fill layout + F11 does the job | |
| const el = ui.stage; | |
| (el.requestFullscreen?.() ?? Promise.reject()).catch(() => log("Browser fullscreen blocked by the frame; press F11 for the browser's fullscreen")); | |
| } else if (document.fullscreenElement) { | |
| document.exitFullscreen?.().catch(() => {}); | |
| } | |
| } | |
| class VoiceTriggers { | |
| constructor(onTrigger) { | |
| this.onTrigger = onTrigger; this.active = false; this.rec = null; this.lastAt = -Infinity; this.wanted = false; | |
| this.recent = []; // [{t, text}] final results from the last few seconds, joined for matching | |
| this.Ctor = window.SpeechRecognition || window.webkitSpeechRecognition || null; | |
| } | |
| toggle(btn) { this.active ? this.stop(btn) : this.start(btn); } | |
| start(btn) { | |
| if (!this.Ctor) { log("Voice: this browser has no speech recognition (use Chrome or Edge)"); return; } | |
| this.wanted = true; | |
| const r = new this.Ctor(); | |
| r.lang = VOICE.lang; r.continuous = true; r.interimResults = true; r.maxAlternatives = 3; | |
| r.onresult = (e) => { | |
| for (let i = e.resultIndex; i < e.results.length; i++) { | |
| const res = e.results[i]; | |
| if (res.isFinal && VOICE.logTranscripts) log(`Heard: "${res[0].transcript.trim()}"`); | |
| for (let k = 0; k < res.length; k++) this.check(res[k].transcript, res.isFinal); | |
| } | |
| }; | |
| r.onaudiostart = () => log("Voice: mic open"); | |
| r.onspeechstart = () => console.debug("[voice] speech start"); | |
| navigator.permissions?.query?.({ name: "microphone" }).then((p) => log(`Mic permission: ${p.state}`)).catch(() => {}); | |
| r.onerror = (e) => { if (e.error !== "no-speech" && e.error !== "aborted") log(`Voice error: ${e.error}`); if (e.error === "not-allowed") this.wanted = false; }; | |
| r.onend = () => { if (this.wanted) { try { r.start(); } catch (_) { setTimeout(() => this.wanted && this.start(btn), 1000); } } else this.setActive(false, btn); }; | |
| try { r.start(); } catch (e) { log(`Voice start failed: ${e.message}`); return; } | |
| this.rec = r; this.setActive(true, btn); log("Voice: listening (laptop mic)"); | |
| } | |
| stop(btn) { this.wanted = false; try { this.rec?.stop(); } catch (_) {} this.setActive(false, btn); log("Voice: off"); } | |
| setActive(on, btn) { this.active = on; if (btn) { btn.textContent = on ? "Voice: listening" : "Voice: off"; btn.setAttribute("aria-pressed", String(on)); } } | |
| check(transcript, isFinal) { | |
| const now = performance.now(); | |
| const clean = transcript.toLowerCase().replace(/[^a-z' ]+/g, " ").replace(/\s+/g, " ").trim(); | |
| if (isFinal) { this.recent.push({ t: now, text: clean }); } | |
| this.recent = this.recent.filter((r) => now - r.t < 4000); | |
| // Match against recent finals + the current phrase, so "hello" ... "Richie" said with a pause still joins up | |
| const t = " " + [...this.recent.map((r) => r.text), isFinal ? "" : clean].join(" ").replace(/\s+/g, " ").trim() + " "; | |
| if (now - this.lastAt < VOICE.cooldownSec * 1000) return; | |
| for (const trig of VOICE.triggers) { | |
| const hit = trig.match ? trig.match.test(t.trim()) : trig.pattern ? trig.pattern.test(t) : (trig.phrases ?? []).some((p) => t.includes(" " + p + " ")); | |
| if (hit) { this.lastAt = now; this.recent = []; this.onTrigger(trig, transcript.trim()); return; } | |
| } | |
| } | |
| } | |
| function roundRect(g, x, y, w, h, r) { | |
| g.beginPath(); g.moveTo(x + r, y); g.arcTo(x + w, y, x + w, y + h, r); g.arcTo(x + w, y + h, x, y + h, r); g.arcTo(x, y + h, x, y, r); g.arcTo(x, y, x + w, y, r); g.closePath(); | |
| } | |
| // --------------------------------------------------------------------------- | |
| // Emotion clips: JSON motion + Opus audio -> 16 kHz mono PCM WAV | |
| // --------------------------------------------------------------------------- | |
| class ClipCache { | |
| constructor() { this.map = new Map(); } | |
| async preload(names) { | |
| const results = await Promise.allSettled(names.map((n) => this.get(n))); | |
| return results.filter((r) => r.status === "fulfilled" && r.value).length; | |
| } | |
| async get(name) { | |
| if (this.map.has(name)) return this.map.get(name); | |
| const p = (async () => { | |
| try { | |
| const [motionRes, audioRes] = await Promise.all([ | |
| fetch(EMOTIONS_BASE + name + ".json"), | |
| fetch(EMOTIONS_BASE + name + ".ogg"), | |
| ]); | |
| if (!motionRes.ok) throw new Error(`motion ${motionRes.status}`); | |
| const motion = normalizeMotion(await motionRes.json()); | |
| let wav; | |
| if (audioRes.ok) wav = await toWav16kMono(await audioRes.arrayBuffer()); | |
| return { motion, wav }; | |
| } catch (e) { | |
| log(`Failed to load ${name}: ${e.message}`); | |
| this.map.delete(name); | |
| return null; | |
| } | |
| })(); | |
| this.map.set(name, p); | |
| return p; | |
| } | |
| } | |
| function normalizeMotion(raw) { | |
| // Flatten nested 4x4 head matrices to 16 floats; leave everything else as-is | |
| const set_target_data = raw.set_target_data.map((f) => ({ | |
| ...f, | |
| head: f.head && Array.isArray(f.head[0]) ? f.head.flat() : f.head, | |
| })); | |
| return { ...raw, set_target_data }; | |
| } | |
| async function toWav16kMono(arrayBuffer) { | |
| const ctx = new (window.AudioContext || window.webkitAudioContext)(); | |
| const decoded = await ctx.decodeAudioData(arrayBuffer); | |
| await ctx.close(); | |
| const targetRate = 16000; | |
| const length = Math.ceil(decoded.duration * targetRate); | |
| const off = new OfflineAudioContext(1, length, targetRate); | |
| const src = off.createBufferSource(); | |
| src.buffer = decoded; | |
| src.connect(off.destination); | |
| src.start(); | |
| const rendered = await off.startRendering(); | |
| return encodeWav(rendered.getChannelData(0), targetRate); | |
| } | |
| function encodeWav(samples, sampleRate) { | |
| const buf = new ArrayBuffer(44 + samples.length * 2); | |
| const v = new DataView(buf); | |
| const str = (o, s) => { for (let i = 0; i < s.length; i++) v.setUint8(o + i, s.charCodeAt(i)); }; | |
| str(0, "RIFF"); v.setUint32(4, 36 + samples.length * 2, true); str(8, "WAVE"); | |
| str(12, "fmt "); v.setUint32(16, 16, true); v.setUint16(20, 1, true); v.setUint16(22, 1, true); | |
| v.setUint32(24, sampleRate, true); v.setUint32(28, sampleRate * 2, true); | |
| v.setUint16(32, 2, true); v.setUint16(34, 16, true); | |
| str(36, "data"); v.setUint32(40, samples.length * 2, true); | |
| let o = 44; | |
| for (let i = 0; i < samples.length; i++, o += 2) { | |
| const s = Math.max(-1, Math.min(1, samples[i])); | |
| v.setInt16(o, s < 0 ? s * 0x8000 : s * 0x7fff, true); | |
| } | |
| return new Blob([buf], { type: "audio/wav" }); | |
| } | |
| // A small family of synthesized chirps; one is picked at random each time | |
| const CHIRPS = [ | |
| { dur: 0.45, parts: [[0.0, 0.18, 900, 1600], [0.22, 0.45, 1100, 2000]] }, // classic double | |
| { dur: 0.6, parts: [[0.0, 0.14, 800, 1500], [0.18, 0.32, 1000, 1800], [0.38, 0.6, 1200, 2200]] }, // triple, climbing | |
| { dur: 0.5, parts: [[0.0, 0.5, 700, 2400]] }, // long swoop up | |
| { dur: 0.55, parts: [[0.0, 0.25, 1500, 900], [0.3, 0.55, 900, 1900]] }, // dip then rise (question) | |
| { dur: 0.7, parts: [[0.0, 0.1, 1400, 1700], [0.12, 0.22, 1400, 1700], [0.24, 0.34, 1400, 1700], [0.4, 0.7, 1200, 2300]] }, // trill + flourish | |
| { dur: 0.5, parts: [[0.0, 0.2, 2000, 1300], [0.25, 0.5, 1300, 2600]] }, // whistle | |
| ]; | |
| function chirpWav(which) { | |
| const sr = 16000; | |
| const c = CHIRPS[which ?? Math.floor(Math.random() * CHIRPS.length)]; | |
| const n = Math.floor(sr * c.dur); | |
| const out = new Float32Array(n); | |
| for (const [t0, t1, f0, f1] of c.parts) { | |
| for (let i = Math.floor(t0 * sr); i < Math.floor(t1 * sr) && i < n; i++) { | |
| const t = i / sr - t0, T = t1 - t0, u = t / T; | |
| const env = Math.sin(Math.PI * u); | |
| out[i] += 0.6 * env * Math.sin(2 * Math.PI * (f0 * t + 0.5 * (f1 - f0) * t * t / T)); | |
| } | |
| } | |
| return encodeWav(out, sr); | |
| } | |
| // --------------------------------------------------------------------------- | |
| // Motion helpers | |
| // --------------------------------------------------------------------------- | |
| // Short nod from the current pose, antennas perk, 0.9 s at 50 Hz | |
| function buildNodMotion({ yawDeg, pitchDeg, bodyYawDeg }) { | |
| // Keep the nod inside a comfortable envelope so the daemon's IK always converges | |
| if (![yawDeg, pitchDeg, bodyYawDeg].every(Number.isFinite)) { yawDeg = 0; pitchDeg = TRACK.restPitchDeg; bodyYawDeg = 0; } | |
| bodyYawDeg = clamp(bodyYawDeg, -TRACK.maxBodyYawDeg, TRACK.maxBodyYawDeg); | |
| yawDeg = clamp(yawDeg, bodyYawDeg - 40, bodyYawDeg + 40); | |
| pitchDeg = clamp(pitchDeg, -30, 15); | |
| const hz = 50, dur = 0.9, n = Math.round(dur * hz); | |
| const time = [], set_target_data = []; | |
| for (let i = 0; i <= n; i++) { | |
| const t = i / hz, u = t / dur; | |
| const dip = u < 0.625 ? 14 * Math.sin(Math.PI * u * 1.6) : 0; // one dip in the first ~0.56 s | |
| const perk = 25 * DEG * Math.sin(Math.PI * u); | |
| time.push(t); | |
| set_target_data.push({ | |
| head: headMatrix(0, (pitchDeg + dip) * DEG, yawDeg * DEG), | |
| antennas: [INIT_ANTENNAS[0] - perk, INIT_ANTENNAS[1] + perk], | |
| body_yaw: bodyYawDeg * DEG, | |
| }); | |
| } | |
| return { time, set_target_data }; | |
| } | |
| // Pre-rendered spoken lines (already 16 kHz mono PCM WAV, so no conversion needed) | |
| class SpeechCache { | |
| constructor() { this.map = new Map(); } | |
| async preloadAll() { | |
| const files = [...new Set(Object.values(SPEECH.lines).flat())]; | |
| const r = await Promise.allSettled(files.map((f) => this.load(f))); | |
| return r.filter((x) => x.status === "fulfilled" && x.value).length; | |
| } | |
| async load(file) { | |
| if (this.map.has(file)) return this.map.get(file); | |
| const p = (async () => { | |
| try { | |
| const res = await fetch(SPEECH.base + file + "?v=" + SPEECH.version); | |
| if (!res.ok) throw new Error(`${res.status}`); | |
| const buf = await res.arrayBuffer(); | |
| return { blob: new Blob([buf], { type: "audio/wav" }), durSec: wavDuration(buf) }; | |
| } catch (e) { log(`Voice line ${file} failed: ${e.message}`); this.map.delete(file); return null; } | |
| })(); | |
| this.map.set(file, p); | |
| return p; | |
| } | |
| async get(key) { | |
| const list = SPEECH.lines[key]; | |
| if (!list?.length) return null; | |
| return this.load(list[Math.floor(Math.random() * list.length)]); | |
| } | |
| } | |
| // Walk the RIFF chunks to find the real data length (ffmpeg and others add LIST chunks before it) | |
| function wavDuration(buf) { | |
| const v = new DataView(buf); | |
| let rate = 16000, channels = 1, bits = 16, pos = 12; | |
| while (pos + 8 <= buf.byteLength) { | |
| const id = String.fromCharCode(v.getUint8(pos), v.getUint8(pos + 1), v.getUint8(pos + 2), v.getUint8(pos + 3)); | |
| const size = v.getUint32(pos + 4, true); | |
| if (id === "fmt ") { channels = v.getUint16(pos + 10, true); rate = v.getUint32(pos + 12, true); bits = v.getUint16(pos + 22, true); } | |
| if (id === "data") return size / (rate * channels * (bits / 8)); | |
| pos += 8 + size + (size & 1); | |
| } | |
| return 1.0; | |
| } | |
| // "Talking": hold the head, small quick nods and antenna flicks for the length of the line | |
| function buildTalkMotion({ yawDeg, pitchDeg, bodyYawDeg, durSec = 1.2 }) { | |
| if (![yawDeg, pitchDeg, bodyYawDeg].every(Number.isFinite)) { yawDeg = 0; pitchDeg = TRACK.restPitchDeg; bodyYawDeg = 0; } | |
| bodyYawDeg = clamp(bodyYawDeg, -TRACK.maxBodyYawDeg, TRACK.maxBodyYawDeg); | |
| yawDeg = clamp(yawDeg, bodyYawDeg - 40, bodyYawDeg + 40); | |
| pitchDeg = clamp(pitchDeg, -30, 15); | |
| const hz = 50, n = Math.round(durSec * hz), time = [], set_target_data = []; | |
| for (let i = 0; i <= n; i++) { | |
| const t = i / hz, u = t / durSec, env = Math.sin(Math.PI * Math.min(1, u)); | |
| const bob = 3 * env * Math.sin(2 * Math.PI * 4.5 * t); // little syllable nods | |
| const perk = 0.35 * env * (0.6 + 0.4 * Math.sin(2 * Math.PI * 3 * t)); | |
| time.push(t); | |
| set_target_data.push({ | |
| head: headMatrix(0, (pitchDeg + bob) * DEG, yawDeg * DEG), | |
| antennas: [INIT_ANTENNAS[0] - perk, INIT_ANTENNAS[1] + perk], | |
| body_yaw: bodyYawDeg * DEG, | |
| }); | |
| } | |
| return { time, set_target_data }; | |
| } | |
| // Antenna wave-back: hold the current head pose, swing one or both antennae, ~1.6 s at 50 Hz | |
| function buildAntennaWave({ yawDeg, pitchDeg, bodyYawDeg, both = false, side = 1 }) { | |
| if (![yawDeg, pitchDeg, bodyYawDeg].every(Number.isFinite)) { yawDeg = 0; pitchDeg = TRACK.restPitchDeg; bodyYawDeg = 0; } | |
| const hz = 50, dur = 1.6, n = Math.round(dur * hz), swings = 3; | |
| const amp = 0.9; // radians of antenna swing | |
| const time = [], set_target_data = []; | |
| const head = headMatrix(0, pitchDeg * DEG, yawDeg * DEG); | |
| for (let i = 0; i <= n; i++) { | |
| const t = i / hz, u = t / dur; | |
| const env = Math.sin(Math.PI * u); // ease in/out | |
| const w = amp * env * Math.sin(2 * Math.PI * swings * u); // the wave itself | |
| const a = [INIT_ANTENNAS[0], INIT_ANTENNAS[1]]; | |
| if (both) { a[0] = INIT_ANTENNAS[0] - w; a[1] = INIT_ANTENNAS[1] + w; } // opposite phase, like two arms | |
| else if (side === 0) a[0] = INIT_ANTENNAS[0] - w; else a[1] = INIT_ANTENNAS[1] + w; | |
| time.push(t); | |
| set_target_data.push({ head, antennas: a, body_yaw: bodyYawDeg * DEG }); | |
| } | |
| return { time, set_target_data }; | |
| } | |
| // Tilt a whole recorded move up/down (rotate about world y; negative = nose up), keeping head pitch in range | |
| function offsetMotionPitch(motion, pitchRad) { | |
| if (Math.abs(pitchRad) < 1e-4) return motion; | |
| const Ry = headMatrix(0, pitchRad, 0); | |
| return { | |
| ...motion, | |
| set_target_data: motion.set_target_data.map((f) => { | |
| if (!f.head) return f; | |
| const h = mat4mul(Ry, f.head); | |
| const rp = matrixToRpyDeg(h); | |
| if (rp.pitch < -TRACK.maxPitchUpDeg || rp.pitch > TRACK.maxPitchDownDeg) return f; // leave extreme frames untilted | |
| return { ...f, head: h }; | |
| }), | |
| }; | |
| } | |
| // Cut a move short (with a quick fade back toward the first frame so it doesn't stop mid-pose) | |
| function truncateMotion(motion, maxSec) { | |
| const idx = motion.time.findIndex((t) => t > maxSec); | |
| if (idx < 0) return motion; | |
| const time = motion.time.slice(0, idx), frames = motion.set_target_data.slice(0, idx); | |
| const first = frames[0], last = frames[frames.length - 1], t0 = time[time.length - 1], hz = 50, n = 25; | |
| for (let i = 1; i <= n; i++) { | |
| const u = i / n, t = t0 + i / hz; | |
| const head = last.head && first.head ? last.head.map((v, k) => v + (first.head[k] - v) * u) : last.head; | |
| const antennas = last.antennas && first.antennas ? last.antennas.map((v, k) => v + (first.antennas[k] - v) * u) : last.antennas; | |
| const body_yaw = typeof last.body_yaw === "number" ? last.body_yaw + ((first.body_yaw ?? last.body_yaw) - last.body_yaw) * u : last.body_yaw; | |
| time.push(t); frames.push({ ...last, head, antennas, body_yaw }); | |
| } | |
| return { ...motion, time, set_target_data: frames }; | |
| } | |
| // Rotate a whole recorded move about world z by `yawRad` so it plays relative to the current body yaw | |
| function offsetMotionYaw(motion, yawRad) { | |
| if (Math.abs(yawRad) < 1e-4) return motion; | |
| const Rz = headMatrix(0, 0, yawRad); | |
| const maxBody = TRACK.maxBodyYawDeg * DEG; | |
| return { | |
| ...motion, | |
| set_target_data: motion.set_target_data.map((f) => ({ | |
| ...f, | |
| head: f.head ? mat4mul(Rz, f.head) : f.head, | |
| body_yaw: typeof f.body_yaw === "number" ? clamp(f.body_yaw + yawRad, -maxBody, maxBody) : f.body_yaw, | |
| })), | |
| }; | |
| } | |
| // Row-major flat 4x4 from roll/pitch/yaw (radians), ZYX convention, zero translation | |
| function headMatrix(r, p, y) { | |
| const cr = Math.cos(r), sr = Math.sin(r), cp = Math.cos(p), sp = Math.sin(p), cy = Math.cos(y), sy = Math.sin(y); | |
| return [ | |
| cy * cp, cy * sp * sr - sy * cr, cy * sp * cr + sy * sr, 0, | |
| sy * cp, sy * sp * sr + cy * cr, sy * sp * cr - cy * sr, 0, | |
| -sp, cp * sr, cp * cr, 0, | |
| 0, 0, 0, 1, | |
| ]; | |
| } | |
| function toNested4x4(m) { | |
| if (Array.isArray(m[0])) return m; | |
| return [m.slice(0, 4), m.slice(4, 8), m.slice(8, 12), m.slice(12, 16)]; | |
| } | |
| function matrixToRpyDeg(m) { | |
| if (Array.isArray(m[0])) m = m.flat(); | |
| const pitch = Math.asin(clamp(-m[8], -1, 1)); | |
| const roll = Math.atan2(m[9], m[10]); | |
| const yaw = Math.atan2(m[4], m[0]); | |
| return { roll: roll / DEG, pitch: pitch / DEG, yaw: yaw / DEG }; | |
| } | |
| function mat4mul(a, b) { | |
| const o = new Array(16); | |
| for (let r = 0; r < 4; r++) for (let c = 0; c < 4; c++) { | |
| let s = 0; | |
| for (let k = 0; k < 4; k++) s += a[r * 4 + k] * b[k * 4 + c]; | |
| o[r * 4 + c] = s; | |
| } | |
| return o; | |
| } | |
| function safeReturnHome(reachy) { | |
| try { reachy.setMotorMode?.("enabled"); } catch (_) {} | |
| const target = { head: headMatrix(0, TRACK.restPitchDeg * DEG, 0), antennas: INIT_ANTENNAS, body_yaw: 0 }; | |
| try { | |
| if (typeof reachy.gotoTarget === "function") reachy.gotoTarget({ ...target, duration: 1.2 }); | |
| else reachy.setTarget(target); | |
| } catch (_) {} | |
| } | |
| // --------------------------------------------------------------------------- | |
| // UI | |
| // --------------------------------------------------------------------------- | |
| function bindUi() { | |
| const $ = (id) => document.getElementById(id); | |
| return { | |
| video: $("video"), overlay: $("overlay"), | |
| btnTrack: $("btnTrack"), btnBody: $("btnBody"), btnMute: $("btnMute"), btnWave: $("btnWave"), btnWake: $("btnWake"), | |
| btnBig: $("btnBig"), btnVoice: $("btnVoice"), stage: document.querySelector(".stage"), | |
| }; | |
| } | |
| function setStatus(text, sub = "", mode = "") { | |
| const el = document.getElementById("status"); | |
| document.getElementById("statusText").textContent = text; | |
| document.getElementById("statusSub").textContent = sub; | |
| el.className = "status" + (mode ? " " + mode : ""); | |
| } | |
| function log(msg) { | |
| const el = document.getElementById("log"); | |
| if (!el) return; | |
| const t = new Date().toLocaleTimeString([], { hour12: false }); | |
| el.textContent = `${t} ${msg}\n` + el.textContent.split("\n").slice(0, 14).join("\n"); | |
| console.log("[howdy-friend]", msg); | |
| } | |
| function syncOverlaySize(canvas, video) { | |
| if (canvas.width !== video.videoWidth || canvas.height !== video.videoHeight) { | |
| canvas.width = video.videoWidth; canvas.height = video.videoHeight; | |
| } | |
| } | |
| const clamp = (v, lo, hi) => Math.max(lo, Math.min(hi, v)); | |
| const lerp = (a, b, t) => a + (b - a) * t; | |