import type { ReachyMiniInstance } from "@pollen-robotics/reachy-mini-sdk"; import { RealtimeSession } from "./realtime"; import { getRobotMicTrack, getRobotSpeakerTrack, primeRemoteTrack, resumeAudio, setPhoneOutput, ttsLevel } from "./audio"; import { countVideoReceivers } from "./camera"; import type { ToolContext } from "./tools"; import type { RobotGestures } from "../robot/gestures"; import type { Lang } from "../i18n"; export type VoiceStatus = "idle" | "connecting" | "listening" | "speaking" | "error"; /** Additive head/antenna accent for an emotion, evaluated at `elapsed` seconds * into a move of length `dur`. Returned degrees are ADDED on top of the base * wobble in the 20 Hz loop — one writer, so no motor jitter. A bell envelope * (rise + fall) blends every accent smoothly in and out of the ongoing motion. * Mirrors the official app's `play_emotion` intents with procedural motion. */ interface EmoteOffset { roll: number; pitch: number; yaw: number; ant: number; } interface EmoteSpec { dur: number; shape: (t: number, env: number, dur: number) => EmoteOffset; } // pitch>0 looks down; antenna>0 perks up. Kept modest so accents read as // body language, not spasms. const EMOTES: Record = { // happy / success / grateful: two brisk nods, antennas up. happy: { dur: 1.4, shape: (t, env) => ({ roll: 0, pitch: 13 * env * Math.sin(t * 9), yaw: 0, ant: 16 * env }) }, // excited / dance: fast antenna flutter + a little body-side sway. excited: { dur: 1.8, shape: (t, env) => ({ roll: 3 * env * Math.sin(t * 5), pitch: -4 * env, yaw: 9 * env * Math.sin(t * 5.5), ant: 22 * env * (0.6 + 0.4 * Math.sin(t * 17)) }) }, // thinking / uncertain: tilt away and glance up, held then released. thinking: { dur: 2.0, shape: (_t, env) => ({ roll: 15 * env, pitch: -7 * env, yaw: 6 * env, ant: 5 * env }) }, // confused: tilt plus a small "huh?" head shake. confused: { dur: 1.6, shape: (t, env) => ({ roll: 12 * env, pitch: 2 * env, yaw: 7 * env * Math.sin(t * 6), ant: -6 * env }) }, // sad / lonely: droop the head and let the antennas fall. sad: { dur: 2.0, shape: (_t, env) => ({ roll: 4 * env, pitch: 15 * env, yaw: 0, ant: -22 * env }) }, // attentive / curious: perk up and lean in. attentive: { dur: 1.4, shape: (_t, env) => ({ roll: 0, pitch: -7 * env, yaw: 0, ant: 15 * env }) }, // loving: slow warm tilt with a gentle nod. loving: { dur: 1.8, shape: (t, env) => ({ roll: 11 * env, pitch: 5 * env * Math.sin(t * 3), yaw: 0, ant: 10 * env }) }, // angry / irritated: sharp quick shake, antennas pinned back. angry: { dur: 1.2, shape: (t, env) => ({ roll: 0, pitch: 0, yaw: 11 * env * Math.sin(t * 12), ant: -12 * env }) }, // surprised: quick jerk up, antennas shoot up. surprised: { dur: 1.3, shape: (_t, env) => ({ roll: 0, pitch: -14 * env, yaw: 0, ant: 26 * env }) }, // greeting / yes: a single clear nod. nod: { dur: 1.2, shape: (t, env) => ({ roll: 0, pitch: 12 * env * Math.sin(t * 5.2), yaw: 0, ant: 8 * env }) }, }; // Synonyms → canonical emote, so the model's free-text intent always lands. const EMOTE_ALIASES: Record = { laughing: "happy", laugh: "happy", joy: "happy", grateful: "happy", success: "happy", proud: "happy", dance: "excited", playful: "excited", celebrate: "excited", thoughtful: "thinking", uncertain: "thinking", wondering: "thinking", downcast: "sad", lonely: "sad", disappointed: "sad", curious: "attentive", interested: "attentive", listening: "attentive", love: "loving", affection: "loving", irritated: "angry", rage: "angry", annoyed: "angry", shocked: "surprised", wow: "surprised", yes: "nod", agree: "nod", greet: "nod", hello: "nod", }; export function resolveEmote(name: string): string | null { const key = name.trim().toLowerCase().replace(/[^a-z]/g, ""); if (key in EMOTES) return key; if (key in EMOTE_ALIASES) return EMOTE_ALIASES[key]; return null; } /** Robot-only voice loop: Reachy's mic in, Reachy's speaker out, and smooth * client-side head motion while it speaks or listens. There is deliberately * no phone-mic/phone-speaker mode — the interaction is with the robot. */ export class VoiceController { private session: RealtimeSession | null = null; status: VoiceStatus = "idle"; constructor( private opts: { getReachy: () => ReachyMiniInstance | null; gestures: RobotGestures | null; getApiKey: () => string | null; getLang: () => Lang; getName: () => string; getVoice: () => string; getActivationMode: () => "always" | "name"; getAllowInterrupt: () => boolean; toolCtx: ToolContext; onStatus: (s: VoiceStatus) => void; onError: (message: string) => void; onTranscript: (role: "user" | "assistant", text: string) => void; }, ) {} get active(): boolean { return this.session !== null; } private setStatus(s: VoiceStatus): void { this.status = s; this.opts.onStatus(s); } async toggle(): Promise { if (this.active) { this.stop(); } else { await this.start(); } } async start(): Promise { const apiKey = this.opts.getApiKey(); if (!apiKey) { this.opts.onError("Add your OpenAI key in Settings to talk to Reachy."); return; } const reachy = this.opts.getReachy(); if (!reachy) { this.opts.onError("Connect to your Reachy first."); return; } this.setStatus("connecting"); await resumeAudio(); // Robot ears: the mic track the robot streams to us over WebRTC. const micTrack = getRobotMicTrack(reachy); if (!micTrack) { console.warn("[voice] no live robot audio receiver track found"); this.opts.onError("Reachy's microphone isn't available — try leaving and reconnecting."); this.setStatus("idle"); return; } // Chrome: remote tracks feed silence into WebAudio unless also consumed by // a media element. Prime it before the mic pump reads it. primeRemoteTrack(micTrack); // Robot voice: replace the SDK's silent placeholder on the WebRTC audio // sender with our TTS bus (SDK 1.8 contract — it never calls getUserMedia). if (!this.injectSpeakerTrack(reachy)) { this.opts.onError("Couldn't reach Reachy's speaker — try leaving and reconnecting."); this.setStatus("idle"); return; } setPhoneOutput(false); // the robot is the only voice console.log(`[voice] video receivers: ${countVideoReceivers(reachy)}`); this.opts.gestures?.cancel(); try { reachy.setMicMuted(false); reachy.sendRaw({ type: "set_wobbling", enabled: false }); // we own motion } catch { /* older daemon */ } this.startMotion(); this.session = new RealtimeSession({ apiKey, lang: this.opts.getLang(), robotName: this.opts.getName(), voice: this.opts.getVoice(), activationMode: this.opts.getActivationMode(), allowInterrupt: this.opts.getAllowInterrupt(), micTrack, // Give the model a body: play_emotion routes here and overlays an accent // on the live motion loop. toolCtx: { ...this.opts.toolCtx, emote: (name: string) => this.emote(name) }, callbacks: { onError: (m) => { this.opts.onError(m); this.setStatus("error"); }, onUserText: (t) => this.opts.onTranscript("user", t), onAssistantText: (t) => this.opts.onTranscript("assistant", t), onSpeakingChange: (speaking) => { this.speaking = speaking; this.setStatus(speaking ? "speaking" : "listening"); }, onClosed: () => { if (this.session) this.stop(); }, }, }); this.session.start(); this.setStatus("listening"); } stop(): void { this.session?.close(); this.session = null; this.speaking = false; this.stopThinking(); this.stopMotion(); const reachy = this.opts.getReachy(); try { this.restoreSpeakerTrack(); reachy?.setMicMuted(true); } catch { /* channel closed */ } this.setStatus("idle"); } // ── Robot-speaker audio injection (replaceTrack on the WebRTC audio sender) ── private audioSender: RTCRtpSender | null = null; private placeholderTrack: MediaStreamTrack | null = null; /** Returns true if our TTS track was attached to the robot's audio sender. */ private injectSpeakerTrack(reachy: ReachyMiniInstance | null): boolean { const pc = reachy?._pc ?? null; const ttsTrack = getRobotSpeakerTrack(); if (!pc || !ttsTrack) return false; const sender = pc.getSenders().find((s) => s.track && s.track.kind === "audio") ?? null; if (!sender) return false; this.audioSender = sender; this.placeholderTrack = sender.track; // the SDK's silent placeholder void sender.replaceTrack(ttsTrack); return true; } private restoreSpeakerTrack(): void { if (this.audioSender) { void this.audioSender.replaceTrack(this.placeholderTrack); this.audioSender = null; this.placeholderTrack = null; } } // ── Client-side motion (~20 Hz, reachy-stories style) ────────────────────── // Speaking: head wobble scaled by a smoothed speech envelope (with a floor so // speech is always visibly animated) + antennas eased up slowly. // Listening: slow attentive sway so the robot feels alive. // Antennas are never oscillated at audio rate — that was the buzz. private speaking = false; private motionRaf: number | null = null; private env = 0; private ant = 0; private lastCmd = 0; private lastLog = 0; // Active emotion accent (overlaid additively on the wobble). private emoteSpec: EmoteSpec | null = null; private emoteStart = 0; private lastActivity = 0; /** Trigger an emotion accent — called by the play_emotion tool and by * contextual reactions. Safe to call anytime; ignored if the name is * unknown. Overlays on the running motion loop, so it never fights it. */ emote(name: string): void { const key = resolveEmote(name); if (!key) return; this.emoteSpec = EMOTES[key]; this.emoteStart = performance.now(); this.lastActivity = this.emoteStart; } // Deliberate "I'm thinking" body language while a slow tool (menu design) // runs — otherwise the robot sits silent and still for 10-20 s. Varied so it // reads as pondering, not a loop: mostly thinking, an occasional glance. private thinkingTimer: ReturnType | null = null; private thinkingBeats = ["thinking", "attentive", "thinking", "confused", "thinking"]; startThinking(): void { if (!this.active || this.thinkingTimer) return; let i = 0; this.emote("thinking"); this.thinkingTimer = setInterval(() => { i = (i + 1) % this.thinkingBeats.length; this.emote(this.thinkingBeats[i]); }, 2600); } stopThinking(): void { if (this.thinkingTimer) { clearInterval(this.thinkingTimer); this.thinkingTimer = null; } } private emoteOffset(now: number): EmoteOffset { if (!this.emoteSpec) return { roll: 0, pitch: 0, yaw: 0, ant: 0 }; const elapsed = (now - this.emoteStart) / 1000; if (elapsed >= this.emoteSpec.dur) { this.emoteSpec = null; return { roll: 0, pitch: 0, yaw: 0, ant: 0 }; } // Bell envelope (0→1→0) so accents rise and settle without a snap. const env = Math.sin(Math.PI * (elapsed / this.emoteSpec.dur)); return this.emoteSpec.shape(elapsed, env, this.emoteSpec.dur); } private startMotion(): void { if (this.motionRaf !== null) return; this.env = 0; this.ant = 0; this.lastActivity = performance.now(); this.emoteSpec = null; const loop = (now: number) => { this.motionRaf = requestAnimationFrame(loop); if (now - this.lastCmd < 50) return; // ~20 Hz this.lastCmd = now; const reachy = this.opts.getReachy(); if (!reachy) return; const t = now / 1000; const level = Math.min(ttsLevel(), 1); // Envelope: while speaking, never below 0.35 so motion is clearly visible // even in quiet passages; decays to 0 when not speaking. const target = this.speaking ? Math.max(level, 0.35) : 0; this.env += (target - this.env) * 0.15; const e = this.env; // Antennas ease toward a raised, slowly swaying pose while speaking. const antennaTarget = this.speaking ? 18 + 7 * Math.sin(t * 1.4) : 0; this.ant += (antennaTarget - this.ant) * 0.08; if (this.speaking) this.lastActivity = now; // Idle policy: after a stretch of silence while listening, throw in a // small "attentive" glance so the robot never looks frozen. Light-touch // version of the official app's weighted idle tool selection. if (!this.speaking && !this.emoteSpec && now - this.lastActivity > 7000) { this.emote(Math.random() < 0.5 ? "attentive" : "thinking"); this.lastActivity = now; } const acc = this.emoteOffset(now); let ok = true; try { if (e > 0.03) { ok = reachy.setHeadRpyDeg( acc.roll + e * 4 * Math.sin(t * 3.1), acc.pitch + e * 9 * Math.sin(t * 2.3) + 1.5 * Math.sin(t * 0.8), acc.yaw + e * 7 * Math.sin(t * 1.7), ); } else { // Listening: slow, attentive sway. ok = reachy.setHeadRpyDeg( acc.roll + 1.5 * Math.sin(t * 0.7), acc.pitch + 2.5 * Math.sin(t * 0.9), acc.yaw + 4 * Math.sin(t * 0.45), ); } reachy.setAntennasDeg(this.ant + acc.ant, this.ant + acc.ant); } catch (err) { ok = false; if (now - this.lastLog > 3000) console.warn("[motion] send failed", err); } if (now - this.lastLog > 3000) { this.lastLog = now; console.log( `[motion] speaking=${this.speaking} level=${level.toFixed(2)} env=${e.toFixed(2)} ant=${this.ant.toFixed(1)} ok=${ok}`, ); } }; this.motionRaf = requestAnimationFrame(loop); } private stopMotion(): void { if (this.motionRaf !== null) { cancelAnimationFrame(this.motionRaf); this.motionRaf = null; } this.env = 0; this.ant = 0; try { const reachy = this.opts.getReachy(); reachy?.setHeadRpyDeg(0, 0, 0); reachy?.setAntennasDeg(0, 0); } catch { /* channel closed */ } } }