Spaces:

jimenezcarrero
/

cookAIware

Running

cookAIware / web /src /voice /controller.ts

Juan Jimenez Carrero

Claude Opus 4.8

feat: sync robot preferences to HF, thinking movement + wait/announce on plan

b5352be 4 months ago

14.6 kB

import type { ReachyMiniInstance } from "@pollen-robotics/reachy-mini-sdk";
import { RealtimeSession } from "./realtime";
import { getRobotMicTrack, getRobotSpeakerTrack, primeRemoteTrack, resumeAudio, setPhoneOutput, ttsLevel } from "./audio";
import { countVideoReceivers } from "./camera";
import type { ToolContext } from "./tools";
import type { RobotGestures } from "../robot/gestures";
import type { Lang } from "../i18n";

export type VoiceStatus = "idle" | "connecting" | "listening" | "speaking" | "error";

/** Additive head/antenna accent for an emotion, evaluated at `elapsed` seconds
 *  into a move of length `dur`. Returned degrees are ADDED on top of the base
 *  wobble in the 20 Hz loop — one writer, so no motor jitter. A bell envelope
 *  (rise + fall) blends every accent smoothly in and out of the ongoing motion.
 *  Mirrors the official app's `play_emotion` intents with procedural motion. */
interface EmoteOffset {
  roll: number;
  pitch: number;
  yaw: number;
  ant: number;
}

interface EmoteSpec {
  dur: number;
  shape: (t: number, env: number, dur: number) => EmoteOffset;
}

// pitch>0 looks down; antenna>0 perks up. Kept modest so accents read as
// body language, not spasms.
const EMOTES: Record<string, EmoteSpec> = {
  // happy / success / grateful: two brisk nods, antennas up.
  happy: { dur: 1.4, shape: (t, env) => ({ roll: 0, pitch: 13 * env * Math.sin(t * 9), yaw: 0, ant: 16 * env }) },
  // excited / dance: fast antenna flutter + a little body-side sway.
  excited: { dur: 1.8, shape: (t, env) => ({ roll: 3 * env * Math.sin(t * 5), pitch: -4 * env, yaw: 9 * env * Math.sin(t * 5.5), ant: 22 * env * (0.6 + 0.4 * Math.sin(t * 17)) }) },
  // thinking / uncertain: tilt away and glance up, held then released.
  thinking: { dur: 2.0, shape: (_t, env) => ({ roll: 15 * env, pitch: -7 * env, yaw: 6 * env, ant: 5 * env }) },
  // confused: tilt plus a small "huh?" head shake.
  confused: { dur: 1.6, shape: (t, env) => ({ roll: 12 * env, pitch: 2 * env, yaw: 7 * env * Math.sin(t * 6), ant: -6 * env }) },
  // sad / lonely: droop the head and let the antennas fall.
  sad: { dur: 2.0, shape: (_t, env) => ({ roll: 4 * env, pitch: 15 * env, yaw: 0, ant: -22 * env }) },
  // attentive / curious: perk up and lean in.
  attentive: { dur: 1.4, shape: (_t, env) => ({ roll: 0, pitch: -7 * env, yaw: 0, ant: 15 * env }) },
  // loving: slow warm tilt with a gentle nod.
  loving: { dur: 1.8, shape: (t, env) => ({ roll: 11 * env, pitch: 5 * env * Math.sin(t * 3), yaw: 0, ant: 10 * env }) },
  // angry / irritated: sharp quick shake, antennas pinned back.
  angry: { dur: 1.2, shape: (t, env) => ({ roll: 0, pitch: 0, yaw: 11 * env * Math.sin(t * 12), ant: -12 * env }) },
  // surprised: quick jerk up, antennas shoot up.
  surprised: { dur: 1.3, shape: (_t, env) => ({ roll: 0, pitch: -14 * env, yaw: 0, ant: 26 * env }) },
  // greeting / yes: a single clear nod.
  nod: { dur: 1.2, shape: (t, env) => ({ roll: 0, pitch: 12 * env * Math.sin(t * 5.2), yaw: 0, ant: 8 * env }) },
};

// Synonyms → canonical emote, so the model's free-text intent always lands.
const EMOTE_ALIASES: Record<string, string> = {
  laughing: "happy",
  laugh: "happy",
  joy: "happy",
  grateful: "happy",
  success: "happy",
  proud: "happy",
  dance: "excited",
  playful: "excited",
  celebrate: "excited",
  thoughtful: "thinking",
  uncertain: "thinking",
  wondering: "thinking",
  downcast: "sad",
  lonely: "sad",
  disappointed: "sad",
  curious: "attentive",
  interested: "attentive",
  listening: "attentive",
  love: "loving",
  affection: "loving",
  irritated: "angry",
  rage: "angry",
  annoyed: "angry",
  shocked: "surprised",
  wow: "surprised",
  yes: "nod",
  agree: "nod",
  greet: "nod",
  hello: "nod",
};

export function resolveEmote(name: string): string | null {
  const key = name.trim().toLowerCase().replace(/[^a-z]/g, "");
  if (key in EMOTES) return key;
  if (key in EMOTE_ALIASES) return EMOTE_ALIASES[key];
  return null;
}

/** Robot-only voice loop: Reachy's mic in, Reachy's speaker out, and smooth
 *  client-side head motion while it speaks or listens. There is deliberately
 *  no phone-mic/phone-speaker mode — the interaction is with the robot. */
export class VoiceController {
  private session: RealtimeSession | null = null;
  status: VoiceStatus = "idle";

  constructor(
    private opts: {
      getReachy: () => ReachyMiniInstance | null;
      gestures: RobotGestures | null;
      getApiKey: () => string | null;
      getLang: () => Lang;
      getName: () => string;
      getVoice: () => string;
      getActivationMode: () => "always" | "name";
      getAllowInterrupt: () => boolean;
      toolCtx: ToolContext;
      onStatus: (s: VoiceStatus) => void;
      onError: (message: string) => void;
      onTranscript: (role: "user" | "assistant", text: string) => void;
    },
  ) {}

  get active(): boolean {
    return this.session !== null;
  }

  private setStatus(s: VoiceStatus): void {
    this.status = s;
    this.opts.onStatus(s);
  }

  async toggle(): Promise<void> {
    if (this.active) {
      this.stop();
    } else {
      await this.start();
    }
  }

  async start(): Promise<void> {
    const apiKey = this.opts.getApiKey();
    if (!apiKey) {
      this.opts.onError("Add your OpenAI key in Settings to talk to Reachy.");
      return;
    }
    const reachy = this.opts.getReachy();
    if (!reachy) {
      this.opts.onError("Connect to your Reachy first.");
      return;
    }
    this.setStatus("connecting");
    await resumeAudio();

    // Robot ears: the mic track the robot streams to us over WebRTC.
    const micTrack = getRobotMicTrack(reachy);
    if (!micTrack) {
      console.warn("[voice] no live robot audio receiver track found");
      this.opts.onError("Reachy's microphone isn't available — try leaving and reconnecting.");
      this.setStatus("idle");
      return;
    }
    // Chrome: remote tracks feed silence into WebAudio unless also consumed by
    // a media element. Prime it before the mic pump reads it.
    primeRemoteTrack(micTrack);

    // Robot voice: replace the SDK's silent placeholder on the WebRTC audio
    // sender with our TTS bus (SDK 1.8 contract — it never calls getUserMedia).
    if (!this.injectSpeakerTrack(reachy)) {
      this.opts.onError("Couldn't reach Reachy's speaker — try leaving and reconnecting.");
      this.setStatus("idle");
      return;
    }
    setPhoneOutput(false); // the robot is the only voice
    console.log(`[voice] video receivers: ${countVideoReceivers(reachy)}`);

    this.opts.gestures?.cancel();
    try {
      reachy.setMicMuted(false);
      reachy.sendRaw({ type: "set_wobbling", enabled: false }); // we own motion
    } catch {
      /* older daemon */
    }
    this.startMotion();

    this.session = new RealtimeSession({
      apiKey,
      lang: this.opts.getLang(),
      robotName: this.opts.getName(),
      voice: this.opts.getVoice(),
      activationMode: this.opts.getActivationMode(),
      allowInterrupt: this.opts.getAllowInterrupt(),
      micTrack,
      // Give the model a body: play_emotion routes here and overlays an accent
      // on the live motion loop.
      toolCtx: { ...this.opts.toolCtx, emote: (name: string) => this.emote(name) },
      callbacks: {
        onError: (m) => {
          this.opts.onError(m);
          this.setStatus("error");
        },
        onUserText: (t) => this.opts.onTranscript("user", t),
        onAssistantText: (t) => this.opts.onTranscript("assistant", t),
        onSpeakingChange: (speaking) => {
          this.speaking = speaking;
          this.setStatus(speaking ? "speaking" : "listening");
        },
        onClosed: () => {
          if (this.session) this.stop();
        },
      },
    });
    this.session.start();
    this.setStatus("listening");
  }

  stop(): void {
    this.session?.close();
    this.session = null;
    this.speaking = false;
    this.stopThinking();
    this.stopMotion();
    const reachy = this.opts.getReachy();
    try {
      this.restoreSpeakerTrack();
      reachy?.setMicMuted(true);
    } catch {
      /* channel closed */
    }
    this.setStatus("idle");
  }

  // ── Robot-speaker audio injection (replaceTrack on the WebRTC audio sender) ──
  private audioSender: RTCRtpSender | null = null;
  private placeholderTrack: MediaStreamTrack | null = null;

  /** Returns true if our TTS track was attached to the robot's audio sender. */
  private injectSpeakerTrack(reachy: ReachyMiniInstance | null): boolean {
    const pc = reachy?._pc ?? null;
    const ttsTrack = getRobotSpeakerTrack();
    if (!pc || !ttsTrack) return false;
    const sender = pc.getSenders().find((s) => s.track && s.track.kind === "audio") ?? null;
    if (!sender) return false;
    this.audioSender = sender;
    this.placeholderTrack = sender.track; // the SDK's silent placeholder
    void sender.replaceTrack(ttsTrack);
    return true;
  }

  private restoreSpeakerTrack(): void {
    if (this.audioSender) {
      void this.audioSender.replaceTrack(this.placeholderTrack);
      this.audioSender = null;
      this.placeholderTrack = null;
    }
  }

  // ── Client-side motion (~20 Hz, reachy-stories style) ──────────────────────
  // Speaking: head wobble scaled by a smoothed speech envelope (with a floor so
  // speech is always visibly animated) + antennas eased up slowly.
  // Listening: slow attentive sway so the robot feels alive.
  // Antennas are never oscillated at audio rate — that was the buzz.
  private speaking = false;
  private motionRaf: number | null = null;
  private env = 0;
  private ant = 0;
  private lastCmd = 0;
  private lastLog = 0;
  // Active emotion accent (overlaid additively on the wobble).
  private emoteSpec: EmoteSpec | null = null;
  private emoteStart = 0;
  private lastActivity = 0;

  /** Trigger an emotion accent — called by the play_emotion tool and by
   *  contextual reactions. Safe to call anytime; ignored if the name is
   *  unknown. Overlays on the running motion loop, so it never fights it. */
  emote(name: string): void {
    const key = resolveEmote(name);
    if (!key) return;
    this.emoteSpec = EMOTES[key];
    this.emoteStart = performance.now();
    this.lastActivity = this.emoteStart;
  }

  // Deliberate "I'm thinking" body language while a slow tool (menu design)
  // runs — otherwise the robot sits silent and still for 10-20 s. Varied so it
  // reads as pondering, not a loop: mostly thinking, an occasional glance.
  private thinkingTimer: ReturnType<typeof setInterval> | null = null;
  private thinkingBeats = ["thinking", "attentive", "thinking", "confused", "thinking"];
  startThinking(): void {
    if (!this.active || this.thinkingTimer) return;
    let i = 0;
    this.emote("thinking");
    this.thinkingTimer = setInterval(() => {
      i = (i + 1) % this.thinkingBeats.length;
      this.emote(this.thinkingBeats[i]);
    }, 2600);
  }
  stopThinking(): void {
    if (this.thinkingTimer) {
      clearInterval(this.thinkingTimer);
      this.thinkingTimer = null;
    }
  }

  private emoteOffset(now: number): EmoteOffset {
    if (!this.emoteSpec) return { roll: 0, pitch: 0, yaw: 0, ant: 0 };
    const elapsed = (now - this.emoteStart) / 1000;
    if (elapsed >= this.emoteSpec.dur) {
      this.emoteSpec = null;
      return { roll: 0, pitch: 0, yaw: 0, ant: 0 };
    }
    // Bell envelope (0→1→0) so accents rise and settle without a snap.
    const env = Math.sin(Math.PI * (elapsed / this.emoteSpec.dur));
    return this.emoteSpec.shape(elapsed, env, this.emoteSpec.dur);
  }

  private startMotion(): void {
    if (this.motionRaf !== null) return;
    this.env = 0;
    this.ant = 0;
    this.lastActivity = performance.now();
    this.emoteSpec = null;
    const loop = (now: number) => {
      this.motionRaf = requestAnimationFrame(loop);
      if (now - this.lastCmd < 50) return; // ~20 Hz
      this.lastCmd = now;
      const reachy = this.opts.getReachy();
      if (!reachy) return;
      const t = now / 1000;
      const level = Math.min(ttsLevel(), 1);
      // Envelope: while speaking, never below 0.35 so motion is clearly visible
      // even in quiet passages; decays to 0 when not speaking.
      const target = this.speaking ? Math.max(level, 0.35) : 0;
      this.env += (target - this.env) * 0.15;
      const e = this.env;
      // Antennas ease toward a raised, slowly swaying pose while speaking.
      const antennaTarget = this.speaking ? 18 + 7 * Math.sin(t * 1.4) : 0;
      this.ant += (antennaTarget - this.ant) * 0.08;
      if (this.speaking) this.lastActivity = now;
      // Idle policy: after a stretch of silence while listening, throw in a
      // small "attentive" glance so the robot never looks frozen. Light-touch
      // version of the official app's weighted idle tool selection.
      if (!this.speaking && !this.emoteSpec && now - this.lastActivity > 7000) {
        this.emote(Math.random() < 0.5 ? "attentive" : "thinking");
        this.lastActivity = now;
      }
      const acc = this.emoteOffset(now);
      let ok = true;
      try {
        if (e > 0.03) {
          ok = reachy.setHeadRpyDeg(
            acc.roll + e * 4 * Math.sin(t * 3.1),
            acc.pitch + e * 9 * Math.sin(t * 2.3) + 1.5 * Math.sin(t * 0.8),
            acc.yaw + e * 7 * Math.sin(t * 1.7),
          );
        } else {
          // Listening: slow, attentive sway.
          ok = reachy.setHeadRpyDeg(
            acc.roll + 1.5 * Math.sin(t * 0.7),
            acc.pitch + 2.5 * Math.sin(t * 0.9),
            acc.yaw + 4 * Math.sin(t * 0.45),
          );
        }
        reachy.setAntennasDeg(this.ant + acc.ant, this.ant + acc.ant);
      } catch (err) {
        ok = false;
        if (now - this.lastLog > 3000) console.warn("[motion] send failed", err);
      }
      if (now - this.lastLog > 3000) {
        this.lastLog = now;
        console.log(
          `[motion] speaking=${this.speaking} level=${level.toFixed(2)} env=${e.toFixed(2)} ant=${this.ant.toFixed(1)} ok=${ok}`,
        );
      }
    };
    this.motionRaf = requestAnimationFrame(loop);
  }

  private stopMotion(): void {
    if (this.motionRaf !== null) {
      cancelAnimationFrame(this.motionRaf);
      this.motionRaf = null;
    }
    this.env = 0;
    this.ant = 0;
    try {
      const reachy = this.opts.getReachy();
      reachy?.setHeadRpyDeg(0, 0, 0);
      reachy?.setAntennasDeg(0, 0);
    } catch {
      /* channel closed */
    }
  }
}

	import type { ReachyMiniInstance } from "@pollen-robotics/reachy-mini-sdk";
	import { RealtimeSession } from "./realtime";
	import { getRobotMicTrack, getRobotSpeakerTrack, primeRemoteTrack, resumeAudio, setPhoneOutput, ttsLevel } from "./audio";
	import { countVideoReceivers } from "./camera";
	import type { ToolContext } from "./tools";
	import type { RobotGestures } from "../robot/gestures";
	import type { Lang } from "../i18n";

	export type VoiceStatus = "idle" \| "connecting" \| "listening" \| "speaking" \| "error";

	/** Additive head/antenna accent for an emotion, evaluated at `elapsed` seconds
	* into a move of length `dur`. Returned degrees are ADDED on top of the base
	* wobble in the 20 Hz loop — one writer, so no motor jitter. A bell envelope
	* (rise + fall) blends every accent smoothly in and out of the ongoing motion.
	* Mirrors the official app's `play_emotion` intents with procedural motion. */
	interface EmoteOffset {
	roll: number;
	pitch: number;
	yaw: number;
	ant: number;
	}

	interface EmoteSpec {
	dur: number;
	shape: (t: number, env: number, dur: number) => EmoteOffset;
	}

	// pitch>0 looks down; antenna>0 perks up. Kept modest so accents read as
	// body language, not spasms.
	const EMOTES: Record<string, EmoteSpec> = {
	// happy / success / grateful: two brisk nods, antennas up.
	happy: { dur: 1.4, shape: (t, env) => ({ roll: 0, pitch: 13 * env * Math.sin(t * 9), yaw: 0, ant: 16 * env }) },
	// excited / dance: fast antenna flutter + a little body-side sway.
	excited: { dur: 1.8, shape: (t, env) => ({ roll: 3 * env * Math.sin(t * 5), pitch: -4 * env, yaw: 9 * env * Math.sin(t * 5.5), ant: 22 * env * (0.6 + 0.4 * Math.sin(t * 17)) }) },
	// thinking / uncertain: tilt away and glance up, held then released.
	thinking: { dur: 2.0, shape: (_t, env) => ({ roll: 15 * env, pitch: -7 * env, yaw: 6 * env, ant: 5 * env }) },
	// confused: tilt plus a small "huh?" head shake.
	confused: { dur: 1.6, shape: (t, env) => ({ roll: 12 * env, pitch: 2 * env, yaw: 7 * env * Math.sin(t * 6), ant: -6 * env }) },
	// sad / lonely: droop the head and let the antennas fall.
	sad: { dur: 2.0, shape: (_t, env) => ({ roll: 4 * env, pitch: 15 * env, yaw: 0, ant: -22 * env }) },
	// attentive / curious: perk up and lean in.
	attentive: { dur: 1.4, shape: (_t, env) => ({ roll: 0, pitch: -7 * env, yaw: 0, ant: 15 * env }) },
	// loving: slow warm tilt with a gentle nod.
	loving: { dur: 1.8, shape: (t, env) => ({ roll: 11 * env, pitch: 5 * env * Math.sin(t * 3), yaw: 0, ant: 10 * env }) },
	// angry / irritated: sharp quick shake, antennas pinned back.
	angry: { dur: 1.2, shape: (t, env) => ({ roll: 0, pitch: 0, yaw: 11 * env * Math.sin(t * 12), ant: -12 * env }) },
	// surprised: quick jerk up, antennas shoot up.
	surprised: { dur: 1.3, shape: (_t, env) => ({ roll: 0, pitch: -14 * env, yaw: 0, ant: 26 * env }) },
	// greeting / yes: a single clear nod.
	nod: { dur: 1.2, shape: (t, env) => ({ roll: 0, pitch: 12 * env * Math.sin(t * 5.2), yaw: 0, ant: 8 * env }) },
	};

	// Synonyms → canonical emote, so the model's free-text intent always lands.
	const EMOTE_ALIASES: Record<string, string> = {
	laughing: "happy",
	laugh: "happy",
	joy: "happy",
	grateful: "happy",
	success: "happy",
	proud: "happy",
	dance: "excited",
	playful: "excited",
	celebrate: "excited",
	thoughtful: "thinking",
	uncertain: "thinking",
	wondering: "thinking",
	downcast: "sad",
	lonely: "sad",
	disappointed: "sad",
	curious: "attentive",
	interested: "attentive",
	listening: "attentive",
	love: "loving",
	affection: "loving",
	irritated: "angry",
	rage: "angry",
	annoyed: "angry",
	shocked: "surprised",
	wow: "surprised",
	yes: "nod",
	agree: "nod",
	greet: "nod",
	hello: "nod",
	};

	export function resolveEmote(name: string): string \| null {
	const key = name.trim().toLowerCase().replace(/[^a-z]/g, "");
	if (key in EMOTES) return key;
	if (key in EMOTE_ALIASES) return EMOTE_ALIASES[key];
	return null;
	}

	/** Robot-only voice loop: Reachy's mic in, Reachy's speaker out, and smooth
	* client-side head motion while it speaks or listens. There is deliberately
	* no phone-mic/phone-speaker mode — the interaction is with the robot. */
	export class VoiceController {
	private session: RealtimeSession \| null = null;
	status: VoiceStatus = "idle";

	constructor(
	private opts: {
	getReachy: () => ReachyMiniInstance \| null;
	gestures: RobotGestures \| null;
	getApiKey: () => string \| null;
	getLang: () => Lang;
	getName: () => string;
	getVoice: () => string;
	getActivationMode: () => "always" \| "name";
	getAllowInterrupt: () => boolean;
	toolCtx: ToolContext;
	onStatus: (s: VoiceStatus) => void;
	onError: (message: string) => void;
	onTranscript: (role: "user" \| "assistant", text: string) => void;
	},
	) {}

	get active(): boolean {
	return this.session !== null;
	}

	private setStatus(s: VoiceStatus): void {
	this.status = s;
	this.opts.onStatus(s);
	}

	async toggle(): Promise<void> {
	if (this.active) {
	this.stop();
	} else {
	await this.start();
	}
	}

	async start(): Promise<void> {
	const apiKey = this.opts.getApiKey();
	if (!apiKey) {
	this.opts.onError("Add your OpenAI key in Settings to talk to Reachy.");
	return;
	}
	const reachy = this.opts.getReachy();
	if (!reachy) {
	this.opts.onError("Connect to your Reachy first.");
	return;
	}
	this.setStatus("connecting");
	await resumeAudio();

	// Robot ears: the mic track the robot streams to us over WebRTC.
	const micTrack = getRobotMicTrack(reachy);
	if (!micTrack) {
	console.warn("[voice] no live robot audio receiver track found");
	this.opts.onError("Reachy's microphone isn't available — try leaving and reconnecting.");
	this.setStatus("idle");
	return;
	}
	// Chrome: remote tracks feed silence into WebAudio unless also consumed by
	// a media element. Prime it before the mic pump reads it.
	primeRemoteTrack(micTrack);

	// Robot voice: replace the SDK's silent placeholder on the WebRTC audio
	// sender with our TTS bus (SDK 1.8 contract — it never calls getUserMedia).
	if (!this.injectSpeakerTrack(reachy)) {
	this.opts.onError("Couldn't reach Reachy's speaker — try leaving and reconnecting.");
	this.setStatus("idle");
	return;
	}
	setPhoneOutput(false); // the robot is the only voice
	console.log(`[voice] video receivers: ${countVideoReceivers(reachy)}`);

	this.opts.gestures?.cancel();
	try {
	reachy.setMicMuted(false);
	reachy.sendRaw({ type: "set_wobbling", enabled: false }); // we own motion
	} catch {
	/* older daemon */
	}
	this.startMotion();

	this.session = new RealtimeSession({
	apiKey,
	lang: this.opts.getLang(),
	robotName: this.opts.getName(),
	voice: this.opts.getVoice(),
	activationMode: this.opts.getActivationMode(),
	allowInterrupt: this.opts.getAllowInterrupt(),
	micTrack,
	// Give the model a body: play_emotion routes here and overlays an accent
	// on the live motion loop.
	toolCtx: { ...this.opts.toolCtx, emote: (name: string) => this.emote(name) },
	callbacks: {
	onError: (m) => {
	this.opts.onError(m);
	this.setStatus("error");
	},
	onUserText: (t) => this.opts.onTranscript("user", t),
	onAssistantText: (t) => this.opts.onTranscript("assistant", t),
	onSpeakingChange: (speaking) => {
	this.speaking = speaking;
	this.setStatus(speaking ? "speaking" : "listening");
	},
	onClosed: () => {
	if (this.session) this.stop();
	},
	},
	});
	this.session.start();
	this.setStatus("listening");
	}

	stop(): void {
	this.session?.close();
	this.session = null;
	this.speaking = false;
	this.stopThinking();
	this.stopMotion();
	const reachy = this.opts.getReachy();
	try {
	this.restoreSpeakerTrack();
	reachy?.setMicMuted(true);
	} catch {
	/* channel closed */
	}
	this.setStatus("idle");
	}

	// ── Robot-speaker audio injection (replaceTrack on the WebRTC audio sender) ──
	private audioSender: RTCRtpSender \| null = null;
	private placeholderTrack: MediaStreamTrack \| null = null;

	/** Returns true if our TTS track was attached to the robot's audio sender. */
	private injectSpeakerTrack(reachy: ReachyMiniInstance \| null): boolean {
	const pc = reachy?._pc ?? null;
	const ttsTrack = getRobotSpeakerTrack();
	if (!pc \|\| !ttsTrack) return false;
	const sender = pc.getSenders().find((s) => s.track && s.track.kind === "audio") ?? null;
	if (!sender) return false;
	this.audioSender = sender;
	this.placeholderTrack = sender.track; // the SDK's silent placeholder
	void sender.replaceTrack(ttsTrack);
	return true;
	}

	private restoreSpeakerTrack(): void {
	if (this.audioSender) {
	void this.audioSender.replaceTrack(this.placeholderTrack);
	this.audioSender = null;
	this.placeholderTrack = null;
	}
	}

	// ── Client-side motion (~20 Hz, reachy-stories style) ──────────────────────
	// Speaking: head wobble scaled by a smoothed speech envelope (with a floor so
	// speech is always visibly animated) + antennas eased up slowly.
	// Listening: slow attentive sway so the robot feels alive.
	// Antennas are never oscillated at audio rate — that was the buzz.
	private speaking = false;
	private motionRaf: number \| null = null;
	private env = 0;
	private ant = 0;
	private lastCmd = 0;
	private lastLog = 0;
	// Active emotion accent (overlaid additively on the wobble).
	private emoteSpec: EmoteSpec \| null = null;
	private emoteStart = 0;
	private lastActivity = 0;

	/** Trigger an emotion accent — called by the play_emotion tool and by
	* contextual reactions. Safe to call anytime; ignored if the name is
	* unknown. Overlays on the running motion loop, so it never fights it. */
	emote(name: string): void {
	const key = resolveEmote(name);
	if (!key) return;
	this.emoteSpec = EMOTES[key];
	this.emoteStart = performance.now();
	this.lastActivity = this.emoteStart;
	}

	// Deliberate "I'm thinking" body language while a slow tool (menu design)
	// runs — otherwise the robot sits silent and still for 10-20 s. Varied so it
	// reads as pondering, not a loop: mostly thinking, an occasional glance.
	private thinkingTimer: ReturnType<typeof setInterval> \| null = null;
	private thinkingBeats = ["thinking", "attentive", "thinking", "confused", "thinking"];
	startThinking(): void {
	if (!this.active \|\| this.thinkingTimer) return;
	let i = 0;
	this.emote("thinking");
	this.thinkingTimer = setInterval(() => {
	i = (i + 1) % this.thinkingBeats.length;
	this.emote(this.thinkingBeats[i]);
	}, 2600);
	}
	stopThinking(): void {
	if (this.thinkingTimer) {
	clearInterval(this.thinkingTimer);
	this.thinkingTimer = null;
	}
	}

	private emoteOffset(now: number): EmoteOffset {
	if (!this.emoteSpec) return { roll: 0, pitch: 0, yaw: 0, ant: 0 };
	const elapsed = (now - this.emoteStart) / 1000;
	if (elapsed >= this.emoteSpec.dur) {
	this.emoteSpec = null;
	return { roll: 0, pitch: 0, yaw: 0, ant: 0 };
	}
	// Bell envelope (0→1→0) so accents rise and settle without a snap.
	const env = Math.sin(Math.PI * (elapsed / this.emoteSpec.dur));
	return this.emoteSpec.shape(elapsed, env, this.emoteSpec.dur);
	}

	private startMotion(): void {
	if (this.motionRaf !== null) return;
	this.env = 0;
	this.ant = 0;
	this.lastActivity = performance.now();
	this.emoteSpec = null;
	const loop = (now: number) => {
	this.motionRaf = requestAnimationFrame(loop);
	if (now - this.lastCmd < 50) return; // ~20 Hz
	this.lastCmd = now;
	const reachy = this.opts.getReachy();
	if (!reachy) return;
	const t = now / 1000;
	const level = Math.min(ttsLevel(), 1);
	// Envelope: while speaking, never below 0.35 so motion is clearly visible
	// even in quiet passages; decays to 0 when not speaking.
	const target = this.speaking ? Math.max(level, 0.35) : 0;
	this.env += (target - this.env) * 0.15;
	const e = this.env;
	// Antennas ease toward a raised, slowly swaying pose while speaking.
	const antennaTarget = this.speaking ? 18 + 7 * Math.sin(t * 1.4) : 0;
	this.ant += (antennaTarget - this.ant) * 0.08;
	if (this.speaking) this.lastActivity = now;
	// Idle policy: after a stretch of silence while listening, throw in a
	// small "attentive" glance so the robot never looks frozen. Light-touch
	// version of the official app's weighted idle tool selection.
	if (!this.speaking && !this.emoteSpec && now - this.lastActivity > 7000) {
	this.emote(Math.random() < 0.5 ? "attentive" : "thinking");
	this.lastActivity = now;
	}
	const acc = this.emoteOffset(now);
	let ok = true;
	try {
	if (e > 0.03) {
	ok = reachy.setHeadRpyDeg(
	acc.roll + e * 4 * Math.sin(t * 3.1),
	acc.pitch + e * 9 * Math.sin(t * 2.3) + 1.5 * Math.sin(t * 0.8),
	acc.yaw + e * 7 * Math.sin(t * 1.7),
	);
	} else {
	// Listening: slow, attentive sway.
	ok = reachy.setHeadRpyDeg(
	acc.roll + 1.5 * Math.sin(t * 0.7),
	acc.pitch + 2.5 * Math.sin(t * 0.9),
	acc.yaw + 4 * Math.sin(t * 0.45),
	);
	}
	reachy.setAntennasDeg(this.ant + acc.ant, this.ant + acc.ant);
	} catch (err) {
	ok = false;
	if (now - this.lastLog > 3000) console.warn("[motion] send failed", err);
	}
	if (now - this.lastLog > 3000) {
	this.lastLog = now;
	console.log(
	`[motion] speaking=${this.speaking} level=${level.toFixed(2)} env=${e.toFixed(2)} ant=${this.ant.toFixed(1)} ok=${ok}`,
	);
	}
	};
	this.motionRaf = requestAnimationFrame(loop);
	}

	private stopMotion(): void {
	if (this.motionRaf !== null) {
	cancelAnimationFrame(this.motionRaf);
	this.motionRaf = null;
	}
	this.env = 0;
	this.ant = 0;
	try {
	const reachy = this.opts.getReachy();
	reachy?.setHeadRpyDeg(0, 0, 0);
	reachy?.setAntennasDeg(0, 0);
	} catch {
	/* channel closed */
	}
	}
	}