Fix repetitive conversation: no model tools, new persona, repeat guard, one-word answers

README.md

@@ -29,12 +29,26 @@ you speak → Silero VAD (WASM) → LFM3-Omni-2B (WebGPU): audio in, interleaved
 - **Model** runs in a worker, so the avatar keeps rendering. It hears the audio
   directly: there is no separate transcription step, and so no user transcript.
 - **Barge-in**: talk over her and generation and playback stop immediately.
-- **The model plays the avatar** through `set_mood` and `make_hand_gesture` tools.
+- **Body language** (moods, thumbs up, shrug, a wave) is read from her words as they
+  are heard, in `src/reactions.js`.
 - **Lip-sync** is derived from the audio by HeadAudio, as in gemma-avatar.
 
 Differences from gemma-avatar, all forced by the model: there is no voice picker
-(LFM3-Omni's interleaved mode has one voice and ignores voice requests) and no
-`make_facial_expression` tool (it made the model loop on emoji calls without speaking).
+(LFM3-Omni's interleaved mode has one voice and ignores voice requests), and the
+model doesn't call avatar tools. Offering any tool schema derails its conversation:
+with the original Space's tools "let's play a quick game" starts a timer, and with
+avatar tools it gestures on every turn, gives shallow replies and can loop without
+speaking. The runtime, message format, generation mode and greedy decoding otherwise
+match the original Space.
+
+## Repetition
+
+LFM3-Omni-2B often re-asks its last question ("Red is a great choice. What's your
+favorite color?"), including with the original Space's exact prompt; rules, examples
+and per-turn notes in the prompt didn't stop it. `public/workers/repeat-guard.js`
+catches a repeated question or sentence in the text stream, which runs ahead of
+playback, then ends the reply just before it and keeps it out of the history. It can't
+cut a reply's first sentence, or audio already heard, and it can't make her lead better.
 
 ## Run
 

index.html

@@ -12,7 +12,7 @@
       rel="stylesheet"
     />
     
-  <link rel="stylesheet" crossorigin href="./chunk-sarxdv2b.css"><script type="module" crossorigin src="./chunk-v7x5vjjc.js"></script></head>
+  <link rel="stylesheet" crossorigin href="./chunk-sarxdv2b.css"><script type="module" crossorigin src="./chunk-qht2aa6q.js"></script></head>
   <body>
     <main id="app">
       <!-- The avatar is the hero: a full-viewport 3D stage. -->

workers/model-worker.js

@@ -2,13 +2,12 @@
 /**
  * LFM3-Omni on WebGPU, off the main thread so the avatar keeps its frame rate.
  * One conversation lives here: the history, the prefix-cached session, and the
- * tool loop. Everything the main thread needs comes back as a stream of events.
+ * context budget. Everything the main thread needs comes back as a stream of events.
  *
  * main -> worker:
- *   { kind: "load", model, tools, system }
+ *   { kind: "load", model, system }
  *   { kind: "respond", id, audio }   a user turn, 16 kHz Float32
  *   { kind: "abort", id }            barge-in: stop generating this turn
- *   { kind: "tool-result", callId, content }
  *   { kind: "system", system }       new persona / instructions
  *   { kind: "reset" }                forget the conversation
  *
@@ -16,7 +15,7 @@
  *   { kind: "progress", loaded?, total?, fraction?, status? }
  *   { kind: "ready", device, sampleRate }   { kind: "failed", name, message }
  *   { kind: "text", id, text }     { kind: "audio", id, samples }
- *   { kind: "tool", id, callId, name, args }
+ *   { kind: "truncate", id, samples, kept }   the reply ends at this audio sample
  *   { kind: "finished", id, reason, usage, generated, error? }
  */
 
@@ -25,11 +24,8 @@ import {
   ContextOverflowError,
   DeviceLostError,
 } from "../vendor/Lfm3OmniForConditionalGeneration.js";
+import { RepeatGuard } from "./repeat-guard.js";
 
-// A reply must end in speech: left to itself the model can keep calling tools
-// and never talk. After this many tool rounds it generates with no tools at all.
-const MAX_TOOL_ROUNDS = 2;
-const TOOL_DONE_NOTE = " Now say your reply out loud; don't call another tool.";
 const LOAD_ATTEMPTS = 3; // Chromium occasionally returns no adapter on a cold GPU process
 const LOAD_RETRY_MS = 1500;
 // Speech costs ~16 tokens/s each way and re-reading history after a trim takes
@@ -39,14 +35,12 @@ const HISTORY_BUDGET_TOKENS = Number(SEARCH.get("historyBudget")) || 3000;
 
 /** @type {any} */ let model = null;
 /** @type {any} */ let session = null;
-/** @type {any[]} */ let tools = [];
 let system = "";
 /** @type {any[]} */ let history = [];
 let idle = Promise.resolve();
+let guard = new RepeatGuard();
 /** @type {Map<number, AbortController>} */
 const running = new Map();
-/** @type {Map<string, (content: string) => void>} */
-const pendingTools = new Map();
 const simulateRtf = Number(SEARCH.get("simulateRtf")) || 0;
 
 const post = (msg, transfer = []) => self.postMessage(msg, { transfer });
@@ -69,8 +63,7 @@ function compactHistory() {
   for (let i = 0; i < Math.floor(exchanges / 2); i++) dropOldestExchange();
 }
 
-async function load({ model: modelId, tools: toolDefs, system: sys }) {
-  tools = toolDefs;
+async function load({ model: modelId, system: sys }) {
   system = sys;
   let lastErr;
   for (let attempt = 1; attempt <= LOAD_ATTEMPTS; attempt++) {
@@ -92,7 +85,7 @@ async function load({ model: modelId, tools: toolDefs, system: sys }) {
       });
       session = model.create_session();
       post({ kind: "progress", status: "Getting ready" });
-      await session.prime(messages(), { tools }).result;
+      await session.prime(messages()).result;
       post({ kind: "ready", device: model.deviceInfo(), sampleRate: model.audio_sampling_rate });
       return;
     } catch (err) {
@@ -107,21 +100,7 @@ async function load({ model: modelId, tools: toolDefs, system: sys }) {
   post({ kind: "failed", name: lastErr?.name ?? "Error", message: lastErr?.message ?? String(lastErr) });
 }
 
-/** @param {number} id @param {any} call @param {AbortSignal} signal */
-function runTool(id, call, signal) {
-  return new Promise((resolve) => {
-    const callId = call.id;
-    const done = (content) => {
-      pendingTools.delete(callId);
-      resolve(content);
-    };
-    pendingTools.set(callId, done);
-    signal.addEventListener("abort", () => done("Stopped."), { once: true });
-    post({ kind: "tool", id, callId, name: call.function.name, args: call.function.arguments });
-  });
-}
-
-/** One user turn: generate, run any tools, generate again until she's done. */
+/** One user turn. */
 async function respond({ id, audio }) {
   const controller = new AbortController();
   running.set(id, controller);
@@ -138,12 +117,10 @@ async function respond({ id, audio }) {
     signal.throwIfAborted();
     history.push({ role: "user", content: [{ type: "audio", audio }] });
 
-    for (let round = 0; round < MAX_TOOL_ROUNDS + 1; round++) {
-      const stream = session.generate(messages(), {
-        generation_mode: "interleaved",
-        ...(round < MAX_TOOL_ROUNDS ? { tools } : {}),
-        signal,
-      });
+    guard.beginReply();
+    /** @type {string | null} */ let keptOnRepeat = null;
+    for (;;) {
+      const stream = session.generate(messages(), { generation_mode: "interleaved", signal });
       let emitted = false;
       const started = performance.now();
       try {
@@ -151,6 +128,13 @@ async function respond({ id, audio }) {
           if (ev.type === "text") {
             emitted = true;
             post({ kind: "text", id, text: ev.text });
+            const repeat = guard.push(ev.text, generated);
+            if (repeat) {
+              keptOnRepeat = repeat.kept;
+              post({ kind: "truncate", id, samples: repeat.cutAt, kept: repeat.kept });
+              controller.abort();
+              break;
+            }
           } else if (ev.type === "audio") {
             emitted = true;
             generated += ev.audio.length;
@@ -163,26 +147,22 @@ async function respond({ id, audio }) {
           }
         }
       } catch (err) {
-        if (!emitted && err instanceof ContextOverflowError && dropOldestExchange()) {
-          round -= 1;
-          continue;
-        }
+        if (!emitted && err instanceof ContextOverflowError && dropOldestExchange()) continue;
         throw err;
       }
       const result = await stream.result;
       usage = result.usage;
-      reason = result.finish_reason;
-      history.push(result.message);
-      if (reason !== "tool_calls") break;
-
-      for (const call of result.message.tool_calls ?? []) {
-        const content = (call.errors?.length ? call.errors.join("; ") : await runTool(id, call, signal)) + TOOL_DONE_NOTE;
-        history.push({ role: "tool", tool_call_id: call.id, name: call.function.name, content });
-      }
-      if (signal.aborted) {
-        reason = "cancelled";
-        break;
+      if (keptOnRepeat !== null) {
+        // History holds what she said, not the repeat, or it would teach her to loop.
+        reason = "stop";
+        history.push({ role: "assistant", content: keptOnRepeat });
+        guard.endReply();
+      } else {
+        reason = result.finish_reason;
+        history.push(result.message);
+        if (reason === "stop") guard.endReply();
       }
+      break;
     }
   } catch (err) {
     if (err?.name === "AbortError" || signal.aborted) {
@@ -211,15 +191,13 @@ self.onmessage = ({ data }) => {
     case "abort":
       running.get(data.id)?.abort();
       break;
-    case "tool-result":
-      pendingTools.get(data.callId)?.(data.content);
-      break;
     case "system":
       system = data.system;
       break;
     case "reset":
       idle = idle.then(() => {
         history = [];
+        guard = new RepeatGuard();
       });
       break;
   }

workers/repeat-guard.js

@@ -0,0 +1,86 @@
+// @ts-check
+/**
+ * LFM3-Omni-2B tends to re-ask its own last question ("Red is a great choice.
+ * What's your favorite color?"), whatever the prompt says. Speech generates
+ * about 3x faster than it plays, so a repeat is usually visible in the text
+ * before it is heard. This spots it, and says where in the audio its sentence
+ * began so the reply can end cleanly just before it. Pure logic, for testing.
+ */
+
+const STOP_WORDS = new Set(
+  "a an and are about do does did for how i in is it me of on or s so that the to what what's whats which who would you your you're yours".split(" "),
+);
+const SIMILAR = 0.6;
+const MIN_REPEATED_STATEMENT_WORDS = 4;
+
+/** @param {string} sentence */
+function words(sentence) {
+  return sentence.toLowerCase().replace(/[’']/g, "'").match(/[a-z0-9']+/g) ?? [];
+}
+
+/** @param {string} sentence */
+function keywords(sentence) {
+  return new Set(words(sentence).filter((w) => !STOP_WORDS.has(w)));
+}
+
+/** @param {string} a @param {string} b */
+function similar(a, b) {
+  const x = keywords(a);
+  const y = keywords(b);
+  if (x.size === 0 || y.size === 0) return words(a).join(" ") === words(b).join(" ");
+  const shared = [...x].filter((w) => y.has(w)).length;
+  return shared / new Set([...x, ...y]).size >= SIMILAR;
+}
+
+export class RepeatGuard {
+  constructor() {
+    /** Everything she has already said in this conversation. @type {string[]} */
+    this.said = [];
+    this.beginReply();
+  }
+
+  beginReply() {
+    this._current = "";
+    this._currentStart = 0;
+    this._kept = "";
+    /** @type {string[]} */
+    this._sentences = [];
+  }
+
+  /**
+   * Feed one text delta, with how many audio samples existed when it arrived.
+   * @param {string} text @param {number} samples
+   * @returns {{ cutAt: number, kept: string } | null} set when a repeat was found
+   */
+  push(text, samples) {
+    for (let i = 0; i < text.length; i++) {
+      const ch = text[i];
+      if (!this._current.trim()) this._currentStart = samples;
+      this._current += ch;
+      const next = text[i + 1];
+      const closes = ch === "?" || ch === "!" || (ch === "." && (next === undefined || /\s/.test(next)));
+      if (!closes) continue;
+      const sentence = this._current.trim();
+      this._current = "";
+      if (this._isRepeat(sentence)) return { cutAt: this._currentStart, kept: this._kept.trim() };
+      this._kept += ` ${sentence}`;
+      this._sentences.push(sentence);
+    }
+    return null;
+  }
+
+  /** The reply is over: remember what was actually said. */
+  endReply() {
+    this.said.push(...this._sentences);
+    this.beginReply();
+  }
+
+  /** @param {string} sentence */
+  _isRepeat(sentence) {
+    // Never cut the only thing she has to say: silence is worse than a repeat.
+    if (this._sentences.length === 0) return false;
+    const isQuestion = sentence.endsWith("?");
+    if (!isQuestion && words(sentence).length < MIN_REPEATED_STATEMENT_WORDS) return false;
+    return this.said.some((s) => s.endsWith("?") === isQuestion && similar(s, sentence));
+  }
+}

workers/vad-segmenter.js

@@ -12,14 +12,14 @@ export const FRAME_SAMPLES = 512; // 32 ms at 16 kHz
 
 export const DEFAULTS = {
   startProb: 0.5,
-  startFrames: 4, // 128 ms
+  startFrames: 3, // 96 ms
   guardedStartProb: 0.75,
   guardedStartFrames: 9, // 288 ms
   endProb: 0.35,
   endFrames: 17, // 544 ms of quiet ends the turn
   trailFrames: 6, // keep 192 ms of the quiet tail
   prerollFrames: 12, // 384 ms before the trigger, so onsets aren't clipped
-  minSpeechFrames: 8, // 256 ms of actual speech, or it was a misfire
+  minSpeechFrames: 5, // 160 ms of speech, or it was a misfire: one-word answers count
   maxFrames: 938, // 30 s
 };
 

worklets/audio-playback.js

@@ -11,6 +11,7 @@
  *     { kind: "lead", samples }      queue this much before starting a turn
  *     { kind: "flush" }              no more audio is coming: start regardless
  *     { kind: "clear" }              wipe the queue (barge-in)
+ *     { kind: "truncate", samples }  end this reply once it has played this far
  *
  *   worklet -> main:
  *     { kind: "progress", played, queued }   input samples, every ~50 ms
@@ -31,6 +32,7 @@ class AudioPlaybackProcessor extends AudioWorkletProcessor {
     this._queued = 0;
     this._played = 0;
     this._lead = 0;
+    this._limit = Infinity;
     this._flushed = false;
     this._playing = false;
     this._fadeIn = 0;
@@ -56,7 +58,12 @@ class AudioPlaybackProcessor extends AudioWorkletProcessor {
           this._queue.push(data.samples);
           this._queued += data.samples.length;
           break;
+        case "truncate":
+          this._limit = data.samples;
+          this._flushed = true;
+          break;
         case "clear":
+          this._limit = Infinity;
           this._queue.length = 0;
           this._queued = 0;
           this._readIdx = 0;
@@ -105,6 +112,12 @@ class AudioPlaybackProcessor extends AudioWorkletProcessor {
 
     for (let i = 0; i < out.length; i++) {
       let sample = 0;
+      if (this._played >= this._limit && this._queue.length > 0) {
+        this._queue.length = 0;
+        this._queued = 0;
+        this._readIdx = 0;
+        this._limit = Infinity;
+      }
       if (this._playing) {
         if (this._queue.length === 0) {
           // Ran dry: settle from the last value so the cut can't click.