Spaces:

townbox
/

ava

Running

ava / worklets /audio-playback.js

Remove repeat guard, restore conservative speech detection, drop persona name line

8545065 verified 4 days ago

4.4 kB

// @ts-check
/**
 * AudioWorkletProcessor that plays streamed Float32 mono speech at the model's
 * rate (24 kHz), interpolated to the AudioContext rate. Adapted from the
 * speech-to-speech playback worklet: a queue with clean fades, plus a start
 * threshold so a GPU slower than realtime builds a cushion before she talks.
 *
 *   main -> worklet:
 *     { kind: "config", inputRate }
 *     { kind: "audio", samples }     (transferable) per chunk
 *     { kind: "lead", samples }      queue this much before starting a turn
 *     { kind: "flush" }              no more audio is coming: start regardless
 *     { kind: "clear" }              wipe the queue (barge-in)
 *
 *   worklet -> main:
 *     { kind: "progress", played, queued }   input samples, every ~50 ms
 *     { kind: "drained", played }            the queue ran dry
 */

const FADE_FRAMES = 32;
const PROGRESS_INTERVAL_SEC = 0.05;

class AudioPlaybackProcessor extends AudioWorkletProcessor {
  constructor() {
    super();
    this._inputRate = 24000;
    this._stepRatio = this._inputRate / sampleRate;
    this._queue = [];
    this._readIdx = 0;
    this._fracPos = 0;
    this._queued = 0;
    this._played = 0;
    this._lead = 0;
    this._flushed = false;
    this._playing = false;
    this._fadeIn = 0;
    this._fadeOut = 0;
    this._lastSample = 0;
    this._sinceProgress = 0;

    this.port.onmessage = (e) => {
      const data = e.data;
      switch (data?.kind) {
        case "config":
          this._inputRate = data.inputRate;
          this._stepRatio = this._inputRate / sampleRate;
          break;
        case "lead":
          this._lead = data.samples;
          this._flushed = false;
          break;
        case "flush":
          this._flushed = true;
          break;
        case "audio":
          this._queue.push(data.samples);
          this._queued += data.samples.length;
          break;
        case "clear":
          this._queue.length = 0;
          this._queued = 0;
          this._readIdx = 0;
          this._fracPos = 0;
          this._played = 0;
          this._flushed = false;
          if (this._playing) this._fadeOut = FADE_FRAMES;
          break;
      }
    };
  }

  _read() {
    const head = this._queue[0];
    const a = head[this._readIdx];
    let b = a;
    if (this._readIdx + 1 < head.length) b = head[this._readIdx + 1];
    else if (this._queue.length > 1) b = this._queue[1][0];
    return a + (b - a) * this._fracPos;
  }

  _advance() {
    this._fracPos += this._stepRatio;
    while (this._fracPos >= 1) {
      this._fracPos -= 1;
      this._readIdx += 1;
      this._queued -= 1;
      this._played += 1;
    }
    while (this._queue.length > 0 && this._readIdx >= this._queue[0].length) {
      this._readIdx -= this._queue[0].length;
      this._queue.shift();
    }
  }

  process(_, outputs) {
    const channels = outputs[0];
    if (!channels || channels.length === 0) return true;
    const out = channels[0];

    if (!this._playing && this._queued > 0 && (this._queued >= this._lead || this._flushed)) {
      this._playing = true;
      this._fadeIn = FADE_FRAMES;
      this._fadeOut = 0;
    }

    for (let i = 0; i < out.length; i++) {
      let sample = 0;
      if (this._playing) {
        if (this._queue.length === 0) {
          // Ran dry: settle from the last value so the cut can't click.
          sample = this._lastSample * (1 - 1 / FADE_FRAMES);
          this._lastSample = sample;
          if (Math.abs(sample) < 1e-4 || this._fadeOut > 0) {
            this._lastSample = 0;
            this._playing = false;
            this._fadeOut = 0;
            this.port.postMessage({ kind: "drained", played: this._played });
          }
        } else {
          sample = this._read();
          this._lastSample = sample;
          this._advance();
          if (this._fadeIn > 0) {
            sample *= 1 - this._fadeIn / FADE_FRAMES;
            this._fadeIn -= 1;
          }
        }
      }
      out[i] = sample;
    }
    for (let c = 1; c < channels.length; c++) channels[c].set(out);

    this._sinceProgress += out.length;
    if (this._sinceProgress >= PROGRESS_INTERVAL_SEC * sampleRate) {
      this._sinceProgress = 0;
      this.port.postMessage({ kind: "progress", played: this._played, queued: this._queued });
    }
    return true;
  }
}

registerProcessor("audio-playback", AudioPlaybackProcessor);

	// @ts-check
	/**
	* AudioWorkletProcessor that plays streamed Float32 mono speech at the model's
	* rate (24 kHz), interpolated to the AudioContext rate. Adapted from the
	* speech-to-speech playback worklet: a queue with clean fades, plus a start
	* threshold so a GPU slower than realtime builds a cushion before she talks.
	*
	* main -> worklet:
	* { kind: "config", inputRate }
	* { kind: "audio", samples } (transferable) per chunk
	* { kind: "lead", samples } queue this much before starting a turn
	* { kind: "flush" } no more audio is coming: start regardless
	* { kind: "clear" } wipe the queue (barge-in)
	*
	* worklet -> main:
	* { kind: "progress", played, queued } input samples, every ~50 ms
	* { kind: "drained", played } the queue ran dry
	*/

	const FADE_FRAMES = 32;
	const PROGRESS_INTERVAL_SEC = 0.05;

	class AudioPlaybackProcessor extends AudioWorkletProcessor {
	constructor() {
	super();
	this._inputRate = 24000;
	this._stepRatio = this._inputRate / sampleRate;
	this._queue = [];
	this._readIdx = 0;
	this._fracPos = 0;
	this._queued = 0;
	this._played = 0;
	this._lead = 0;
	this._flushed = false;
	this._playing = false;
	this._fadeIn = 0;
	this._fadeOut = 0;
	this._lastSample = 0;
	this._sinceProgress = 0;

	this.port.onmessage = (e) => {
	const data = e.data;
	switch (data?.kind) {
	case "config":
	this._inputRate = data.inputRate;
	this._stepRatio = this._inputRate / sampleRate;
	break;
	case "lead":
	this._lead = data.samples;
	this._flushed = false;
	break;
	case "flush":
	this._flushed = true;
	break;
	case "audio":
	this._queue.push(data.samples);
	this._queued += data.samples.length;
	break;
	case "clear":
	this._queue.length = 0;
	this._queued = 0;
	this._readIdx = 0;
	this._fracPos = 0;
	this._played = 0;
	this._flushed = false;
	if (this._playing) this._fadeOut = FADE_FRAMES;
	break;
	}
	};
	}

	_read() {
	const head = this._queue[0];
	const a = head[this._readIdx];
	let b = a;
	if (this._readIdx + 1 < head.length) b = head[this._readIdx + 1];
	else if (this._queue.length > 1) b = this._queue[1][0];
	return a + (b - a) * this._fracPos;
	}

	_advance() {
	this._fracPos += this._stepRatio;
	while (this._fracPos >= 1) {
	this._fracPos -= 1;
	this._readIdx += 1;
	this._queued -= 1;
	this._played += 1;
	}
	while (this._queue.length > 0 && this._readIdx >= this._queue[0].length) {
	this._readIdx -= this._queue[0].length;
	this._queue.shift();
	}
	}

	process(_, outputs) {
	const channels = outputs[0];
	if (!channels \|\| channels.length === 0) return true;
	const out = channels[0];

	if (!this._playing && this._queued > 0 && (this._queued >= this._lead \|\| this._flushed)) {
	this._playing = true;
	this._fadeIn = FADE_FRAMES;
	this._fadeOut = 0;
	}

	for (let i = 0; i < out.length; i++) {
	let sample = 0;
	if (this._playing) {
	if (this._queue.length === 0) {
	// Ran dry: settle from the last value so the cut can't click.
	sample = this._lastSample * (1 - 1 / FADE_FRAMES);
	this._lastSample = sample;
	if (Math.abs(sample) < 1e-4 \|\| this._fadeOut > 0) {
	this._lastSample = 0;
	this._playing = false;
	this._fadeOut = 0;
	this.port.postMessage({ kind: "drained", played: this._played });
	}
	} else {
	sample = this._read();
	this._lastSample = sample;
	this._advance();
	if (this._fadeIn > 0) {
	sample *= 1 - this._fadeIn / FADE_FRAMES;
	this._fadeIn -= 1;
	}
	}
	}
	out[i] = sample;
	}
	for (let c = 1; c < channels.length; c++) channels[c].set(out);

	this._sinceProgress += out.length;
	if (this._sinceProgress >= PROGRESS_INTERVAL_SEC * sampleRate) {
	this._sinceProgress = 0;
	this.port.postMessage({ kind: "progress", played: this._played, queued: this._queued });
	}
	return true;
	}
	}

	registerProcessor("audio-playback", AudioPlaybackProcessor);