Spaces:
Running
Running
File size: 37,160 Bytes
38fccac aa333cd 38fccac aa333cd 38fccac aa333cd 38fccac aa333cd 38fccac aa333cd 38fccac aa333cd 38fccac aa333cd 38fccac aa333cd 38fccac aa333cd 38fccac aa333cd 38fccac aa333cd 38fccac aa333cd 38fccac aa333cd 38fccac aa333cd 38fccac aa333cd 38fccac aa333cd 38fccac aa333cd 38fccac | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 | """Speech-only WebSocket. Audio in, transcript or Piper audio out. No language model.
Protocol ``sonic-speech-stream-v1``. Everything a v1 client sends and reads is
unchanged; what a session negotiated through ``stream_options`` (see
``speech_stream.store.OPTION_VALUES``) is additive and off unless asked for.
HV3 hardening, and the failure each part prevents:
* **Backpressure counted what was SENT, not what was waiting.** ``queued_ms``
grew by one frame per frame and was never reduced, so the limit was a cap on
the length of a reply: with the default 2000 ms, every Piper answer longer
than two seconds died mid-sentence with "outbound audio queue is full". The
limit now bounds audio *ahead of playback* β wall-clock pacing by default,
the client's own ``playback.ack`` when it negotiated ``flow_control: ack`` β
and the server waits when it is ahead instead of ending a healthy turn.
* **One slot for transcription and speech.** A user who spoke while she was
answering had the utterance refused ("a speech turn is already running") in
``duplex_speech``. Recognition and synthesis now have their own slots.
* **A cancel left the cancelled turn holding the slot** until its Piper thread
noticed, so the ``tts.speak`` that follows a barge-in was refused. A cancel
now releases the slot at once; the old thread finishes unobserved.
* **Input was not checked against what the server decodes.** A 48 kHz frame
was accepted and transcribed as 16 kHz (three times slow, garbage text), and
an odd byte count crashed the PCM decoder. Both are refused with a reason.
"""
from __future__ import annotations
import asyncio
import contextlib
import json
import threading
import time
from collections import deque
from collections.abc import Awaitable, Callable, Iterator
from dataclasses import dataclass
from uuid import uuid4
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
from starlette.websockets import WebSocketDisconnected
from sonicagent_rt.audio.pcm import pcm16_to_floats
from sonicagent_rt.auth.service_token import enforce_origin
from sonicagent_rt.config import Settings
from sonicagent_rt.pipeline.cancellation import CancellationToken
from sonicagent_rt.protocol.binary_frame import BinaryAudioFrame, BinaryFrameError, decode_frame, encode_frame
from sonicagent_rt.protocol.constants import FLAG_END_OF_UTTERANCE, Codec, MessageType
from sonicagent_rt.protocol.sequence import SequenceError, SequenceValidator
from sonicagent_rt.speech_stream.chunks import iter_pcm16, split_frames
from sonicagent_rt.speech_stream.store import MAX_TTS_QUEUE, SpeechStreamSession
from sonicagent_rt.stt.errors import Cancelled, SpeechError
from sonicagent_rt.tts.models import validate_voice_id, voice_record
from sonicagent_rt.vad.factory import create_endpoint
router = APIRouter()
_MAX_FRAME_BYTES = 64 * 1024
#: Longest single input frame accepted. The contract is 20β40 ms; 200 ms leaves
#: room for a client that batches under load without letting one frame carry a
#: whole utterance past the per-utterance limit.
_MAX_FRAME_MS = 200
#: Transcriptions allowed to wait behind the running one (client endpointing).
_MAX_STT_PENDING = 2
#: With ``flow_control: ack``, how long the server waits for playback to move
#: before it concludes the client has stopped reading.
_ACK_STALL_SEC = 10.0
_PACE_STEP_SEC = 0.02
Emit = Callable[[dict[str, object]], Awaitable[None]]
EmitBytes = Callable[[bytes], Awaitable[None]]
@router.websocket("/audio/stream/{session_id}")
async def speech_stream(websocket: WebSocket, session_id: str) -> None:
settings: Settings = websocket.app.state.settings
token = websocket.query_params.get("resume_token", "")
try:
if not settings.voice_enabled:
raise SpeechError("voice endpoints are disabled")
enforce_origin(websocket.headers.get("origin"), settings)
session = websocket.app.state.speech_streams.acquire(session_id, token)
except SpeechError:
await websocket.close(code=1008)
return
await websocket.accept()
opened = time.monotonic()
send_lock = asyncio.Lock()
async def emit(payload: dict[str, object]) -> None:
payload.setdefault("t_ms", int((time.monotonic() - opened) * 1000))
async with send_lock:
await websocket.send_json(payload)
async def emit_bytes(payload: bytes) -> None:
async with send_lock:
await websocket.send_bytes(payload)
await emit(
{
"type": "session.ready",
"protocol": "sonic-speech-stream-v1",
"session_id": session.session_id,
"mode": session.mode,
"llm": False,
"partial_results": False,
"options": session.options.as_dict(),
}
)
stream = _Stream(websocket, session, settings, emit, emit_bytes)
try:
while True:
message = await websocket.receive()
if message.get("type") == "websocket.disconnect":
break
if message.get("text"):
if await stream.on_text(message["text"]):
await websocket.close(code=1000)
break
elif message.get("bytes") is not None:
await stream.on_audio(message["bytes"])
except (WebSocketDisconnect, WebSocketDisconnected, RuntimeError):
pass
finally:
await stream.close()
websocket.app.state.speech_streams.release(session_id)
_Chunk = tuple[int, bytes, float] # sample rate, PCM16, seconds spent synthesizing it
def _pull(iterator: Iterator[tuple[int, bytes]]) -> _Chunk | None:
started = time.perf_counter()
try:
rate, pcm = next(iterator)
except StopIteration:
return None
return rate, pcm, time.perf_counter() - started
def _pull_ahead(iterator: Iterator[tuple[int, bytes]]) -> asyncio.Future[_Chunk | None]:
"""Synthesize the iterator's next sentence in a worker thread, starting now."""
future = asyncio.ensure_future(asyncio.to_thread(_pull, iterator))
future.add_done_callback(_retrieved)
return future
def _retrieved(future: asyncio.Future[_Chunk | None]) -> None:
# A lookahead abandoned by a cancel may finish with ``Cancelled``; nobody awaits it.
if not future.cancelled():
future.exception()
@dataclass
class _Lookahead:
turn_id: str
iterator: Iterator[tuple[int, bytes]]
first: asyncio.Future[_Chunk | None]
stop: threading.Event
class _Slot:
"""One kind of work (recognition or synthesis): its task and its cancellation."""
def __init__(self) -> None:
self.task: asyncio.Task[None] | None = None
self.cancellation = CancellationToken()
self.stop = threading.Event()
def busy(self) -> bool:
return self.task is not None and not self.task.done()
def fresh(self) -> tuple[CancellationToken, threading.Event]:
self.cancellation = CancellationToken()
self.stop = threading.Event()
return self.cancellation, self.stop
def cancel(self) -> None:
"""Stop the running work and free the slot NOW.
The worker thread (Piper, faster-whisper) is told through ``stop`` and
finishes on its own; the asyncio task is cancelled so it emits nothing
more, and the slot is free for the next request in the same tick.
"""
self.stop.set()
self.cancellation.cancel()
if self.task is not None and not self.task.done():
self.task.cancel()
self.task = None
async def drain(self) -> None:
task = self.task
self.cancel()
if task is not None:
with contextlib.suppress(asyncio.CancelledError, Exception):
await task
class _Stream:
def __init__(
self,
websocket: WebSocket,
session: SpeechStreamSession,
settings: Settings,
emit: Emit,
emit_bytes: EmitBytes,
) -> None:
self.websocket = websocket
self.session = session
self.settings = settings
self.options = session.options
self.emit = emit
self.emit_bytes = emit_bytes
self.endpoint = create_endpoint(settings)
self.sequences = SequenceValidator()
self.started_audio = False
self.stt = _Slot()
self.tts = _Slot()
self.stt_pending: deque[tuple[list[float], int | None]] = deque()
self.tts_pending: deque[tuple[str, str, str]] = deque()
# Client endpointing: the utterance being held until its commit.
self.utterance: list[float] = []
self.utterance_speech = False
self.max_utterance_samples = int(
min(settings.vad_max_utterance_sec, settings.max_audio_seconds) * settings.input_sample_rate
)
# Audio sent and not yet played, for the WHOLE session β not per phrase. With the queue,
# phrases follow each other into one playback buffer on the client, so a per-phrase
# budget let a headset hold ~2 s of unplayed audio *per queued phrase* (measured end to
# end: 6 s of speech delivered in 2.2 s against a 2 s budget).
# - flow_control none: a virtual playout clock β when the last sent sample will have
# played if the client plays continuously from now on.
# - flow_control ack: per phrase [sent_ms, played_ms], summed; a phrase leaves the
# table once the client reports it fully played.
self.playout_end = 0.0
self.unplayed: dict[str, list[float]] = {}
self.last_ack_at: float | None = None
self.current_tts: str | None = None
# Lookahead: the next queued phrase, its synthesis already started while the current
# phrase's last frames are still being paced out. See ``_Lookahead``.
self.lookahead: _Lookahead | None = None
# ββ Control events βββββββββββββββββββββββββββββββββββββββββββββββ
async def on_text(self, raw: str) -> bool:
try:
event = json.loads(raw)
except json.JSONDecodeError:
await self.emit(
{"type": "session.error", "code": "invalid_request", "message": "control event is not JSON"}
)
return False
if not isinstance(event, dict):
await self.emit(
{
"type": "session.error",
"code": "invalid_request",
"message": "control event is not an object",
}
)
return False
kind = event.get("type")
if kind == "ping":
await self.emit({"type": "pong"})
elif kind == "turn.cancel":
await self._on_cancel(event)
elif kind == "session.close":
self.stt.cancel()
self.tts.cancel()
self.stt_pending.clear()
self.tts_pending.clear()
self._drop_lookahead()
await self.emit({"type": "session.closed"})
return True
elif kind == "tts.speak":
await self._on_speak(event)
elif kind == "playback.ack":
self._on_ack(event)
elif kind == "tts.warm":
await self._on_warm(event)
else:
await self.emit(
{"type": "session.error", "code": "invalid_request", "message": "unsupported control event"}
)
return False
async def _on_cancel(self, event: dict[str, object]) -> None:
# v1 had one slot, so a cancel stopped whatever was running. ``target``
# narrows it; without one both stop, which is the v1 behaviour.
target = str(event.get("target") or "all").lower()
if target not in ("all", "stt", "tts"):
await self.emit(
{
"type": "session.error",
"code": "invalid_request",
"message": "target must be all, stt or tts",
}
)
return
if target in ("all", "tts"):
self.tts.cancel()
self.tts_pending.clear()
self._drop_lookahead()
self._forget_playback()
if target in ("all", "stt"):
self.stt.cancel()
self.stt_pending.clear()
self.utterance = []
self.utterance_speech = False
with contextlib.suppress(Exception):
self.endpoint.flush()
await self.emit(
{
"type": "turn.cancelled",
"turn_id": event.get("turn_id"),
"target": target,
"clear_audio": target in ("all", "tts"),
}
)
async def _on_speak(self, event: dict[str, object]) -> None:
if self.session.mode == "stt_only":
await self.emit(
{"type": "session.error", "code": "invalid_request", "message": "this session does not speak"}
)
return
text = str(event.get("text") or "").strip()
turn_id = str(event.get("turn_id") or f"turn_{uuid4().hex}")
if not text:
await self.emit(
{
"type": "session.error",
"turn_id": turn_id,
"code": "invalid_request",
"message": "text is empty",
}
)
return
# HV9. A phrase may name its own voice (a preview, or Companion <-> Secretary without a
# new session). Only a registry voice; never a silent substitute for one that is not there.
voice = self.session.voice_id
if event.get("voice_id"):
voice = str(event.get("voice_id")).strip()
try:
validate_voice_id(voice)
except SpeechError:
voice = ""
if not voice or voice_record(voice) is None:
await self.emit(
{
"type": "session.error",
"turn_id": turn_id,
"code": "voice_unavailable",
"message": "voice is not in the voice registry",
}
)
return
if self.tts.busy():
if self.options.tts_queue and len(self.tts_pending) < MAX_TTS_QUEUE:
self.tts_pending.append((text, turn_id, voice))
await self.emit({"type": "tts.queued", "turn_id": turn_id, "position": len(self.tts_pending)})
return
await self.emit(
{
"type": "session.error",
"turn_id": turn_id,
"code": "rate_limited",
"message": "a speech turn is already running",
}
)
return
self._start_speech(text, turn_id, voice)
def _start_speech(self, text: str, turn_id: str, voice: str) -> None:
cancellation, stop = self.tts.fresh()
self.tts.task = asyncio.create_task(self._speak_then_next(text, turn_id, voice, cancellation, stop))
async def _speak_then_next(
self,
text: str,
turn_id: str,
voice: str,
cancellation: CancellationToken,
stop: threading.Event,
ahead: _Lookahead | None = None,
) -> None:
await self._speak(text, turn_id, voice, cancellation, stop, ahead)
if cancellation.cancelled or stop.is_set() or not self.tts_pending:
return
next_text, next_turn, next_voice = self.tts_pending.popleft()
# Started from inside the finishing task: the slot is handed straight
# over, so no ``tts.speak`` arriving in between can jump the queue.
cancellation, stop = self.tts.fresh()
ahead = self.lookahead
if ahead is not None and ahead.turn_id != next_turn:
self._drop_lookahead()
ahead = None
if ahead is not None:
# Its synthesis began during the previous phrase: it keeps the stop event it was
# started with, which is now the slot's, so a cancel still reaches its thread.
self.lookahead = None
stop = self.tts.stop = ahead.stop
self.tts.task = asyncio.create_task(
self._speak_then_next(next_text, next_turn, next_voice, cancellation, stop, ahead)
)
async def _on_warm(self, event: dict[str, object]) -> None:
"""HV9. Load the selected voice's model before her first sentence needs it."""
voice = str(event.get("voice_id") or self.session.voice_id).strip()
try:
validate_voice_id(voice)
if voice_record(voice) is None:
raise SpeechError("voice is not in the voice registry")
warm = getattr(self.websocket.app.state.tts, "warm", None)
if not callable(warm):
raise SpeechError("this speech engine cannot preload voices")
result = await asyncio.to_thread(warm, voice)
except SpeechError as exc:
await self.emit({"type": "session.error", "code": "voice_unavailable", "message": exc.message})
return
await self.emit({"type": "tts.warmed", **result})
def _on_ack(self, event: dict[str, object]) -> None:
if self.options.flow_control != "ack":
return # v1 clients never send it; a stray one is harmless
turn_id = str(event.get("turn_id") or "")
try:
played = float(event.get("played_ms")) # type: ignore[arg-type]
except (TypeError, ValueError):
return
entry = self.unplayed.get(turn_id)
if entry is None:
return
if played >= entry[1]:
entry[1] = played
self.last_ack_at = time.monotonic()
if played > 0:
# Playback is sequential: a phrase that has started playing means every phrase
# sent before it has finished. Without this, a client that acknowledges only the
# phrase it is receiving leaves the previous phrase's tail in the budget forever.
for earlier in list(self.unplayed):
if earlier == turn_id:
break
self.unplayed.pop(earlier, None)
# Fully played and no longer being sent: it no longer holds any of the budget.
if entry[1] >= entry[0] and not (self.tts.busy() and self.current_tts == turn_id):
self.unplayed.pop(turn_id, None)
def _ahead_ms(self) -> float:
"""Audio handed to the client and not yet played, across every phrase."""
if self.options.flow_control == "ack":
return sum(max(0.0, sent - played) for sent, played in self.unplayed.values())
return max(0.0, (self.playout_end - time.monotonic()) * 1000.0)
def _note_sent(self, turn_id: str, ms: float) -> None:
if self.options.flow_control == "ack":
self.unplayed.setdefault(turn_id, [0.0, 0.0])[0] += ms
else:
self.playout_end = max(self.playout_end, time.monotonic()) + ms / 1000.0
def _forget_playback(self) -> None:
"""The client cleared its audio (a tts cancel): nothing it held is waiting any more."""
self.unplayed.clear()
self.playout_end = 0.0
# ββ Microphone audio βββββββββββββββββββββββββββββββββββββββββββββ
async def on_audio(self, payload: bytes) -> None:
if self.session.mode == "tts_only":
await self.emit(
{
"type": "session.error",
"code": "invalid_request",
"message": "this session does not accept audio",
}
)
return
if len(payload) > _MAX_FRAME_BYTES:
await self.emit(
{"type": "session.error", "code": "audio_too_large", "message": "audio frame is too large"}
)
return
try:
frame = decode_frame(payload)
if frame.message_type != int(MessageType.AUDIO_INPUT):
raise BinaryFrameError("client frames must be microphone audio")
if frame.sample_rate != self.settings.input_sample_rate:
raise BinaryFrameError(
f"unsupported sample rate {frame.sample_rate}: send mono PCM16 at "
f"{self.settings.input_sample_rate} Hz"
)
if len(frame.payload) % 2:
raise BinaryFrameError("PCM16 payload must be a whole number of 16-bit samples")
if len(frame.payload) // 2 > self.settings.input_sample_rate * _MAX_FRAME_MS // 1000:
raise BinaryFrameError(f"audio frame is longer than {_MAX_FRAME_MS} ms")
self.sequences.accept(frame.sequence)
samples = pcm16_to_floats(frame.payload)
except (BinaryFrameError, SequenceError, ValueError) as exc:
await self.emit({"type": "session.error", "code": "invalid_audio", "message": str(exc)})
return
if not self.started_audio:
self.started_audio = True
await self.emit({"type": "audio.input.started"})
committed = bool(frame.flags & FLAG_END_OF_UTTERANCE)
if self.options.endpointing == "client":
await self._client_endpointed(samples, committed, frame.sequence)
return
for event in self.endpoint.push(samples):
if event.kind == "started":
await self.emit({"type": "speech.started"})
elif event.kind == "utterance":
await self._transcribe(event.samples, None)
if committed:
await self.emit({"type": "audio.input.committed", "commit_seq": frame.sequence})
for event in self.endpoint.flush():
if event.kind == "utterance":
await self._transcribe(event.samples, frame.sequence)
async def _client_endpointed(self, samples: list[float], committed: bool, sequence: int) -> None:
# The VAD still runs, but only to say "speech.started" for the UI; the
# client's commit decides where the utterance ends.
for event in self.endpoint.push(samples):
if event.kind == "started" and not self.utterance_speech:
self.utterance_speech = True
await self.emit({"type": "speech.started"})
self.utterance.extend(samples)
if len(self.utterance) > self.max_utterance_samples:
self.utterance = []
self.utterance_speech = False
with contextlib.suppress(Exception):
self.endpoint.flush()
await self.emit(
{
"type": "session.error",
"code": "audio_too_long",
"commit_seq": None,
"message": "utterance is longer than the server accepts; it was discarded",
}
)
return
if not committed:
return
held, self.utterance = self.utterance, []
self.utterance_speech = False
with contextlib.suppress(Exception):
self.endpoint.flush()
await self.emit({"type": "audio.input.committed", "commit_seq": sequence})
await self._transcribe(held, sequence)
async def _transcribe(self, samples: list[float], commit_seq: int | None) -> None:
if self.stt.busy():
if self.options.endpointing == "client" and len(self.stt_pending) < _MAX_STT_PENDING:
self.stt_pending.append((samples, commit_seq))
return
await self.emit(
{
"type": "session.error",
"code": "rate_limited",
"commit_seq": commit_seq,
"message": "a speech turn is already running",
}
)
return
self._start_transcription(samples, commit_seq)
def _start_transcription(self, samples: list[float], commit_seq: int | None) -> None:
cancellation, stop = self.stt.fresh()
self.stt.task = asyncio.create_task(
self._transcribe_then_next(samples, commit_seq, cancellation, stop)
)
async def _transcribe_then_next(
self,
samples: list[float],
commit_seq: int | None,
cancellation: CancellationToken,
stop: threading.Event,
) -> None:
await self._finish_utterance(samples, commit_seq, cancellation, stop)
if cancellation.cancelled or stop.is_set() or not self.stt_pending:
return
next_samples, next_seq = self.stt_pending.popleft()
cancellation, stop = self.stt.fresh()
self.stt.task = asyncio.create_task(
self._transcribe_then_next(next_samples, next_seq, cancellation, stop)
)
async def _finish_utterance(
self,
samples: list[float],
commit_seq: int | None,
cancellation: CancellationToken,
stop: threading.Event,
) -> None:
turn_id = f"turn_{uuid4().hex}"
audio_ms = round(len(samples) * 1000 / self.settings.input_sample_rate)
tag: dict[str, object] = {} if commit_seq is None else {"commit_seq": commit_seq}
try:
await self.emit({"type": "speech.stopped", "turn_id": turn_id, **tag})
if cancellation.cancelled or stop.is_set():
return
if not samples:
# A commit with no audio still answers, so the client is never
# left waiting for a final that is not coming.
await self.emit(
{
"type": "transcript.final",
"turn_id": turn_id,
"text": "",
"language": self.session.language,
"is_final": True,
"no_speech": True,
"audio_ms": 0,
**tag,
}
)
return
started = time.perf_counter()
transcript = await asyncio.to_thread(
self.websocket.app.state.stt.transcribe,
samples,
sample_rate=self.settings.input_sample_rate,
language=self.session.language,
model=self.settings.stt_model,
request_id=turn_id,
cancel=stop,
)
if cancellation.cancelled or stop.is_set():
return
elapsed = round((time.perf_counter() - started) * 1000, 1)
await self.emit(
{
"type": "transcript.final",
"turn_id": turn_id,
"text": transcript.text,
"language": transcript.language,
"is_final": True,
"audio_ms": audio_ms,
**tag,
}
)
await self.emit(
{
"type": "metrics.turn",
"turn_id": turn_id,
"stt_ms": elapsed,
"audio_ms": audio_ms,
"partials": 0,
"llm": False,
**tag,
}
)
except Cancelled:
return
except asyncio.CancelledError:
raise
except SpeechError as exc:
await self.emit(
{"type": "session.error", "turn_id": turn_id, "code": exc.code, "message": exc.message, **tag}
)
except (WebSocketDisconnect, WebSocketDisconnected):
return
# ββ Synthesis ββββββββββββββββββββββββββββββββββββββββββββββββββββ
async def _speak(
self,
text: str,
turn_id: str,
voice: str,
cancellation: CancellationToken,
stop: threading.Event,
ahead: _Lookahead | None = None,
) -> None:
settings = self.settings
frame_ms = int(self.options.output_frame_ms)
limit_ms = settings.max_outbound_queue_ms
started = time.perf_counter()
first_ms: float | None = None
sequence = 0
sent_ms = 0.0
self.current_tts = turn_id
synth_s = 0.0
engine = self.websocket.app.state.tts
is_loaded = getattr(engine, "is_loaded", None)
cold = None
if callable(is_loaded):
with contextlib.suppress(Exception):
cold = not is_loaded(voice)
record = voice_record(voice)
await self.emit(
{
"type": "tts.started",
"turn_id": turn_id,
"voice_id": voice,
"role": record.role if record else None,
"cold": cold,
}
)
if ahead is not None:
iterator, pending = ahead.iterator, ahead.first
else:
iterator = iter(
iter_pcm16(engine, text, voice=voice, language=self.session.language, cancel=stop)
)
pending = _pull_ahead(iterator)
try:
while True:
if cancellation.cancelled or stop.is_set():
return
chunk = await pending
if chunk is None:
break
rate, pcm, took = chunk
synth_s += took
# Lookahead. Piper renders a whole sentence before the first sample of it exists,
# and pacing holds at most ``limit_ms`` of unplayed audio on the client. Pulling
# the next sentence only after this one's last frame was sent left the client
# ``limit_ms`` to cover a whole sentence's synthesis: a 6 s Italian sentence in a
# "high" voice on the 2-vCPU Space took longer, and the headset went silent
# mid-reply (measured: 261-627 ms gaps, Serena High). Synthesis of the next
# sentence now runs while this one is paced out; nothing is sent early, so the
# protocol, the budget and the size of a burst are unchanged.
pending = _pull_ahead(iterator)
if first_ms is None:
first_ms = round((time.perf_counter() - started) * 1000, 1)
await self.emit(
{
"type": "tts.audio.meta",
"turn_id": turn_id,
"sample_rate": rate,
"codec": "pcm_s16le",
"incremental": "sentence",
"frame_ms": frame_ms,
}
)
for piece in split_frames(pcm, rate, frame_ms):
if cancellation.cancelled or stop.is_set():
return
self._look_ahead(pending)
if limit_ms <= 0:
# A zero budget cannot hold even one frame: refuse, as v1 did.
await self._refuse_queue(turn_id, cancellation, stop, "outbound audio queue is full")
return
if not await self._wait_for_room(turn_id, limit_ms, cancellation, stop):
return
timestamp = sequence * frame_ms * 1000
if self.options.chunk_events == "all":
await self.emit(
{
"type": "tts.audio.chunk",
"turn_id": turn_id,
"sequence": sequence,
"sample_rate": rate,
"timestamp_us": timestamp,
"bytes": len(piece),
}
)
await self.emit_bytes(
encode_frame(
BinaryAudioFrame(
message_type=int(MessageType.AUDIO_OUTPUT),
codec=int(Codec.PCM16),
flags=0,
sample_rate=rate,
sequence=sequence,
timestamp_us=timestamp,
payload=piece,
)
)
)
piece_ms = len(piece) / 2 / rate * 1000.0
sent_ms += piece_ms
self._note_sent(turn_id, piece_ms)
sequence += 1
except Cancelled:
return
except SpeechError as exc:
await self.emit(
{"type": "session.error", "turn_id": turn_id, "code": exc.code, "message": exc.message}
)
return
except (WebSocketDisconnect, WebSocketDisconnected):
return
await self.emit(
{"type": "tts.audio.done", "turn_id": turn_id, "frames": sequence, "audio_ms": round(sent_ms)}
)
await self.emit(
{
"type": "metrics.turn",
"turn_id": turn_id,
"tts_first_ms": first_ms,
"frames": sequence,
"audio_ms": round(sent_ms),
# HV9. Per voice and profile: time in synthesis (a cold load included, on the
# first sentence), its real-time factor, and whether the model was cold.
"voice_id": voice,
"role": record.role if record else None,
"cold": cold,
"synth_ms": round(synth_s * 1000, 1),
"rtf": round(synth_s * 1000 / sent_ms, 3) if sent_ms else None,
"llm": False,
}
)
def _look_ahead(self, pending: asyncio.Future[_Chunk | None]) -> None:
"""This phrase has no sentence left: start the next queued phrase's synthesis now.
Phrases of a reply arrive as separate ``tts.speak`` events (one sentence each, as the
Avatar splits them), so the sentence lookahead alone stops at every phrase boundary.
The next phrase's first sentence is synthesized while this phrase's tail is paced out;
``_speak_then_next`` hands it over with the stop event it was started with.
"""
if self.lookahead is not None or not self.tts_pending or not pending.done():
return
if pending.cancelled() or pending.exception() is not None or pending.result() is not None:
return
text, turn_id, voice = self.tts_pending[0]
stop = threading.Event()
engine = self.websocket.app.state.tts
iterator = iter(iter_pcm16(engine, text, voice=voice, language=self.session.language, cancel=stop))
self.lookahead = _Lookahead(turn_id, iterator, _pull_ahead(iterator), stop)
def _drop_lookahead(self) -> None:
ahead, self.lookahead = self.lookahead, None
if ahead is not None:
ahead.stop.set() # its thread stops at the next sentence; nothing was sent
async def _wait_for_room(
self,
turn_id: str,
limit_ms: int,
cancellation: CancellationToken,
stop: threading.Event,
) -> bool:
"""Hold the next frame while the client has ``limit_ms`` or more unplayed.
Pacing, not refusing: the reply is healthy, the client is simply not
there yet. Only a client that negotiated acks and then stopped
acknowledging for ``_ACK_STALL_SEC`` is treated as gone.
"""
waiting_since = time.monotonic()
while self._ahead_ms() >= limit_ms:
if cancellation.cancelled or stop.is_set():
return False
if self.options.flow_control == "ack":
last = self.last_ack_at or waiting_since
if time.monotonic() - max(last, waiting_since) > _ACK_STALL_SEC:
await self._refuse_queue(
turn_id, cancellation, stop, "client stopped acknowledging playback"
)
return False
await asyncio.sleep(_PACE_STEP_SEC)
return True
async def _refuse_queue(
self, turn_id: str, cancellation: CancellationToken, stop: threading.Event, message: str
) -> None:
cancellation.cancel()
stop.set()
await self.emit(
{"type": "session.error", "turn_id": turn_id, "code": "rate_limited", "message": message}
)
async def close(self) -> None:
self.stt_pending.clear()
self.tts_pending.clear()
self._drop_lookahead()
await self.stt.drain()
await self.tts.drain()
|