Remove unused runtime source snapshot

runtime/src/speech_to_speech/LLM/README.md

@@ -1,139 +0,0 @@
-# LLM Summary
-
-## Available LLM backends (`--llm_backend`)
-
-Runtime-supported values in `s2s_pipeline.py`:
-
-- `transformers` → `language_model.py` (Transformers backend)
-- `mlx-lm` → `language_model.py` (MLX backend)
-- `responses-api` → `responses_api_language_model.py`
-
-## Usage
-
-### 1) Transformers (`--llm_backend transformers`)
-
-- Handler: `LanguageModelHandler`
-- Typical use: local GPU/CPU inference using Hugging Face Transformers
-- Backend-specific args prefix: `--llm_*`
-- Shared args (from base): `--model_name`, `--chat_size`, `--init_chat_prompt`, `--enable_lang_prompt`
-
-```bash
-python s2s_pipeline.py \
-  --llm_backend transformers \
-  --model_name Qwen/Qwen3-4B-Instruct-2507 \
-  --llm_device cuda \
-  --llm_torch_dtype float16 \
-  --llm_gen_max_new_tokens 128
-```
-
-Common options:
-- `--llm_gen_min_new_tokens`
-- `--llm_gen_temperature`
-- `--llm_gen_do_sample`
-- `--chat_size`
-- `--init_chat_prompt`
-
-### 2) MLX-LM (`--llm_backend mlx-lm`)
-
-- Handler: `LanguageModelHandler`
-- Typical use: Apple Silicon local inference
-- Backend-specific args prefix: same as Transformers (`--llm_*`)
-
-```bash
-python s2s_pipeline.py \
-  --llm_backend mlx-lm \
-  --model_name mlx-community/Qwen3-4B-Instruct-2507-bf16 \
-  --llm_device mps \
-  --llm_gen_max_new_tokens 128
-```
-
-Common options:
-- `--llm_gen_temperature`
-- `--llm_gen_do_sample`
-- `--chat_size`
-- `--init_chat_prompt`
-
-### 3) OpenAI-compatible API (`--llm_backend responses-api`)
-
-- Handler: `ResponsesApiModelHandler`
-- Typical use: remote model serving via OpenAI-compatible endpoints
-- Backend-specific args prefix: `--responses_api_*`
-- Shared args (from base): `--model_name`, `--chat_size`, `--init_chat_prompt`, `--enable_lang_prompt`
-
-```bash
-python s2s_pipeline.py \
-  --llm_backend responses-api \
-  --model_name gpt-5.4-mini \
-  --responses_api_api_key YOUR_API_KEY \
-  --responses_api_base_url https://api.example.com/v1 \
-  --responses_api_stream true
-```
-
-Common options:
-- `--chat_size`
-- `--init_chat_prompt`
-- `--user_role`
-
-## LLM Behavior
-
-When STT is set to language auto-detection (`--language auto`), LLM handlers can receive `(text, language_code)` and prepend a language control instruction like:
-
-- `Please reply to my message in <language>.`
-
-This helps the assistant respond in the detected language. The behavior is opt-in via `--enable_lang_prompt` (shared across all backends); it defaults to `False`.
-
-## Setup
-
-### CUDA setup
-
-```bash
-python s2s_pipeline.py \
-  --llm_backend transformers \
-  --model_name microsoft/Phi-3-mini-4k-instruct
-```
-
-### Local Mac setup
-
-```bash
-python s2s_pipeline.py \
-  --local_mac_optimal_settings \
-  --model_name mlx-community/Qwen3-4B-Instruct-2507-bf16
-```
-
-`--local_mac_optimal_settings` already sets `--llm_backend mlx-lm` and will default the model to `mlx-community/Qwen3-4B-Instruct-2507-bf16` if not overridden.
-
-### Realtime (OpenAI-compatible) setup
-
-Run the server in realtime mode, then connect with the realtime client:
-
-```bash
-# 1. Start the pipeline in realtime mode
-python s2s_pipeline.py \
-  --mode realtime \
-  --llm_backend mlx-lm \
-  --model_name mlx-community/Qwen3-4B-Instruct-2507-bf16 \
-  --ws_host 0.0.0.0 \
-  --ws_port 8765
-
-# 2. Connect with the realtime client
-python listen_and_play_realtime.py --host 127.0.0.1 --port 8765
-```
-
-Or with `--local_mac_optimal_settings` on Apple Silicon:
-
-```bash
-python s2s_pipeline.py \
-  --local_mac_optimal_settings \
-  --mode realtime \
-  --ws_host 0.0.0.0 \
-  --ws_port 8765
-```
-
-### Remote API setup
-
-```bash
-python s2s_pipeline.py \
-  --llm_backend responses-api \
-  --model_name gpt-5.4-mini \
-  --responses_api_api_key YOUR_API_KEY
-```

runtime/src/speech_to_speech/LLM/audio_input_notifier.py

@@ -1,82 +0,0 @@
-from __future__ import annotations
-
-import logging
-from collections.abc import Iterator
-from queue import Queue
-from threading import Event
-
-from speech_to_speech.api.openai_realtime.runtime_config import RuntimeConfig
-from speech_to_speech.baseHandler import BaseHandler
-from speech_to_speech.pipeline.events import AudioInputCompletedEvent
-from speech_to_speech.pipeline.handler_types import LLMIn
-from speech_to_speech.pipeline.messages import GenerateResponseRequest, VADAudio
-from speech_to_speech.pipeline.queue_types import TextEventItem
-from speech_to_speech.pipeline.speculative_turns import SpeculativeTurnTracker
-
-logger = logging.getLogger(__name__)
-
-
-class AudioInputNotifier(BaseHandler[VADAudio, LLMIn]):
-    """Bridge final VAD audio directly into the LLM stage."""
-
-    def setup(
-        self,
-        runtime_config: RuntimeConfig | None = None,
-        should_listen: Event | None = None,
-        sample_rate: int = 16000,
-        speculative_turns: SpeculativeTurnTracker | None = None,
-        text_output_queue: Queue[TextEventItem] | None = None,
-    ) -> None:
-        self.runtime_config = runtime_config
-        self.should_listen = should_listen
-        self.sample_rate = sample_rate
-        self.speculative_turns = speculative_turns
-        self.text_output_queue = text_output_queue
-
-    def should_process_input(self, item: VADAudio) -> bool:
-        if item.mode == "progressive":
-            return False
-        if self.speculative_turns is None or item.turn_id is None or item.turn_revision is None:
-            return True
-        return self.speculative_turns.is_latest_after_pending_reopen(item.turn_id, item.turn_revision)
-
-    def process(self, vad_audio: VADAudio) -> Iterator[LLMIn]:
-        audio_duration_s = len(vad_audio.audio) / self.sample_rate if self.sample_rate else 0.0
-        logger.info(
-            "Audio input completed: %.3fs turn=%s rev=%s",
-            audio_duration_s,
-            vad_audio.turn_id,
-            vad_audio.turn_revision,
-        )
-
-        # Realtime mode has no setup-level RuntimeConfig. Route the completed
-        # audio through RealtimeService so response lifecycle and usage
-        # bookkeeping happen before the LLM request is queued.
-        if self.runtime_config is None and self.text_output_queue is not None:
-            self.text_output_queue.put(
-                AudioInputCompletedEvent(
-                    audio=vad_audio.audio,
-                    audio_sample_rate=self.sample_rate,
-                    audio_duration_s=audio_duration_s,
-                    turn_id=vad_audio.turn_id,
-                    turn_revision=vad_audio.turn_revision,
-                    speech_stopped_at_s=vad_audio.created_at_s,
-                )
-            )
-            return
-
-        runtime_config = vad_audio.runtime_config or self.runtime_config
-        if runtime_config is None:
-            logger.error("AudioInputNotifier received audio without RuntimeConfig; dropping LLM request")
-            if self.should_listen is not None:
-                self.should_listen.set()
-            return
-
-        yield GenerateResponseRequest(
-            runtime_config=runtime_config,
-            audio=vad_audio.audio,
-            audio_sample_rate=self.sample_rate,
-            turn_id=vad_audio.turn_id,
-            turn_revision=vad_audio.turn_revision,
-            speech_stopped_at_s=vad_audio.created_at_s,
-        )

runtime/src/speech_to_speech/LLM/base_openai_compatible_language_model.py

@@ -1,833 +0,0 @@
-from __future__ import annotations
-
-import base64
-import io
-import ipaddress
-import logging
-import os
-import wave
-from abc import ABC, abstractmethod
-from collections.abc import Callable, Generator, Iterator
-from typing import Any, Literal, Optional
-from urllib.parse import urlparse
-
-import httpx
-import numpy as np
-from nltk import sent_tokenize
-from openai import OpenAI
-from openai.types.realtime.conversation_item import (
-    RealtimeConversationItemAssistantMessage,
-    RealtimeConversationItemFunctionCall,
-)
-from openai.types.realtime.realtime_conversation_item_assistant_message import (
-    Content as AssistantContent,
-)
-from openai.types.responses import ResponseFunctionToolCall
-from pydantic import BaseModel, ConfigDict, Field
-
-from speech_to_speech.baseHandler import BaseHandler
-from speech_to_speech.LLM.chat import (
-    Chat,
-    ChatItemError,
-    SupportedItem,
-    build_active_chat,
-    make_system_message,
-    make_user_audio_message,
-    make_user_message,
-)
-from speech_to_speech.LLM.compaction_prompt import CompactGenerateFn, build_compactor
-from speech_to_speech.LLM.text_prompt import build_text_system_prompt
-from speech_to_speech.LLM.utils import remove_unspeechable, resolve_auto_language
-from speech_to_speech.LLM.voice_prompt import build_voice_system_prompt
-from speech_to_speech.pipeline.cancel_scope import CancelScope
-from speech_to_speech.pipeline.handler_types import LLMIn, LLMOut
-from speech_to_speech.pipeline.messages import (
-    EndOfResponse,
-    LLMResponseChunk,
-    TokenUsage,
-)
-from speech_to_speech.pipeline.speculative_turns import SpeculativeTurnTracker
-from speech_to_speech.utils.utils import is_out_of_band, response_wants_audio
-
-logger = logging.getLogger(__name__)
-
-# About 18–24 seconds of default SDK backoff before warmup fails.
-WARMUP_MAX_RETRIES = 6
-
-
-# ── Normalised provider events ────────────────────────────────────────────────
-# Each backend's stream/response is mapped to this small vocabulary so the shared
-# speech-pipeline logic (sentence batching, cancellation, history, token usage)
-# lives in one place. Subclasses differ only in how they produce these events.
-
-
-class TextDelta(BaseModel):
-    """Incremental assistant text. Always RAW (unfiltered); the base applies
-    ``remove_unspeechable`` for the audio path."""
-
-    text: str
-
-
-class AssistantMessage(BaseModel):
-    """A complete assistant turn to write back to history."""
-
-    content: list[AssistantContent]
-
-
-class ToolCall(BaseModel):
-    """A complete function tool call (``call_id`` / ``id`` already regenerated)."""
-
-    item: ResponseFunctionToolCall
-
-
-class Usage(BaseModel):
-    """Token accounting for the turn."""
-
-    input_tokens: int
-    output_tokens: int
-
-
-ProviderEvent = TextDelta | AssistantMessage | ToolCall | Usage
-SerializeFn = Callable[[Chat], Any]
-RequestFn = Callable[[Any, dict[str, Any]], Any]
-EventIteratorFn = Callable[[Any], Iterator[ProviderEvent]]
-
-
-class _Turn(BaseModel):
-    """Per-request context threaded through generation (immutable for the turn)."""
-
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-
-    language_code: Optional[str]
-    gen: int | None
-    runtime_config: Any
-    response: Any
-    turn_id: str | None
-    turn_revision: int | None
-    speech_stopped_at_s: float | None
-    wants_audio: bool
-
-
-class _GenState(BaseModel):
-    """Mutable accumulators collected while consuming a turn's events."""
-
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-
-    tools: list[ResponseFunctionToolCall] = Field(default_factory=list)
-    pending: list[SupportedItem] = Field(default_factory=list)
-    recorded_item_ids: set[str] = Field(default_factory=set)
-    recorded_call_ids: set[str] = Field(default_factory=set)
-    clean_text: str = ""  # filtered text, kept only for the debug log
-    input_tokens: int = 0
-    output_tokens: int = 0
-
-
-class BaseOpenAICompatibleHandler(BaseHandler[LLMIn, LLMOut], ABC):
-    """Shared lifecycle for OpenAI-compatible LLM backends (Responses & Chat
-    Completions).
-
-    Subclasses implement four hooks — :meth:`warmup`,
-    :meth:`_build_compaction_generate_fn`, :meth:`_serialize`, :meth:`_request`,
-    :meth:`_iter_events` and :meth:`_build_optional_kwargs` — and inherit the
-    request/response orchestration: speculative-turn gating, cancellation,
-    sentence batching, text-only vs audio handling, history write-back, token
-    usage, out-of-band handling and error termination.
-    """
-
-    # ── setup ─────────────────────────────────────────────────────────────────
-
-    def setup(
-        self,
-        model_name: str = "gpt-5.4-mini",
-        device: str = "cuda",
-        gen_kwargs: dict[str, Any] = {},
-        base_url: Optional[str] = None,
-        api_key: Optional[str] = None,
-        stream: bool = True,
-        user_role: str = "user",
-        cancel_scope: CancelScope | None = None,
-        speculative_turns: SpeculativeTurnTracker | None = None,
-        disable_thinking: bool = True,
-        reasoning_effort: Optional[str] = None,
-        request_timeout_s: float = 20.0,
-        stream_batch_sentences: int = 3,
-        enable_lang_prompt: bool = False,
-        compact_history: bool = False,
-        audio_max_tokens: int = 256,
-        audio_temperature: float = 0.0,
-        audio_content_type: Literal["input_audio", "audio_url"] = "input_audio",
-        audio_history_turns: int = 1,
-        **_kwargs: Any,
-    ) -> None:
-        self.cancel_scope = cancel_scope
-        self.speculative_turns = speculative_turns
-        self.model_name = model_name
-        self.stream = stream
-        self.stream_batch_sentences = max(1, stream_batch_sentences)
-        self.enable_lang_prompt = enable_lang_prompt
-        self.gen_kwargs = dict(gen_kwargs)
-        self.audio_max_tokens = audio_max_tokens
-        self.audio_temperature = audio_temperature
-        if audio_content_type not in {"input_audio", "audio_url"}:
-            raise ValueError("audio_content_type must be either 'input_audio' or 'audio_url'.")
-        self.audio_content_type = audio_content_type
-        self.audio_history_turns = max(0, audio_history_turns)
-        self.request_timeout_s = float(request_timeout_s)
-        self.request_timeout = httpx.Timeout(
-            self.request_timeout_s,
-            connect=min(10.0, self.request_timeout_s),
-        )
-
-        self.user_role = user_role
-        if (
-            api_key is None
-            and not os.environ.get("OPENAI_API_KEY")
-            and base_url is not None
-            and self._is_local_base_url(base_url)
-        ):
-            api_key = "none"
-        self.client = OpenAI(api_key=api_key, base_url=base_url)
-        self._extra_body = self._build_extra_body(base_url, disable_thinking, reasoning_effort)
-        self.compactor = build_compactor(self._build_compaction_generate_fn()) if compact_history else None
-        self.warmup()
-
-    @staticmethod
-    def _is_official_openai(base_url: Optional[str]) -> bool:
-        """Whether ``base_url`` points at the official OpenAI server.
-
-        Normalises a trailing slash so ``https://api.openai.com/v1/`` is also
-        recognised; the official server rejects the provider-specific extra_body
-        keys we send to vLLM / the HF router.
-        """
-        if base_url is None:
-            return False
-        return base_url.rstrip("/") == "https://api.openai.com/v1"
-
-    @staticmethod
-    def _is_local_base_url(base_url: str) -> bool:
-        """Whether *base_url* points at localhost or a loopback IP address."""
-        host = urlparse(base_url).hostname
-        if host is None:
-            return False
-        if host.rstrip(".").lower() == "localhost":
-            return True
-        try:
-            return ipaddress.ip_address(host).is_loopback
-        except ValueError:
-            return False
-
-    @classmethod
-    def _build_extra_body(
-        cls,
-        base_url: Optional[str],
-        disable_thinking: bool,
-        reasoning_effort: Optional[str],
-    ) -> Optional[dict[str, Any]]:
-        """Build the provider-specific ``extra_body`` used to disable reasoning.
-
-        Providers differ in how reasoning is turned off: vLLM/Qwen honour
-        ``chat_template_kwargs.enable_thinking=false``, while others (e.g. GLM via
-        the HF router) ignore that and require ``reasoning_effort='none'``. A
-        non-empty ``reasoning_effort`` therefore takes precedence; otherwise we fall
-        back to the chat-template flag. None of this applies to the official
-        OpenAI server, which rejects unknown extra_body keys.
-        """
-        if base_url is None or cls._is_official_openai(base_url):
-            return None
-        if reasoning_effort:
-            return {"reasoning_effort": reasoning_effort}
-        if disable_thinking:
-            return {"chat_template_kwargs": {"enable_thinking": False}}
-        return None
-
-    # ── subclass hooks ──────────────────────────────────────────────────────--
-
-    @abstractmethod
-    def warmup(self) -> None:
-        """Issue a cheap request so the model/connection is ready before serving."""
-        ...
-
-    @abstractmethod
-    def _build_compaction_generate_fn(self) -> CompactGenerateFn:
-        """Return a ``(system, user) -> text`` fn used to compact long histories."""
-        ...
-
-    @abstractmethod
-    def _serialize(self, active_chat: Chat) -> Any:
-        """Serialise the chat to the backend's request payload (input/messages)."""
-        ...
-
-    @abstractmethod
-    def _request(self, api_input: Any, optional_kwargs: dict[str, Any], runtime_config: Any) -> Any:
-        """Issue the create() call and return the response or stream."""
-        ...
-
-    @abstractmethod
-    def _iter_stream_events(self, api_response: Any) -> Iterator[ProviderEvent]:
-        """Map a streaming response to normalised :data:`ProviderEvent`s."""
-        ...
-
-    @abstractmethod
-    def _iter_response_events(self, api_response: Any) -> Iterator[ProviderEvent]:
-        """Map a non-streaming response to normalised :data:`ProviderEvent`s."""
-        ...
-
-    def _iter_events(self, api_response: Any) -> Iterator[ProviderEvent]:
-        """Dispatch to the stream/non-stream mapper. ``self.stream`` is the single
-        source of truth (it set the request's ``stream=`` flag), so the response
-        type always matches it."""
-        if self.stream:
-            yield from self._iter_stream_events(api_response)
-        else:
-            yield from self._iter_response_events(api_response)
-
-    @abstractmethod
-    def _build_optional_kwargs(self, req_tools: Any, req_tool_choice: Any) -> dict[str, Any]:
-        """Build the per-request tools/tool_choice kwargs in the backend's shape."""
-        ...
-
-    # ── audio-input protocol hooks ───────────────────────────────────────────
-
-    def _serialize_audio(self, active_chat: Chat) -> Any:
-        """Serialize an audio turn using the selected backend's native protocol."""
-        return self._serialize(active_chat)
-
-    def _build_audio_optional_kwargs(
-        self,
-        response: Any,
-        req_tools: Any,
-        req_tool_choice: Any,
-    ) -> dict[str, Any]:
-        """Build audio request parameters in the selected backend's shape."""
-        kwargs = self._build_optional_kwargs(req_tools, req_tool_choice)
-        max_tokens = getattr(response, "max_output_tokens", None) if response is not None else None
-        kwargs.setdefault("max_tokens", max_tokens or self.audio_max_tokens)
-        kwargs.setdefault("temperature", self.audio_temperature)
-        return kwargs
-
-    def _request_audio(self, api_input: Any, optional_kwargs: dict[str, Any], runtime_config: Any) -> Any:
-        return self._request(api_input, optional_kwargs, runtime_config)
-
-    def _iter_audio_events(self, api_response: Any) -> Iterator[ProviderEvent]:
-        yield from self._iter_events(api_response)
-
-    @staticmethod
-    def _audio_to_wav_base64(audio: np.ndarray, sample_rate: int) -> str:
-        """Encode a mono 16-bit WAV payload without touching the filesystem."""
-        audio_array = np.asarray(audio)
-        if audio_array.ndim > 1:
-            audio_array = np.mean(audio_array, axis=1)
-        if np.issubdtype(audio_array.dtype, np.floating):
-            pcm = (np.clip(audio_array, -1.0, 1.0) * 32767.0).astype("<i2")
-        else:
-            pcm = np.clip(audio_array, -32768, 32767).astype("<i2")
-
-        with io.BytesIO() as wav_io:
-            with wave.open(wav_io, "wb") as wav_file:
-                wav_file.setnchannels(1)
-                wav_file.setsampwidth(2)
-                wav_file.setframerate(sample_rate)
-                wav_file.writeframes(pcm.tobytes())
-            return base64.b64encode(wav_io.getvalue()).decode("ascii")
-
-    # ── speculative-turn / cancellation gating ─────────────────────────────────
-
-    def _turn_is_latest(self, turn_id: str | None, turn_revision: int | None) -> bool:
-        return self.speculative_turns is None or self.speculative_turns.is_latest(turn_id, turn_revision)
-
-    def _generation_is_stale(self, gen: int | None) -> bool:
-        return gen is not None and self.cancel_scope is not None and self.cancel_scope.is_stale(gen)
-
-    def _turn_output_allowed(self, turn_id: str | None, turn_revision: int | None) -> bool:
-        if self.speculative_turns is None:
-            return True
-        return self.speculative_turns.is_latest_after_reopen_grace(turn_id, turn_revision)
-
-    def _apply_config(
-        self,
-        chat: Chat,
-        instructions: Optional[str],
-        wants_audio: bool = True,
-    ) -> None:
-        if instructions:
-            builder = build_voice_system_prompt if wants_audio else build_text_system_prompt
-            full_instructions = builder(instructions)
-            chat.add_item(make_system_message(full_instructions))
-
-    # ── output helpers ──────────────────────────────────────────────────────--
-
-    def _chunk(
-        self,
-        turn: _Turn,
-        *,
-        text: str = "",
-        tools: list[ResponseFunctionToolCall] | None = None,
-        language_code: Optional[str] = None,
-    ) -> LLMResponseChunk:
-        return LLMResponseChunk(
-            text=text,
-            language_code=language_code if language_code is not None else turn.language_code,
-            tools=tools or [],
-            runtime_config=turn.runtime_config,
-            response=turn.response,
-            turn_id=turn.turn_id,
-            turn_revision=turn.turn_revision,
-            speech_stopped_at_s=turn.speech_stopped_at_s,
-            cancel_generation=turn.gen,
-        )
-
-    def _record_tool_call(self, state: _GenState, turn: _Turn, item: ResponseFunctionToolCall) -> Iterator[LLMOut]:
-        """Emit a tool call, persisting it (and any assistant text seen so far)
-        to history *before* it is forwarded to the client.
-
-        The function_call must already exist in the conversation by the time the
-        client returns its ``function_call_output``; otherwise a fast client
-        races ahead of the deferred end-of-turn write-back and the output is
-        rejected ("No function_call with call_id ... found"), which makes the
-        model re-issue the same tool call. The call lands in ``_pending_tool_calls``
-        (not serialized until its output pairs it), so eager recording is safe.
-
-        Out-of-band turns never touch the default conversation, and a stale turn
-        records nothing (it is not forwarded to the client either)."""
-        state.tools.append(item)
-        fc_item = RealtimeConversationItemFunctionCall(
-            type="function_call",
-            name=item.name,
-            arguments=item.arguments,
-            call_id=item.call_id,
-            id=item.id,
-            status=item.status,
-        )
-        if self._generation_is_stale(turn.gen) or not self._turn_output_allowed(turn.turn_id, turn.turn_revision):
-            logger.info("LLM generation cancelled (stale speculative turn)")
-            return
-        if not is_out_of_band(turn.response):
-            # Flush assistant text accumulated before this call first (so history
-            # order matches what the client received), then persist the call —
-            # all before the chunk leaves for the client.
-            chat = turn.runtime_config.chat
-            for pending_item in state.pending:
-                recorded = chat.add_item(pending_item)
-                if recorded.id is not None:
-                    state.recorded_item_ids.add(recorded.id)
-            state.pending.clear()
-            recorded_call = chat.add_item(fc_item)
-            if recorded_call.id is not None:
-                state.recorded_item_ids.add(recorded_call.id)
-            state.recorded_call_ids.add(item.call_id)
-        yield self._chunk(turn, tools=[item])
-
-    # ── consumption ─────────────────────────────────────────────────────────--
-
-    def _consume_streaming(
-        self,
-        events: Iterator[ProviderEvent],
-        state: _GenState,
-        turn: _Turn,
-    ) -> Generator[LLMOut, None, bool]:
-        cancelled = False
-        printable_text = ""
-        sentence_batch: list[str] = []
-
-        def _flush(batch: list[str]) -> Iterator[LLMOut]:
-            if not batch:
-                return
-            if not self._turn_output_allowed(turn.turn_id, turn.turn_revision):
-                logger.info("LLM generation cancelled (stale speculative turn)")
-                return
-            yield self._chunk(turn, text=" ".join(batch))
-
-        for event in events:
-            if self._generation_is_stale(turn.gen) or not self._turn_is_latest(turn.turn_id, turn.turn_revision):
-                logger.info("LLM generation cancelled (interruption)")
-                cancelled = True
-                break
-
-            if isinstance(event, Usage):
-                state.input_tokens = event.input_tokens
-                state.output_tokens = event.output_tokens
-            elif isinstance(event, AssistantMessage):
-                state.pending.append(
-                    RealtimeConversationItemAssistantMessage(type="message", role="assistant", content=event.content)
-                )
-            elif isinstance(event, ToolCall):
-                # Flush any pending spoken text before emitting the tool call.
-                if printable_text.strip():
-                    sentence_batch.append(printable_text.strip())
-                    printable_text = ""
-                if sentence_batch:
-                    if not self._turn_output_allowed(turn.turn_id, turn.turn_revision):
-                        logger.info("LLM generation cancelled (stale speculative turn)")
-                        cancelled = True
-                        break
-                    yield from _flush(sentence_batch)
-                    sentence_batch = []
-                yield from self._record_tool_call(state, turn, event.item)
-            elif isinstance(event, TextDelta):
-                if not turn.wants_audio:
-                    # Text-only: forward verbatim. Keep every character (no
-                    # remove_unspeechable, which strips TTS-unfriendly symbols) and
-                    # don't sentence-split (sent_tokenize collapses newlines/markdown).
-                    state.clean_text += event.text
-                    if event.text:
-                        if not self._turn_output_allowed(turn.turn_id, turn.turn_revision):
-                            logger.info("LLM generation cancelled (stale speculative turn)")
-                            cancelled = True
-                            break
-                        yield self._chunk(turn, text=event.text)
-                    continue
-                new_text = remove_unspeechable(event.text)
-                state.clean_text += new_text
-                printable_text += new_text
-                sentences = sent_tokenize(printable_text)
-                if len(sentences) > 1:
-                    for s in sentences[:-1]:
-                        sentence_batch.append(s)
-                        if len(sentence_batch) >= self.stream_batch_sentences:
-                            if not self._turn_output_allowed(turn.turn_id, turn.turn_revision):
-                                logger.info("LLM generation cancelled (stale speculative turn)")
-                                cancelled = True
-                                break
-                            yield from _flush(sentence_batch)
-                            sentence_batch = []
-                    if cancelled:
-                        break
-                    printable_text = sentences[-1]
-
-        if not cancelled:
-            if printable_text.strip():
-                sentence_batch.append(printable_text.strip())
-            if sentence_batch:
-                if self._generation_is_stale(turn.gen):
-                    logger.info("LLM generation cancelled (interruption)")
-                else:
-                    logger.debug(f"Clean text: {state.clean_text}")
-                    yield from _flush(sentence_batch)
-            logger.info(f"Tools: {state.tools}")
-        return (
-            not cancelled
-            and not self._generation_is_stale(turn.gen)
-            and self._turn_is_latest(turn.turn_id, turn.turn_revision)
-            and self._turn_output_allowed(turn.turn_id, turn.turn_revision)
-        )
-
-    def _consume_nonstreaming(
-        self,
-        events: Iterator[ProviderEvent],
-        state: _GenState,
-        turn: _Turn,
-    ) -> Generator[LLMOut, None, bool]:
-        if self._generation_is_stale(turn.gen) or not self._turn_is_latest(turn.turn_id, turn.turn_revision):
-            logger.info("LLM generation cancelled (interruption)")
-            return False
-        for event in events:
-            if isinstance(event, Usage):
-                state.input_tokens = event.input_tokens
-                state.output_tokens = event.output_tokens
-            elif isinstance(event, AssistantMessage):
-                state.pending.append(
-                    RealtimeConversationItemAssistantMessage(type="message", role="assistant", content=event.content)
-                )
-            elif isinstance(event, ToolCall):
-                yield from self._record_tool_call(state, turn, event.item)
-            elif isinstance(event, TextDelta):
-                # Text-only keeps every character verbatim; audio strips
-                # TTS-unfriendly symbols via remove_unspeechable.
-                spoken = event.text if not turn.wants_audio else remove_unspeechable(event.text)
-                state.clean_text += spoken
-                out = spoken if not turn.wants_audio else spoken.strip()
-                if (
-                    out
-                    and not self._generation_is_stale(turn.gen)
-                    and self._turn_output_allowed(turn.turn_id, turn.turn_revision)
-                ):
-                    yield self._chunk(turn, text=out)
-        logger.debug(f"Clean text: {state.clean_text}")
-        logger.info(f"Tools: {state.tools}")
-        return (
-            not self._generation_is_stale(turn.gen)
-            and self._turn_is_latest(turn.turn_id, turn.turn_revision)
-            and self._turn_output_allowed(turn.turn_id, turn.turn_revision)
-        )
-
-    # ── orchestration ─────────────────────────────────────────────────────────
-
-    def _generate(
-        self,
-        active_chat: Chat,
-        original_chat: Chat,
-        turn: _Turn,
-        optional_kwargs: dict[str, Any],
-        *,
-        serialize_fn: SerializeFn | None = None,
-        request_fn: RequestFn | None = None,
-        event_iterator_fn: EventIteratorFn | None = None,
-        transactional_user_message_id: str | None = None,
-        history_commit_fn: Callable[[], None] | None = None,
-    ) -> Generator[LLMOut, None, bool]:
-        api_response: Any = None
-        state = _GenState()
-        error_message: str | None = None
-        generation_completed = False
-        history_committed = False
-        transaction_rolled_back = False
-        consumed_image_ids: set[str] = set()
-
-        def rollback_transaction() -> None:
-            nonlocal transaction_rolled_back
-            if transactional_user_message_id is None or history_committed or transaction_rolled_back:
-                return
-            original_chat.rollback_generation(
-                transactional_user_message_id,
-                item_ids=state.recorded_item_ids,
-                call_ids=state.recorded_call_ids,
-            )
-            transaction_rolled_back = True
-
-        try:
-            try:
-                api_input = (serialize_fn or self._serialize)(active_chat)
-                # Images the model actually sees this turn; only these are stripped on
-                # write-back, so an image a fast client injects mid-generation for the
-                # next turn survives (it is not in this serialized snapshot).
-                consumed_image_ids = active_chat.image_message_ids()
-                if not api_input:
-                    # Nothing to send: empty `instructions` and no `input` (in the response,
-                    # the default conversation, or the out-of-band context). The provider
-                    # would reject this; fail with a clear message instead of an opaque error.
-                    error_message = "Cannot generate a response: no instructions and no input were provided."
-                else:
-                    if request_fn is not None:
-                        api_response = request_fn(api_input, optional_kwargs)
-                    else:
-                        api_response = self._request(api_input, optional_kwargs, turn.runtime_config)
-                if api_response is not None:
-                    events = (event_iterator_fn or self._iter_events)(api_response)
-                    if self.stream:
-                        generation_completed = yield from self._consume_streaming(events, state, turn)
-                    else:
-                        generation_completed = yield from self._consume_nonstreaming(events, state, turn)
-            except httpx.ReadTimeout:
-                logger.warning(
-                    "OpenAI API read timed out after %.1fs; ending the current response",
-                    self.request_timeout_s,
-                )
-                if not self._generation_is_stale(turn.gen) and self._turn_output_allowed(
-                    turn.turn_id, turn.turn_revision
-                ):
-                    # Canned apology carries no language_code (mirrors the prior handlers).
-                    yield LLMResponseChunk(
-                        text="Wow I'm a bit slow today, could you repeat that?",
-                        runtime_config=turn.runtime_config,
-                        response=turn.response,
-                        turn_id=turn.turn_id,
-                        turn_revision=turn.turn_revision,
-                        speech_stopped_at_s=turn.speech_stopped_at_s,
-                        cancel_generation=turn.gen,
-                    )
-            except Exception as exc:
-                # Any other generation failure must still terminate the response: record
-                # the error and fall through to the EndOfResponse below. Without this the
-                # exception would escape process() and no EndOfResponse would be emitted,
-                # leaving st.in_response stuck and locking every subsequent response.
-                logger.exception("LLM generation failed; ending the current response")
-                if error_message is None:
-                    error_message = f"Language model generation failed: {exc}"
-
-            can_commit = (
-                error_message is None
-                and generation_completed
-                and not self._generation_is_stale(turn.gen)
-                and self._turn_is_latest(turn.turn_id, turn.turn_revision)
-                and self._turn_output_allowed(turn.turn_id, turn.turn_revision)
-            )
-            if can_commit:
-                try:
-                    # Out-of-band responses emit output and usage but never write back to the
-                    # default conversation (their context was a throwaway chat).
-                    if not is_out_of_band(turn.response):
-                        # Tool calls (and any assistant text preceding them) were already
-                        # written eagerly in _record_tool_call; only trailing items remain.
-                        for item in state.pending:
-                            recorded = original_chat.add_item(item)
-                            if recorded.id is not None:
-                                state.recorded_item_ids.add(recorded.id)
-                        original_chat.strip_images(consumed_image_ids)
-                        if history_commit_fn is not None:
-                            history_commit_fn()
-                        original_chat.trim_if_needed(self.compactor)
-                    history_committed = True
-                except Exception as exc:
-                    logger.exception("LLM history commit failed; rolling back the current response")
-                    error_message = f"Language model history commit failed: {exc}"
-
-            rollback_transaction()
-            if history_committed and (state.input_tokens or state.output_tokens):
-                yield TokenUsage(
-                    input_tokens=state.input_tokens,
-                    output_tokens=state.output_tokens,
-                    turn_id=turn.turn_id,
-                    turn_revision=turn.turn_revision,
-                )
-            yield EndOfResponse(
-                turn_id=turn.turn_id,
-                turn_revision=turn.turn_revision,
-                cancel_generation=turn.gen,
-                error=error_message,
-            )
-            return history_committed
-        finally:
-            if api_response is not None and hasattr(api_response, "close"):
-                try:
-                    api_response.close()
-                except Exception:
-                    pass
-            rollback_transaction()
-
-    def _process_audio(self, request: LLMIn) -> Iterator[LLMOut]:
-        """Process an audio-input turn through the selected backend protocol."""
-        assert request.audio is not None
-        runtime_config = request.runtime_config
-        response = request.response
-        turn_id = request.turn_id
-        turn_revision = request.turn_revision
-        speech_stopped_at_s = request.speech_stopped_at_s
-        if not self._turn_is_latest(turn_id, turn_revision):
-            logger.info("Skipping stale LLM request for turn=%s rev=%s", turn_id, turn_revision)
-            yield EndOfResponse(turn_id=turn_id, turn_revision=turn_revision)
-            return
-
-        original_chat = runtime_config.chat
-        if is_out_of_band(response):
-            try:
-                active_chat = build_active_chat(original_chat, response)
-            except ChatItemError as exc:
-                logger.info("Out-of-band response rejected: %s", exc)
-                yield EndOfResponse(turn_id=turn_id, turn_revision=turn_revision, error=str(exc))
-                return
-        else:
-            active_chat = original_chat.copy()
-
-        language_code = request.language_code
-        instructions = (
-            response.instructions if response and response.instructions else runtime_config.session.instructions
-        ) or ""
-        req_tools = response.tools if response and response.tools else runtime_config.session.tools
-        req_tool_choice = (
-            response.tool_choice if response and response.tool_choice else runtime_config.session.tool_choice
-        )
-        wants_audio = response_wants_audio(response)
-        self._apply_config(active_chat, instructions, wants_audio)
-        language_code, lang_name = resolve_auto_language(language_code)
-        if lang_name and self.enable_lang_prompt:
-            active_chat.add_item(make_user_message(f"Please reply to my message in {lang_name}."))
-
-        audio_b64 = self._audio_to_wav_base64(request.audio, request.audio_sample_rate)
-        audio_message = active_chat.add_item(make_user_audio_message(audio_b64))
-        optional_kwargs = self._build_audio_optional_kwargs(response, req_tools, req_tool_choice)
-
-        transactional_user_message_id: str | None = None
-        history_commit_fn: Callable[[], None] | None = None
-        if not is_out_of_band(response):
-            provisional_message = make_user_audio_message(audio_b64)
-            provisional_message.id = audio_message.id
-            original_chat.add_item(provisional_message)
-            assert provisional_message.id is not None
-            transactional_user_message_id = provisional_message.id
-
-            def commit_audio_history() -> None:
-                original_chat.compact_audio_history(self.audio_history_turns)
-
-            history_commit_fn = commit_audio_history
-
-        # CancelScope.is_stale(gen) is checked when the stream iterator advances; a
-        # blocked read inside httpx cannot be aborted by cancel_scope.cancel() from
-        # the websocket router. Mitigations: request_timeout_s / ReadTimeout.
-        gen = self.cancel_scope.generation if self.cancel_scope else None
-        turn = _Turn(
-            language_code=language_code,
-            gen=gen,
-            runtime_config=runtime_config,
-            response=response,
-            turn_id=turn_id,
-            turn_revision=turn_revision,
-            speech_stopped_at_s=speech_stopped_at_s,
-            wants_audio=wants_audio,
-        )
-        yield from self._generate(
-            active_chat,
-            original_chat,
-            turn,
-            optional_kwargs,
-            serialize_fn=self._serialize_audio,
-            request_fn=lambda api_input, kwargs: self._request_audio(api_input, kwargs, runtime_config),
-            event_iterator_fn=self._iter_audio_events,
-            transactional_user_message_id=transactional_user_message_id,
-            history_commit_fn=history_commit_fn,
-        )
-
-    def process(self, request: LLMIn) -> Iterator[LLMOut]:
-        """Process a language model request and yield LLMResponseChunks."""
-        if request.audio is not None:
-            yield from self._process_audio(request)
-            return
-
-        runtime_config = request.runtime_config
-        response = request.response
-        turn_id = request.turn_id
-        turn_revision = request.turn_revision
-        speech_stopped_at_s = request.speech_stopped_at_s
-        if not self._turn_is_latest(turn_id, turn_revision):
-            logger.info("Skipping stale LLM request for turn=%s rev=%s", turn_id, turn_revision)
-            yield EndOfResponse(turn_id=turn_id, turn_revision=turn_revision)
-            return
-
-        original_chat = runtime_config.chat
-        if is_out_of_band(response):
-            try:
-                active_chat = build_active_chat(original_chat, response)
-            except ChatItemError as exc:
-                logger.info("Out-of-band response rejected: %s", exc)
-                yield EndOfResponse(turn_id=turn_id, turn_revision=turn_revision, error=str(exc))
-                return
-        else:
-            active_chat = original_chat.copy()
-        language_code = request.language_code
-        instructions = (
-            response.instructions if response and response.instructions else runtime_config.session.instructions
-        ) or ""
-        req_tools = response.tools if response and response.tools else runtime_config.session.tools
-        req_tool_choice = (
-            response.tool_choice if response and response.tool_choice else runtime_config.session.tool_choice
-        )
-        wants_audio = response_wants_audio(response)
-        self._apply_config(active_chat, instructions, wants_audio)
-        language_code, lang_name = resolve_auto_language(language_code)
-        if lang_name and self.enable_lang_prompt:
-            active_chat.add_item(make_user_message(f"Please reply to my message in {lang_name}."))
-
-        optional_kwargs = self._build_optional_kwargs(req_tools, req_tool_choice)
-
-        # CancelScope.is_stale(gen) is checked when the stream iterator advances; a
-        # blocked read inside httpx cannot be aborted by cancel_scope.cancel() from
-        # the websocket router. Mitigations: request_timeout_s / ReadTimeout.
-        gen = self.cancel_scope.generation if self.cancel_scope else None
-
-        turn = _Turn(
-            language_code=language_code,
-            gen=gen,
-            runtime_config=runtime_config,
-            response=response,
-            turn_id=turn_id,
-            turn_revision=turn_revision,
-            speech_stopped_at_s=speech_stopped_at_s,
-            wants_audio=wants_audio,
-        )
-        yield from self._generate(active_chat, original_chat, turn, optional_kwargs)
-
-    @property
-    def timing_log_level(self) -> int:
-        return logging.INFO
-
-    def should_log_timing(self, output: LLMOut) -> bool:
-        return isinstance(output, LLMResponseChunk) and self.last_time > self.min_time_to_debug

runtime/src/speech_to_speech/LLM/chat.py

@@ -1,835 +0,0 @@
-from __future__ import annotations
-
-import json
-import logging
-import threading
-from collections.abc import Callable
-from typing import Any, Literal, Union
-
-from openai.types.realtime import ConversationItem
-from openai.types.realtime.conversation_item import (
-    RealtimeConversationItemAssistantMessage,
-    RealtimeConversationItemFunctionCall,
-    RealtimeConversationItemFunctionCallOutput,
-    RealtimeConversationItemSystemMessage,
-    RealtimeConversationItemUserMessage,
-)
-from openai.types.realtime.realtime_conversation_item_assistant_message import (
-    Content as AssistantContent,
-)
-from openai.types.realtime.realtime_conversation_item_system_message import Content as SystemContent
-from openai.types.realtime.realtime_conversation_item_user_message import Content as UserContent
-from openai.types.realtime.realtime_response_create_params import RealtimeResponseCreateParams
-from openai.types.responses.response_input_image_param import ResponseInputImageParam
-from openai.types.responses.response_input_message_content_list_param import (
-    ResponseInputMessageContentListParam,
-)
-from openai.types.responses.response_input_param import (
-    FunctionCallOutput,
-    ResponseFunctionToolCallParam,
-    ResponseInputItemParam,
-    ResponseInputParam,
-    ResponseOutputMessageParam,
-)
-from openai.types.responses.response_input_param import (
-    Message as ResponseMessage,
-)
-from openai.types.responses.response_input_text_param import ResponseInputTextParam
-from openai.types.responses.response_output_text_param import ResponseOutputTextParam
-from pydantic import BaseModel
-
-from speech_to_speech.utils.utils import _generate_id
-
-logger = logging.getLogger(__name__)
-
-AUDIO_INPUT_HISTORY_PLACEHOLDER = "[User audio input]"
-
-
-class ChatItemError(Exception):
-    """Raised when a conversation item fails validation in :meth:`Chat.add_item`."""
-
-
-class CompactionResult(BaseModel):
-    """Output of a :data:`CompactFn` summarization run."""
-
-    user_summary: str
-    assistant_summary: str
-
-
-def _ensure_id(value: str | None, prefix: str) -> str:
-    if value is None:
-        return _generate_id(prefix)
-    if not value.startswith(f"{prefix}_"):
-        raise ChatItemError(f"ID must start with '{prefix}_', got {value!r}")
-    return value
-
-
-SupportedItem = Union[
-    RealtimeConversationItemSystemMessage,
-    RealtimeConversationItemUserMessage,
-    RealtimeConversationItemAssistantMessage,
-    RealtimeConversationItemFunctionCall,
-    RealtimeConversationItemFunctionCallOutput,
-]
-
-
-CompactFn = Callable[[ResponseInputParam], CompactionResult]
-
-
-class Chat:
-    """Manages conversation history with bounded size to avoid OOM issues.
-
-    The buffer stores ``ConversationItem`` objects (user messages, assistant
-    messages, function calls, function call outputs).  System messages are
-    stored separately in ``init_chat_message`` and never placed in the buffer.
-
-    History bounding is decided per ``add_item`` call via the ``compactor``
-    argument:
-
-    - ``compactor=None``: when the user-turn count exceeds ``size`` the oldest
-      complete turn is evicted in place. Synchronous, lossy, no LLM involvement.
-    - ``compactor=<fn>``: when ``size`` is exceeded, ``fn`` is invoked in a
-      background thread to summarize older turns into a single user/assistant
-      pair (with pending function calls preserved). Single-flight: while a
-      compaction is running, additional triggers are silently bypassed.
-    """
-
-    def __init__(self, size: int) -> None:
-        self.size = size
-        self.init_chat_message: RealtimeConversationItemSystemMessage | None = None
-        # ``size`` is the number of user turns to keep.  When exceeded the
-        # oldest complete turn (everything up to the next user message)
-        # is evicted -- or, with a compactor, summarized in the background.
-        self.buffer: list[SupportedItem] = []
-        self._pending_tool_calls: dict[str, RealtimeConversationItemFunctionCall] = {}
-        self._user_turn_count: int = 0
-
-        # All state mutations and serializations go through _lock. Public methods
-        # acquire it once; internal callers that already hold it use the
-        # ``_locked`` helpers, so no reentry is needed (regular Lock is safe).
-        self._lock = threading.Lock()
-        self._compact_in_flight: bool = False
-        self._compact_thread: threading.Thread | None = None
-        self._shutdown = threading.Event()
-        self._gen_counter = 0
-
-    # ── Internal mutators (caller holds _lock) ─────────────────
-
-    def _evict_oldest_turn(self) -> None:
-        """Remove items from the front until the next user message boundary."""
-        if not self.buffer:
-            return
-        first = self.buffer.pop(0)
-        if isinstance(first, RealtimeConversationItemUserMessage):
-            self._user_turn_count -= 1
-        while self.buffer and not isinstance(self.buffer[0], RealtimeConversationItemUserMessage):
-            self.buffer.pop(0)
-
-    def _has_call_id_in_buffer(self, call_id: str) -> bool:
-        for entry in self.buffer:
-            if isinstance(entry, RealtimeConversationItemFunctionCall) and entry.call_id == call_id:
-                return True
-        return False
-
-    def _mark_call_completed(
-        self, call_id: str, status: Literal["completed", "incomplete", "in_progress"] | None = None
-    ) -> None:
-        """Set ``status`` to ``"completed"`` on the matching function_call."""
-        for entry in self.buffer:
-            if isinstance(entry, RealtimeConversationItemFunctionCall) and entry.call_id == call_id:
-                entry.status = "completed" if status is None else status
-                return
-
-    def append_tool_output(self, call_id: str, output_item: RealtimeConversationItemFunctionCallOutput) -> None:
-        """Append a ``function_call_output``, re-injecting its ``function_call`` if evicted.
-
-        Also marks the paired ``function_call`` as ``"completed"`` if its
-        status was ``None``.
-
-        Raises :class:`ChatItemError` if *call_id* is unknown.
-        """
-        with self._lock:
-            self._append_tool_output_locked(call_id, output_item)
-
-    def _append_tool_output_locked(self, call_id: str, output_item: RealtimeConversationItemFunctionCallOutput) -> None:
-        """Body of :meth:`append_tool_output`. Caller must hold ``_lock``."""
-        if self._has_call_id_in_buffer(call_id):
-            self._pending_tool_calls.pop(call_id, None)
-            self._mark_call_completed(call_id, output_item.status)
-            self.buffer.append(output_item)
-            return
-
-        if call_id in self._pending_tool_calls:
-            logger.info("Re-injecting evicted function_call for call_id=%s", call_id)
-            fc = self._pending_tool_calls.pop(call_id)
-            fc.status = "completed" if output_item.status is None else output_item.status
-            self.buffer.append(fc)
-            self.buffer.append(output_item)
-            return
-
-        raise ChatItemError(f"No function_call with call_id '{call_id}' found in conversation history.")
-
-    def init_chat(self, message: RealtimeConversationItemSystemMessage) -> None:
-        with self._lock:
-            self.init_chat_message = message
-
-    def add_item(self, item: SupportedItem) -> SupportedItem:
-        """Validate and route a conversation item into the chat buffer.
-
-        Does not enforce the soft size limit — call :meth:`trim_if_needed`
-        explicitly after each successful generation to evict or compact old
-        turns. A hard upper bound at ``2 * size`` is enforced inline as a
-        runaway-client safety net: if the user-turn count exceeds it, the
-        oldest complete turn is evicted (lossy, no compaction).
-
-        Raises :class:`ChatItemError` if the item fails validation.
-        """
-        with self._lock:
-            if isinstance(item, RealtimeConversationItemSystemMessage):
-                item.id = _ensure_id(item.id, "sys")
-                self.init_chat_message = item
-                logger.debug("Set system message via conversation item")
-
-            elif isinstance(item, RealtimeConversationItemUserMessage):
-                item.id = _ensure_id(item.id, "msg")
-                item.content = [
-                    p
-                    for p in item.content
-                    if (p.type == "input_text" and p.text)
-                    or (p.type == "input_image" and p.image_url)
-                    or (p.type == "input_audio" and p.audio)
-                ]
-                if not item.content:
-                    raise ChatItemError(
-                        "Message has no supported content. Supported modalities: input_text, input_image, input_audio."
-                    )
-                self.buffer.append(item)
-                self._user_turn_count += 1
-                logger.debug("Added user message to chat (%d parts)", len(item.content))
-
-            elif isinstance(item, RealtimeConversationItemAssistantMessage):
-                item.id = _ensure_id(item.id, "msg")
-                item.content = [p for p in item.content if p.type == "output_text" and p.text]
-                if not item.content:
-                    return item
-                self.buffer.append(item)
-                logger.debug("Added assistant message to chat (%d parts)", len(item.content))
-
-            elif isinstance(item, RealtimeConversationItemFunctionCall):
-                item.id = _ensure_id(item.id, "fc")
-                item.call_id = _ensure_id(item.call_id, "call")
-                self._pending_tool_calls[item.call_id] = item
-                logger.debug("Added function_call to chat (call_id=%s)", item.call_id)
-
-            elif isinstance(item, RealtimeConversationItemFunctionCallOutput):
-                item.id = _ensure_id(item.id, "fco")
-                self._append_tool_output_locked(item.call_id, item)
-                logger.debug("Added function_call_output to chat (call_id=%s)", item.call_id)
-
-            else:
-                raise ChatItemError(f"Unsupported item type: {getattr(item, 'type', None)}")
-
-            if self.size > 0 and self._user_turn_count > 2 * self.size:
-                logger.warning(
-                    "Chat buffer exceeded hard cap (%d > 2 * size=%d); evicting oldest turn",
-                    self._user_turn_count,
-                    self.size,
-                )
-                while self._user_turn_count > 2 * self.size:
-                    self._evict_oldest_turn()
-
-            return item
-
-    def trim_if_needed(self, compactor: CompactFn | None = None) -> None:
-        """Enforce the size limit after a generation completes. Fires when
-        ``user_turn_count > size``.
-
-        - ``compactor=None``: synchronous eviction of the oldest complete turn.
-        - ``compactor=<fn>``: launch a background compaction (single-flight).
-
-        Call once after each successful generation, not inside :meth:`add_item`.
-        """
-        with self._lock:
-            if self._user_turn_count <= self.size:
-                return
-            if compactor is not None:
-                self._maybe_trigger_compaction(compactor)
-            else:
-                while self._user_turn_count > self.size:
-                    self._evict_oldest_turn()
-
-    def replace_user_message_text(self, item_id: str, text: str) -> bool:
-        """Replace the text content of an existing user message.
-
-        Used by speculative turn revisions: the conversation turn remains the
-        same, but the STT transcript is superseded by a transcription of a
-        longer raw-audio buffer.
-        """
-
-        with self._lock:
-            for item in self.buffer:
-                if not isinstance(item, RealtimeConversationItemUserMessage) or item.id != item_id:
-                    continue
-                item.content = [UserContent(type="input_text", text=text)]
-                logger.debug("Replaced speculative user message %s", item_id)
-                return True
-        return False
-
-    def remove_user_message(self, item_id: str) -> bool:
-        """Remove an existing user message from the bounded chat buffer."""
-
-        with self._lock:
-            for index, item in enumerate(self.buffer):
-                if not isinstance(item, RealtimeConversationItemUserMessage) or item.id != item_id:
-                    continue
-                del self.buffer[index]
-                self._user_turn_count -= 1
-                logger.debug("Removed speculative user message %s", item_id)
-                return True
-        return False
-
-    def rollback_generation(
-        self,
-        user_message_id: str,
-        *,
-        item_ids: set[str],
-        call_ids: set[str],
-    ) -> None:
-        """Remove only the provisional state written by one failed generation.
-
-        A tool output may be appended by a fast client while generation is still
-        streaming, so rollback matches both item IDs and tool ``call_id`` values.
-        Unrelated messages injected concurrently for a later turn are preserved.
-        """
-
-        with self._lock:
-            kept: list[SupportedItem] = []
-            for item in self.buffer:
-                remove = item.id == user_message_id or item.id in item_ids
-                if isinstance(item, (RealtimeConversationItemFunctionCall, RealtimeConversationItemFunctionCallOutput)):
-                    remove = remove or item.call_id in call_ids
-                if not remove:
-                    kept.append(item)
-            self.buffer = kept
-            for call_id in call_ids:
-                self._pending_tool_calls.pop(call_id, None)
-            self._user_turn_count = sum(isinstance(item, RealtimeConversationItemUserMessage) for item in self.buffer)
-            logger.debug("Rolled back failed generation for user message %s", user_message_id)
-
-    def compact_audio_history(self, max_audio_turns: int) -> None:
-        """Retain only the newest bounded set of audio turns.
-
-        Older audio parts are replaced with a textual placeholder so the user
-        role and its paired assistant response remain valid in serialized
-        history. The newest turns keep their audio semantics for subsequent
-        Chat Completions requests.
-        """
-
-        with self._lock:
-            remaining = max(0, max_audio_turns)
-            for item in reversed(self.buffer):
-                if not isinstance(item, RealtimeConversationItemUserMessage):
-                    continue
-                if not any(part.type == "input_audio" for part in item.content):
-                    continue
-                if remaining:
-                    remaining -= 1
-                    continue
-                replacement_added = False
-                compacted: list[UserContent] = []
-                for part in item.content:
-                    if part.type != "input_audio":
-                        compacted.append(part)
-                    elif not replacement_added:
-                        compacted.append(UserContent(type="input_text", text=AUDIO_INPUT_HISTORY_PLACEHOLDER))
-                        replacement_added = True
-                item.content = compacted
-
-    def to_responses_api_chat(self, items: list[SupportedItem] | None = None) -> ResponseInputParam:
-        """Serialize the chat (system prompt + buffer) for the OpenAI Responses API.
-
-        If *items* is provided, serialize that slice instead of the live buffer
-        (used by the compaction snapshot).
-        """
-        with self._lock:
-            return self._to_responses_api_chat_locked(items if items is not None else self.buffer)
-
-    def _to_responses_api_chat_locked(self, items: list[SupportedItem]) -> ResponseInputParam:
-        """Body of :meth:`to_responses_api_chat`. Caller must hold ``_lock``."""
-        buffer_items = list(items)
-        result: list[ResponseInputItemParam] = []
-        if self.init_chat_message:
-            result.append(
-                ResponseMessage(
-                    content=[
-                        ResponseInputTextParam(text=p.text or "A helpful AI assistant.", type="input_text")
-                        for p in self.init_chat_message.content
-                    ],
-                    role="system",
-                    type="message",
-                )
-            )
-        for item in buffer_items:
-            assert item.id is not None and item.id != "", f"item.id is {item.id}"
-            if isinstance(item, RealtimeConversationItemUserMessage):
-                content: ResponseInputMessageContentListParam = []
-                audio_placeholder_added = False
-                for user_part in item.content:
-                    if user_part.type == "input_text" and user_part.text is not None:
-                        content.append(ResponseInputTextParam(text=user_part.text or "", type="input_text"))
-                    elif user_part.type == "input_image" and user_part.image_url is not None:
-                        img = ResponseInputImageParam(type="input_image", detail=user_part.detail or "auto")
-                        if user_part.image_url is not None:
-                            img["image_url"] = user_part.image_url
-                        content.append(img)
-                    elif user_part.type == "input_audio" and not audio_placeholder_added:
-                        content.append(ResponseInputTextParam(text=AUDIO_INPUT_HISTORY_PLACEHOLDER, type="input_text"))
-                        audio_placeholder_added = True
-                if content:
-                    result.append(ResponseMessage(content=content, role="user", type="message"))
-            elif isinstance(item, RealtimeConversationItemAssistantMessage):
-                assistant_content: list[ResponseOutputTextParam] = []
-                for assistant_part in item.content:
-                    if assistant_part.type == "output_text" and assistant_part.text is not None:
-                        assistant_content.append(
-                            ResponseOutputTextParam(text=assistant_part.text, type="output_text", annotations=[])
-                        )
-                if assistant_content:
-                    result.append(
-                        ResponseOutputMessageParam(
-                            id=item.id,
-                            content=assistant_content,
-                            role="assistant",
-                            status=item.status or "completed",
-                            type="message",
-                        )
-                    )
-            elif isinstance(item, RealtimeConversationItemFunctionCall) and item.call_id is not None:
-                assert item.call_id is not None and item.call_id != ""
-                function_call = ResponseFunctionToolCallParam(
-                    arguments=item.arguments,
-                    call_id=item.call_id,
-                    name=item.name,
-                    type="function_call",
-                    id=item.id,
-                )
-                if item.id is not None:
-                    function_call["id"] = item.id
-                if item.status is not None:
-                    function_call["status"] = item.status
-                result.append(function_call)
-            elif isinstance(item, RealtimeConversationItemFunctionCallOutput):
-                function_call_output = FunctionCallOutput(
-                    call_id=item.call_id,
-                    output=item.output,
-                    type="function_call_output",
-                )
-                if item.id is not None:
-                    function_call_output["id"] = item.id
-                if item.status is not None:
-                    function_call_output["status"] = item.status
-                result.append(function_call_output)
-        return result
-
-    def to_transformers_chat(self) -> list[dict[str, Any]]:
-        """Serialize the full chat for HuggingFace transformers ``apply_chat_template``.
-
-        User messages with only text produce a plain string ``content`` value.
-        User messages containing images keep ``content`` as a list of dicts so
-        VLM pipelines can process them.
-        """
-        with self._lock:
-            messages: list[TransformersChatMessage] = []
-            if self.init_chat_message:
-                text = " ".join(p.text for p in self.init_chat_message.content if p.text)
-                messages.append(TransformersSystemMessage(content=text))
-            for item in self.buffer:
-                if isinstance(item, RealtimeConversationItemUserMessage):
-                    has_media = any(p.type in {"input_image", "input_audio"} for p in item.content)
-                    if has_media:
-                        messages.append(
-                            TransformersUserMessage(content=[p.model_dump(exclude_none=True) for p in item.content])
-                        )
-                    else:
-                        text = " ".join(p.text for p in item.content if p.type == "input_text" and p.text)
-                        messages.append(TransformersUserMessage(content=text))
-                elif isinstance(item, RealtimeConversationItemAssistantMessage):
-                    text = " ".join(p.text for p in item.content if p.text)
-                    messages.append(TransformersAssistantMessage(content=text))
-                elif isinstance(item, RealtimeConversationItemFunctionCall):
-                    assert item.call_id is not None and item.call_id != ""
-                    args: Any = item.arguments
-                    try:
-                        args = json.loads(args) if isinstance(args, str) else args
-                    except (json.JSONDecodeError, TypeError):
-                        args = {}
-                    messages.append(
-                        TransformersFunctionCallMessage(
-                            tool_calls=[
-                                TransformersToolCall(
-                                    id=item.call_id,
-                                    function=TransformersToolCallFunction(name=item.name, arguments=args),
-                                )
-                            ]
-                        )
-                    )
-                elif isinstance(item, RealtimeConversationItemFunctionCallOutput):
-                    name = ""
-                    for prev in reversed(messages):
-                        if isinstance(prev, TransformersFunctionCallMessage):
-                            for tc in prev.tool_calls:
-                                if tc.id == item.call_id:
-                                    name = tc.function.name
-                                    break
-                            if name:
-                                break
-                    messages.append(
-                        TransformersToolMessage(
-                            tool_call_id=item.call_id,
-                            name=name,
-                            content=item.output,
-                        )
-                    )
-            return [m.model_dump() for m in messages]
-
-    def copy(self) -> Chat:
-        """Return a shallow snapshot safe for concurrent read access."""
-        with self._lock:
-            clone = Chat(self.size)
-            clone.init_chat_message = self.init_chat_message
-            clone.buffer = list(self.buffer)
-            clone._pending_tool_calls = dict(self._pending_tool_calls)
-            clone._user_turn_count = self._user_turn_count
-            return clone
-
-    def reset(self) -> None:
-        """Clear all conversation state. Cancels any in-flight compaction splice."""
-        with self._lock:
-            self._gen_counter += 1
-            self._compact_in_flight = False
-            self.buffer = []
-            self.init_chat_message = None
-            self._pending_tool_calls = {}
-            self._user_turn_count = 0
-
-    def close(self) -> None:
-        """Permanently shut down the chat. In-flight compaction splice is suppressed.
-
-        The compaction worker (a daemon thread) is not joined: it may be blocked
-        in an LLM call. Process exit reaps it.
-        """
-        self._shutdown.set()
-        with self._lock:
-            self._gen_counter += 1
-            self._compact_in_flight = False
-
-    def image_message_ids(self) -> set[str]:
-        """IDs of user messages currently carrying ``input_image`` content."""
-        with self._lock:
-            return {
-                item.id
-                for item in self.buffer
-                if isinstance(item, RealtimeConversationItemUserMessage)
-                and item.id is not None
-                and any(p.type == "input_image" for p in item.content)
-            }
-
-    def strip_images(self, only_ids: set[str] | None = None) -> None:
-        """Remove image content parts from user messages in the buffer.
-
-        Called after appending the assistant response so images don't persist
-        across turns. With *only_ids*, strip only those message IDs — the images
-        the just-completed response actually consumed (captured before the
-        request was sent). This leaves intact an image a fast client injected
-        mid-generation for the *next* turn, which the current response never saw.
-        Without *only_ids*, every image is stripped.
-        """
-        with self._lock:
-            for item in self.buffer:
-                if isinstance(item, RealtimeConversationItemUserMessage):
-                    if only_ids is not None and item.id not in only_ids:
-                        continue
-                    item.content = [p for p in item.content if p.type != "input_image"]
-
-    # ── Compaction internals ──────────────────────────────────
-
-    def _snapshot_for_compaction(
-        self,
-    ) -> tuple[ResponseInputParam, set[str], int]:
-        """Compute the snapshot of items eligible for compaction.
-
-        Caller must hold ``_lock``. Returns
-        ``(serialized_snapshot, marker_ids, n_turns)``. ``marker_ids``
-        identifies the buffer items that may be removed when the splice runs.
-        Always leaves the most recent user turn untouched (it may be in-flight).
-        Returns an empty result if there are fewer than 2 compactable turns.
-        """
-        n_turns = max(0, self._user_turn_count - 1)
-        if n_turns < 2:
-            return [], set(), n_turns
-
-        # Slice up to (but not including) the (n_turns + 1)-th user message.
-        user_seen = 0
-        end_idx = len(self.buffer)
-        for i, entry in enumerate(self.buffer):
-            if isinstance(entry, RealtimeConversationItemUserMessage):
-                user_seen += 1
-                if user_seen == n_turns + 1:
-                    end_idx = i
-                    break
-
-        items_to_compact = self.buffer[:end_idx]
-        marker_ids = {entry.id for entry in items_to_compact if entry.id is not None}
-        snapshot = self._to_responses_api_chat_locked(items=items_to_compact)
-        # Strip media parts so the summarizer doesn't have to handle them.
-        for raw in snapshot:
-            if not isinstance(raw, dict) or raw.get("role") != "user":
-                continue
-            msg: dict[str, Any] = raw  # type: ignore[assignment]
-            content = msg.get("content")
-            if isinstance(content, list):
-                msg["content"] = [
-                    c for c in content if not (isinstance(c, dict) and c.get("type") in {"input_image", "input_audio"})
-                ]
-        return snapshot, marker_ids, n_turns
-
-    def _maybe_trigger_compaction(self, compactor: CompactFn) -> None:
-        """Start a background compaction worker. Bypass silently if one is running.
-
-        Caller must hold ``_lock``.
-        """
-        if self._shutdown.is_set() or self._compact_in_flight:
-            return
-        snapshot, marker_ids, n_turns = self._snapshot_for_compaction()
-        if n_turns < 2 or not marker_ids:
-            return
-        gen = self._gen_counter
-        self._compact_in_flight = True
-        thread = threading.Thread(
-            target=self._compact_worker,
-            args=(compactor, snapshot, marker_ids, gen),
-            daemon=True,
-            name="chat-compact",
-        )
-        self._compact_thread = thread
-        logger.info(
-            "Chat compaction triggered: compacting %d turn(s) (%d item(s)), buffer size=%d",
-            n_turns,
-            len(marker_ids),
-            len(self.buffer),
-        )
-        thread.start()
-
-    def _compact_worker(
-        self,
-        compactor: CompactFn,
-        snapshot: ResponseInputParam,
-        marker_ids: set[str],
-        gen: int,
-    ) -> None:
-        """Worker thread entry point."""
-        try:
-            if self._shutdown.is_set() or self._gen_counter != gen:
-                return
-            try:
-                result = compactor(snapshot)
-            except Exception:
-                logger.exception("Chat compaction failed; chat unchanged")
-                return
-            if not isinstance(result, CompactionResult):
-                logger.error("Compactor must return a CompactionResult, got %r", type(result).__name__)
-                return
-            if self._shutdown.is_set() or self._gen_counter != gen:
-                return
-            self._apply_compaction(result, marker_ids, gen)
-        finally:
-            # Don't clobber the flag if reset/close has advanced the gen.
-            with self._lock:
-                if self._gen_counter == gen:
-                    self._compact_in_flight = False
-
-    def _apply_compaction(
-        self,
-        result: CompactionResult,
-        marker_ids: set[str],
-        gen: int,
-    ) -> None:
-        """Splice the summary in front of items not consumed by compaction.
-
-        FC/FCO pairing is left entirely to :meth:`add_item` / :meth:`append_tool_output`.
-        Compaction only drops items; it never inserts an FC into the buffer.
-        Pending FCs (no FCO yet) stay in ``_pending_tool_calls`` and will be
-        appended adjacent to their FCO when it arrives.
-        """
-        with self._lock:
-            if self._shutdown.is_set() or self._gen_counter != gen:
-                return
-            # Keep FC if its FCO is outside the compacted range -- otherwise
-            # the FCO in `remaining` would be orphaned.
-            fco_call_ids_in_range = {
-                x.call_id
-                for x in self.buffer
-                if isinstance(x, RealtimeConversationItemFunctionCallOutput) and x.id in marker_ids
-            }
-            fc_ids_to_keep = {
-                x.id
-                for x in self.buffer
-                if x.id in marker_ids
-                and isinstance(x, RealtimeConversationItemFunctionCall)
-                and x.call_id not in fco_call_ids_in_range
-            }
-            drop_ids = marker_ids - fc_ids_to_keep
-            remaining = [x for x in self.buffer if x.id not in drop_ids]
-
-            user_msg = make_user_message(result.user_summary)
-            user_msg.id = _generate_id("msg")
-            asst_msg = make_assistant_message(result.assistant_summary)
-            asst_msg.id = _generate_id("msg")
-
-            self.buffer = [user_msg, asst_msg, *remaining]
-            self._user_turn_count = sum(1 for x in self.buffer if isinstance(x, RealtimeConversationItemUserMessage))
-            logger.info(
-                "Chat compaction applied: buffer now %d item(s), %d user turn(s)",
-                len(self.buffer),
-                self._user_turn_count,
-            )
-
-
-# ---------------------------------------------------------------------------
-# Transformers chat message models
-# ---------------------------------------------------------------------------
-
-
-class TransformersToolCallFunction(BaseModel):
-    name: str
-    arguments: dict[str, Any]
-
-
-class TransformersToolCall(BaseModel):
-    type: Literal["function"] = "function"
-    id: str
-    function: TransformersToolCallFunction
-
-
-class TransformersSystemMessage(BaseModel):
-    role: Literal["system"] = "system"
-    content: str
-
-
-class TransformersUserMessage(BaseModel):
-    role: Literal["user"] = "user"
-    content: str | list[dict[str, Any]]
-
-
-class TransformersAssistantMessage(BaseModel):
-    role: Literal["assistant"] = "assistant"
-    content: str
-
-
-class TransformersFunctionCallMessage(BaseModel):
-    role: Literal["assistant"] = "assistant"
-    tool_calls: list[TransformersToolCall]
-
-
-class TransformersToolMessage(BaseModel):
-    role: Literal["tool"] = "tool"
-    tool_call_id: str
-    name: str
-    content: str
-
-
-TransformersChatMessage = Union[
-    TransformersSystemMessage,
-    TransformersUserMessage,
-    TransformersAssistantMessage,
-    TransformersFunctionCallMessage,
-    TransformersToolMessage,
-]
-
-
-# ---------------------------------------------------------------------------
-# Factory helpers -- hide verbose constructors behind simple calls
-# ---------------------------------------------------------------------------
-
-
-def make_user_message(text: str) -> RealtimeConversationItemUserMessage:
-    return RealtimeConversationItemUserMessage(
-        type="message",
-        role="user",
-        content=[UserContent(type="input_text", text=text)],
-    )
-
-
-def make_user_audio_message(audio_b64: str) -> RealtimeConversationItemUserMessage:
-    return RealtimeConversationItemUserMessage(
-        type="message",
-        role="user",
-        content=[UserContent(type="input_audio", audio=audio_b64)],
-    )
-
-
-def make_assistant_message(text: str) -> RealtimeConversationItemAssistantMessage:
-    return RealtimeConversationItemAssistantMessage(
-        type="message",
-        role="assistant",
-        content=[AssistantContent(type="output_text", text=text)],
-    )
-
-
-def make_system_message(text: str) -> RealtimeConversationItemSystemMessage:
-    return RealtimeConversationItemSystemMessage(
-        type="message",
-        role="system",
-        content=[SystemContent(type="input_text", text=text)],
-    )
-
-
-def add_supported_item(chat: Chat, item: ConversationItem) -> None:
-    """Narrow a protocol ``ConversationItem`` to a :data:`SupportedItem` and add it to *chat*.
-
-    Raises :class:`ChatItemError` on validation failure or unsupported type. Shared
-    by the conversation handler (in-band item injection) and the language-model
-    handlers (seeding an out-of-band response's throwaway chat from ``response.input``).
-    """
-    # call_id on function_call items must be client-supplied: it is referenced later by
-    # function_call_output items, so we cannot silently generate one here.
-    if isinstance(item, RealtimeConversationItemFunctionCall) and (
-        item.call_id is None or not item.call_id.startswith("call_")
-    ):
-        raise ChatItemError("function_call item is missing a call_id. The call_id should start with 'call_'.")
-
-    if isinstance(
-        item,
-        (
-            RealtimeConversationItemSystemMessage,
-            RealtimeConversationItemUserMessage,
-            RealtimeConversationItemAssistantMessage,
-            RealtimeConversationItemFunctionCall,
-            RealtimeConversationItemFunctionCallOutput,
-        ),
-    ):
-        chat.add_item(item)
-        return
-
-    raise ChatItemError(f"Unsupported item type: {getattr(item, 'type', None)}")
-
-
-def build_active_chat(original_chat: Chat, response: RealtimeResponseCreateParams | None) -> Chat:
-    """Build the chat an *out-of-band* response generates against (caller ensures out-of-band).
-
-    Mirrors the OpenAI realtime semantics for ``input``:
-
-    - ``input is None`` -> a read-only **copy of the default conversation** (the
-      out-of-band response reads history but never commits back).
-    - ``input == []`` -> a **fresh, empty chat** (context cleared; only the
-      system prompt, added later by the handler, will be present).
-    - ``input == [...]`` -> a **fresh chat seeded** with those items.
-
-    Raises :class:`ChatItemError` if an ``input`` item fails validation.
-    """
-    if response is not None and response.input is not None:
-        fresh = Chat(original_chat.size)
-        for item in response.input:
-            add_supported_item(fresh, item)
-        return fresh
-    return original_chat.copy()

runtime/src/speech_to_speech/LLM/chat_completions_language_model.py

@@ -1,360 +0,0 @@
-from __future__ import annotations
-
-import json
-import logging
-import time
-from collections.abc import Iterator
-from typing import Any, cast
-
-from openai import Stream
-from openai.types.chat import (
-    ChatCompletionChunk,
-    ChatCompletionContentPartImageParam,
-    ChatCompletionContentPartParam,
-    ChatCompletionContentPartTextParam,
-    ChatCompletionNamedToolChoiceParam,
-    ChatCompletionToolChoiceOptionParam,
-    ChatCompletionToolParam,
-)
-from openai.types.chat.chat_completion_content_part_image_param import ImageURL
-from openai.types.chat.chat_completion_named_tool_choice_param import Function as NamedToolChoiceFunction
-from openai.types.realtime.realtime_conversation_item_assistant_message import (
-    Content as AssistantContent,
-)
-from openai.types.responses import ResponseFunctionToolCall
-from openai.types.shared_params import FunctionDefinition
-
-from speech_to_speech.LLM.base_openai_compatible_language_model import (
-    WARMUP_MAX_RETRIES,
-    AssistantMessage,
-    BaseOpenAICompatibleHandler,
-    ProviderEvent,
-    TextDelta,
-    ToolCall,
-    Usage,
-)
-from speech_to_speech.LLM.chat import Chat
-from speech_to_speech.LLM.compaction_prompt import CompactGenerateFn
-from speech_to_speech.utils.utils import _generate_id
-
-logger = logging.getLogger(__name__)
-
-
-def _to_chat_tools(req_tools: Any) -> list[ChatCompletionToolParam] | None:
-    """Convert Responses-API function tools to Chat-Completions tool format.
-
-    Responses tools are flat ``{type:"function", name, description, parameters}``;
-    Chat Completions nests them under a ``function`` key. Items already in the
-    nested form (or non-function tools) are passed through untouched.
-    """
-    if not req_tools:
-        return None
-    chat_tools: list[ChatCompletionToolParam] = []
-    for t in req_tools:
-        d = t if isinstance(t, dict) else t.model_dump(exclude_none=True)
-        if d.get("type") == "function" and "function" not in d:
-            fn = FunctionDefinition(name=d["name"])
-            if d.get("description") is not None:
-                fn["description"] = d["description"]
-            if d.get("parameters") is not None:
-                fn["parameters"] = d["parameters"]
-            chat_tools.append(ChatCompletionToolParam(type="function", function=fn))
-        else:
-            chat_tools.append(cast("ChatCompletionToolParam", d))
-    return chat_tools
-
-
-def _to_chat_tool_choice(tool_choice: Any) -> ChatCompletionToolChoiceOptionParam:
-    """Convert a Responses-API tool_choice to Chat-Completions form.
-
-    The string forms ("auto"/"required"/"none") are identical across both APIs;
-    only the forced-function object differs (flat ``name`` vs nested ``function``).
-    """
-    if isinstance(tool_choice, dict) and tool_choice.get("type") == "function" and "name" in tool_choice:
-        return ChatCompletionNamedToolChoiceParam(
-            type="function", function=NamedToolChoiceFunction(name=tool_choice["name"])
-        )
-    if tool_choice is not None and not isinstance(tool_choice, (str, dict)):
-        d = tool_choice.model_dump(exclude_none=True)
-        if d.get("type") == "function" and "name" in d:
-            return ChatCompletionNamedToolChoiceParam(type="function", function=NamedToolChoiceFunction(name=d["name"]))
-        return cast("ChatCompletionToolChoiceOptionParam", d)
-    return cast("ChatCompletionToolChoiceOptionParam", tool_choice)
-
-
-def _build_chat_optional_kwargs(req_tools: Any, req_tool_choice: Any) -> dict[str, Any]:
-    """Build Chat Completions tool arguments from the Responses-style session config."""
-    optional_kwargs: dict[str, Any] = {}
-    chat_tools = _to_chat_tools(req_tools)
-    if chat_tools is not None:
-        optional_kwargs["tools"] = chat_tools
-    if req_tool_choice is not None:
-        optional_kwargs["tool_choice"] = _to_chat_tool_choice(req_tool_choice)
-    return optional_kwargs
-
-
-def _to_chat_content_part(
-    part: dict[str, Any],
-    audio_content_type: str = "input_audio",
-) -> ChatCompletionContentPartParam:
-    """Convert one Realtime/transformers content part to Chat Completions shape."""
-    ptype = part.get("type")
-    if ptype == "input_text":
-        return ChatCompletionContentPartTextParam(type="text", text=part.get("text") or "")
-    if ptype == "input_image":
-        raw_url: Any = part.get("image_url")
-        if isinstance(raw_url, dict):
-            image_url = cast("ImageURL", raw_url)
-        else:
-            image_url = ImageURL(url=raw_url)
-            detail = part.get("detail")
-            if detail is not None:
-                image_url["detail"] = detail
-        return ChatCompletionContentPartImageParam(type="image_url", image_url=image_url)
-    if ptype == "input_audio":
-        audio_b64 = part.get("audio") or ""
-        if audio_content_type == "audio_url":
-            return cast(
-                "ChatCompletionContentPartParam",
-                {
-                    "type": "audio_url",
-                    "audio_url": {
-                        "url": f"data:audio/wav;base64,{audio_b64}",
-                    },
-                },
-            )
-        return cast(
-            "ChatCompletionContentPartParam",
-            {
-                "type": "input_audio",
-                "input_audio": {
-                    "data": audio_b64,
-                    "format": "wav",
-                },
-            },
-        )
-    return cast("ChatCompletionContentPartParam", part)
-
-
-def _chat_messages(
-    chat: Chat,
-    audio_content_type: str = "input_audio",
-) -> list[dict[str, Any]]:
-    """Serialise chat history, including media and tool messages, for Chat Completions."""
-    messages = chat.to_transformers_chat()
-    for message in messages:
-        for tool_call in message.get("tool_calls") or []:
-            fn = tool_call.get("function")
-            if fn is not None and not isinstance(fn.get("arguments"), str):
-                fn["arguments"] = json.dumps(fn.get("arguments") or {}, ensure_ascii=False)
-        content = message.get("content")
-        if isinstance(content, list):
-            message["content"] = [
-                _to_chat_content_part(part, audio_content_type=audio_content_type) for part in content
-            ]
-        if message.get("role") == "tool":
-            message.pop("name", None)
-    return messages
-
-
-def _request_chat_completions(
-    *,
-    client: Any,
-    model_name: str,
-    messages: list[dict[str, Any]],
-    stream: bool,
-    extra_body: dict[str, Any] | None,
-    timeout: Any,
-    optional_kwargs: dict[str, Any],
-) -> Any:
-    """Issue a Chat Completions request with consistent streaming usage accounting."""
-    create_kwargs = dict(optional_kwargs)
-    if stream:
-        create_kwargs["stream_options"] = {"include_usage": True}
-    return client.chat.completions.create(
-        model=model_name,
-        messages=messages,
-        stream=stream,
-        extra_body=extra_body,
-        timeout=timeout,
-        **create_kwargs,
-    )
-
-
-def _tool_calls_from_accum(tool_accum: dict[int, dict[str, str]]) -> Iterator[ToolCall]:
-    """Turn accumulated Chat Completions tool deltas into normalized tool-call events."""
-    for index in sorted(tool_accum):
-        entry = tool_accum[index]
-        if not entry["name"]:
-            continue
-        yield ToolCall(
-            item=ResponseFunctionToolCall(
-                type="function_call",
-                name=entry["name"],
-                arguments=entry["args"] or "{}",
-                call_id=_generate_id("call"),
-                id=_generate_id("fc"),
-                status="completed",
-            )
-        )
-
-
-def _iter_chat_stream_events(api_response: Stream[ChatCompletionChunk]) -> Iterator[ProviderEvent]:
-    """Normalize a streaming Chat Completions response."""
-    tool_accum: dict[int, dict[str, str]] = {}
-    usage: Usage | None = None
-    raw_text = ""
-    for chunk in api_response:
-        if chunk.usage is not None:
-            usage = Usage(
-                input_tokens=chunk.usage.prompt_tokens or 0,
-                output_tokens=chunk.usage.completion_tokens or 0,
-            )
-        if not chunk.choices:
-            continue
-        delta = chunk.choices[0].delta
-        if delta.tool_calls:
-            for tool_call in delta.tool_calls:
-                entry = tool_accum.setdefault(tool_call.index, {"name": "", "args": "", "id": ""})
-                if tool_call.id:
-                    entry["id"] = tool_call.id
-                if tool_call.function is not None:
-                    if tool_call.function.name:
-                        entry["name"] = tool_call.function.name
-                    if tool_call.function.arguments:
-                        entry["args"] += tool_call.function.arguments
-        text_piece = delta.content or getattr(delta, "refusal", None)
-        if text_piece:
-            raw_text += text_piece
-            yield TextDelta(text=text_piece)
-
-    if raw_text.strip():
-        yield AssistantMessage(content=[AssistantContent(type="output_text", text=raw_text)])
-    yield from _tool_calls_from_accum(tool_accum)
-    if usage is not None:
-        yield usage
-
-
-def _iter_chat_response_events(api_response: Any) -> Iterator[ProviderEvent]:
-    """Normalize a non-streaming Chat Completions response."""
-    usage = api_response.usage
-    if usage:
-        yield Usage(input_tokens=usage.prompt_tokens or 0, output_tokens=usage.completion_tokens or 0)
-    message = api_response.choices[0].message if api_response.choices else None
-    if message is None:
-        return
-    raw_content = message.content or getattr(message, "refusal", None)
-    if raw_content:
-        yield AssistantMessage(content=[AssistantContent(type="output_text", text=raw_content)])
-        yield TextDelta(text=raw_content)
-    tool_accum: dict[int, dict[str, str]] = {}
-    for tool_call in message.tool_calls or []:
-        tool_accum[len(tool_accum)] = {
-            "name": tool_call.function.name or "",
-            "args": tool_call.function.arguments or "",
-            "id": tool_call.id or "",
-        }
-    yield from _tool_calls_from_accum(tool_accum)
-
-
-class ChatCompletionsApiModelHandler(BaseOpenAICompatibleHandler):
-    """LLM handler that talks to an OpenAI-compatible ``/v1/chat/completions`` server.
-
-    Functionally mirrors :class:`ResponsesApiModelHandler` but uses the mature
-    Chat Completions streaming tool-call protocol (``choices[].delta.tool_calls``)
-    instead of ``/v1/responses``. This is the robust path for vLLM + Qwen tool
-    calling. The conversation is serialised with :meth:`Chat.to_transformers_chat`,
-    which already emits OpenAI chat messages including ``tool_calls``/``tool`` roles.
-    """
-
-    def warmup(self) -> None:
-        logger.info(f"Warming up {self.__class__.__name__}")
-        start = time.time()
-        self.client.with_options(max_retries=WARMUP_MAX_RETRIES).chat.completions.create(
-            model=self.model_name,
-            messages=[
-                {"role": "system", "content": "You are a helpful assistant"},
-                {"role": "user", "content": "Hello"},
-            ],
-            extra_body=self._extra_body,
-            timeout=self.request_timeout,
-        )
-        end = time.time()
-        logger.info(f"{self.__class__.__name__}:  warmed up! time: {(end - start):.3f} s")
-
-    def _build_compaction_generate_fn(self) -> CompactGenerateFn:
-        """Return a generate fn that calls Chat Completions for compaction."""
-        client = self.client
-        model_name = self.model_name
-        timeout = self.request_timeout
-        extra_body = self._extra_body
-
-        def generate(system: str, user: str) -> str:
-            response = client.chat.completions.create(
-                model=model_name,
-                messages=[
-                    {"role": "system", "content": system},
-                    {"role": "user", "content": user},
-                ],
-                extra_body=extra_body,
-                timeout=timeout,
-            )
-            return response.choices[0].message.content or ""
-
-        return generate
-
-    @staticmethod
-    def _to_chat_content_part(
-        part: dict[str, Any],
-        audio_content_type: str = "input_audio",
-    ) -> ChatCompletionContentPartParam:
-        return _to_chat_content_part(part, audio_content_type=audio_content_type)
-
-    @classmethod
-    def _chat_messages(
-        cls,
-        chat: Chat,
-        audio_content_type: str = "input_audio",
-    ) -> list[dict[str, Any]]:
-        return _chat_messages(chat, audio_content_type=audio_content_type)
-
-    # ── base hooks ──────────────────────────────────────────────────────────--
-
-    def _serialize(self, active_chat: Chat) -> list[dict[str, Any]]:
-        return self._chat_messages(active_chat, audio_content_type=self.audio_content_type)
-
-    def _build_optional_kwargs(self, req_tools: Any, req_tool_choice: Any) -> dict[str, Any]:
-        return _build_chat_optional_kwargs(req_tools, req_tool_choice)
-
-    def _request(
-        self,
-        api_input: list[dict[str, Any]],
-        optional_kwargs: dict[str, Any],
-        runtime_config: Any,
-    ) -> Any:
-        extra_body = self._extra_body
-        session_reasoning_effort = getattr(runtime_config.session, "reasoning_effort", None)
-        if session_reasoning_effort:
-            extra_body = {"reasoning_effort": session_reasoning_effort}
-        return _request_chat_completions(
-            client=self.client,
-            model_name=self.model_name,
-            messages=api_input,
-            stream=self.stream,
-            extra_body=extra_body,
-            timeout=self.request_timeout,
-            optional_kwargs=optional_kwargs,
-        )
-
-    def _iter_stream_events(self, api_response: Stream[ChatCompletionChunk]) -> Iterator[ProviderEvent]:
-        yield from _iter_chat_stream_events(api_response)
-
-    def _iter_response_events(self, api_response: Any) -> Iterator[ProviderEvent]:
-        yield from _iter_chat_response_events(api_response)
-
-    @staticmethod
-    def _tool_calls_from_accum(tool_accum: dict[int, dict[str, str]]) -> Iterator[ToolCall]:
-        yield from _tool_calls_from_accum(tool_accum)
-
-    def on_session_end(self) -> None:
-        logger.debug("Chat Completions API language model session state reset")

runtime/src/speech_to_speech/LLM/compaction_prompt.py

@@ -1,181 +0,0 @@
-"""Prompt template and factory for the conversation compaction (history summarization) function.
-
-Compaction reduces an unbounded conversation history to a tight user/assistant
-summary pair, letting the pipeline continue indefinitely without running out of
-context window.  The factory :func:`build_compactor` returns a :data:`CompactFn`
-compatible with :meth:`~speech_to_speech.LLM.chat.Chat.trim_if_needed`.
-
-:data:`CompactGenerateFn` is the backend-agnostic generation interface:
-``(system_prompt: str, user_prompt: str) -> response_text: str``.
-Each handler wraps its own model into this interface and passes it to
-:func:`build_compactor`.
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import re
-from collections.abc import Callable
-from typing import Any
-
-from speech_to_speech.LLM.chat import CompactFn, CompactionResult
-
-logger = logging.getLogger(__name__)
-
-# Callable[[system_prompt, user_prompt], response_text]
-CompactGenerateFn = Callable[[str, str], str]
-
-# ---------------------------------------------------------------------------
-# Prompts
-# ---------------------------------------------------------------------------
-
-COMPACTION_SYSTEM_PROMPT = """\
-You are a conversation memory compressor for a real-time voice AI assistant.
-
-Your task: read a multi-turn conversation and produce a dense summary so the
-assistant can continue naturally, as if it remembers everything that was said.
-
-Output a single JSON object with exactly two string fields:
-  "user_summary"      — 1–5 sentences capturing what the user has been asking
-                        about, any preferences or constraints they have stated,
-                        and where the conversation stands from their perspective.
-  "assistant_summary" — 1–5 sentences capturing what the assistant has
-                        explained, decided, or done (including tool calls and
-                        their results), plus any open questions or commitments.
-
-Rules:
-- Be information-dense: preserve names, numbers, file paths, error messages, and
-  other specifics that would be needed to continue the conversation correctly.
-- Omit small-talk and filler that carries no forward context.
-- Write in third person, past tense
-  (e.g. "The user asked about…", "The assistant provided…").
-- Emit only the JSON object — no markdown, no code fences, no extra keys.\
-"""
-
-COMPACTION_USER_TEMPLATE = """\
-Summarize the following conversation.  Return only the JSON object.
-
---- CONVERSATION START ---
-{conversation}
---- CONVERSATION END ---\
-"""
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def _render_transcript(snapshot: list[Any]) -> str:
-    """Render a ResponseInputParam snapshot as a readable plain-text transcript."""
-    lines: list[str] = []
-    for item in snapshot:
-        if not isinstance(item, dict):
-            continue
-        item_type = item.get("type", "message")
-        role: str = item.get("role", "")
-
-        if role == "system":
-            continue
-
-        if item_type == "function_call":
-            name = item.get("name", "")
-            args = item.get("arguments", "")
-            lines.append(f"[Tool call: {name}({args})]")
-            continue
-
-        if item_type == "function_call_output":
-            out = item.get("output", "")
-            lines.append(f"[Tool result: {out}]")
-            continue
-
-        # Regular user / assistant message
-        content = item.get("content", "")
-        if isinstance(content, list):
-            text = " ".join(
-                c.get("text", "")
-                for c in content
-                if isinstance(c, dict) and c.get("type") in ("input_text", "output_text")
-            ).strip()
-        elif isinstance(content, str):
-            text = content.strip()
-        else:
-            continue
-
-        if text:
-            label = role.capitalize() if role else "Unknown"
-            lines.append(f"{label}: {text}")
-
-    return "\n\n".join(lines)
-
-
-_JSON_BLOCK_RE = re.compile(r"```(?:json)?\s*(\{.*?\})\s*```", re.DOTALL)
-
-
-def _extract_json(text: str) -> dict[str, Any]:
-    """Extract the first JSON object from *text*, stripping markdown code fences."""
-    try:
-        return json.loads(text)
-    except json.JSONDecodeError:
-        pass
-
-    m = _JSON_BLOCK_RE.search(text)
-    if m:
-        return json.loads(m.group(1))
-
-    start = text.find("{")
-    end = text.rfind("}")
-    if start != -1 and end > start:
-        return json.loads(text[start : end + 1])
-
-    raise ValueError(f"No JSON object found in compaction response: {text!r}")
-
-
-# ---------------------------------------------------------------------------
-# Factory
-# ---------------------------------------------------------------------------
-
-
-def build_compactor(generate_fn: CompactGenerateFn) -> CompactFn:
-    """Return a :data:`CompactFn` that summarizes history using *generate_fn*.
-
-    *generate_fn* is the only model-specific dependency: it receives
-    ``(system_prompt, user_prompt)`` and returns the model's text response.
-    Both :class:`~speech_to_speech.LLM.openai_api_language_model.OpenApiModelHandler`
-    and :class:`~speech_to_speech.LLM.language_model.BaseLanguageModelHandler`
-    subclasses expose a ``_build_compaction_generate_fn()`` method that wraps
-    their respective backend into this interface.
-
-    The returned callable is safe to call from a background thread.
-
-    Args:
-        generate_fn: Backend-agnostic text generation callable.
-
-    Returns:
-        A callable ``(snapshot: ResponseInputParam) -> CompactionResult``.
-    """
-
-    def compact(snapshot: list[Any]) -> CompactionResult:
-        transcript = _render_transcript(snapshot)
-        if not transcript.strip():
-            logger.warning("Compaction called with an empty transcript; returning empty summaries")
-            return CompactionResult(user_summary="", assistant_summary="")
-
-        user_content = COMPACTION_USER_TEMPLATE.format(conversation=transcript)
-        raw_text = generate_fn(COMPACTION_SYSTEM_PROMPT, user_content)
-
-        data = _extract_json(raw_text)
-        user_summary = str(data.get("user_summary", "")).strip()
-        assistant_summary = str(data.get("assistant_summary", "")).strip()
-
-        if not user_summary or not assistant_summary:
-            raise ValueError(f"Compaction response missing required fields. Got: {data!r}")
-
-        logger.debug(
-            "Compaction complete. user=%d chars  assistant=%d chars",
-            len(user_summary),
-            len(assistant_summary),
-        )
-        return CompactionResult(user_summary=user_summary, assistant_summary=assistant_summary)
-
-    return compact

runtime/src/speech_to_speech/LLM/language_model.py

@@ -1,1011 +0,0 @@
-from __future__ import annotations
-
-import logging
-from abc import ABC, abstractmethod
-from collections.abc import Iterable, Iterator, Sized
-from queue import Empty
-from threading import Lock, Thread
-from typing import Any, Literal, Optional, Protocol, runtime_checkable
-
-import torch
-from nltk import sent_tokenize
-from openai.types.realtime.realtime_conversation_item_function_call import (
-    RealtimeConversationItemFunctionCall,
-)
-from openai.types.realtime.realtime_response_create_params import RealtimeResponseCreateParams
-from openai.types.responses import ResponseFunctionToolCall
-from pydantic import BaseModel, ConfigDict, Field
-from transformers import (
-    AutoModelForCausalLM,
-    AutoModelForImageTextToText,
-    AutoProcessor,
-    AutoTokenizer,
-    Pipeline,
-    PreTrainedModel,
-    PreTrainedTokenizerBase,
-    StoppingCriteria,
-    StoppingCriteriaList,
-    TextIteratorStreamer,
-    pipeline,
-)
-
-from speech_to_speech.api.openai_realtime.runtime_config import RuntimeConfig
-from speech_to_speech.baseHandler import BaseHandler
-from speech_to_speech.LLM.chat import (
-    Chat,
-    ChatItemError,
-    build_active_chat,
-    make_assistant_message,
-    make_system_message,
-    make_user_message,
-)
-from speech_to_speech.LLM.compaction_prompt import CompactGenerateFn, build_compactor
-from speech_to_speech.LLM.text_prompt import build_text_system_prompt
-from speech_to_speech.LLM.tool_call.function_call import extract_function_calls_from_text
-from speech_to_speech.LLM.tool_call.function_tool import FunctionTool
-from speech_to_speech.LLM.tool_call.tool_prompt import END_CODE, ENTER_CODE, build_block_regex, build_tool_system_prompt
-from speech_to_speech.LLM.utils import image_url_to_pil, remove_unspeechable, resolve_auto_language
-from speech_to_speech.LLM.voice_prompt import build_voice_system_prompt
-from speech_to_speech.pipeline.cancel_scope import CancelScope
-from speech_to_speech.pipeline.handler_types import LLMIn, LLMOut
-from speech_to_speech.pipeline.messages import (
-    EndOfResponse,
-    GenerateResponseRequest,
-    LLMResponseChunk,
-    TokenUsage,
-)
-from speech_to_speech.pipeline.speculative_turns import SpeculativeTurnTracker
-from speech_to_speech.utils.utils import is_out_of_band, response_wants_audio
-
-try:
-    import mlx.core as mx
-    from mlx_lm import generate as mlx_generate
-    from mlx_lm import load as mlx_load
-    from mlx_lm import stream_generate as mlx_stream_generate
-
-    from speech_to_speech.utils.mlx_lock import MLXLockContext
-
-    HAS_MLX = True
-except ImportError:
-    HAS_MLX = False
-
-try:
-    from mlx_vlm import load as mlx_vlm_load
-    from mlx_vlm import stream_generate as mlx_vlm_stream_generate
-
-    HAS_MLX_VLM = True
-except ImportError:
-    HAS_MLX_VLM = False
-
-logger = logging.getLogger(__name__)
-
-
-@runtime_checkable
-class _Tokenizer(Protocol):
-    """Minimal interface the base class needs from any tokenizer."""
-
-    def encode(self, text: str, /, **kwargs: Any) -> list[int] | Sized: ...
-
-
-@runtime_checkable
-class _Processor(Protocol):
-    """Minimal interface the VLM handler needs from any processor."""
-
-    tokenizer: _Tokenizer
-
-    def apply_chat_template(self, conversation: Any, /, **kwargs: Any) -> Any: ...
-    def __call__(self, **kwargs: Any) -> Any: ...
-
-
-class _CancelCriteria(StoppingCriteria):
-    """Stopping criteria that can be signalled externally to abort generation."""
-
-    def __init__(self) -> None:
-        self._cancelled = False
-
-    def cancel(self) -> None:
-        self._cancelled = True
-
-    def reset(self) -> None:
-        self._cancelled = False
-
-    def __call__(self, input_ids: Any, scores: Any, **kwargs: Any) -> bool:
-        return self._cancelled
-
-
-class StreamContext(BaseModel):
-    """Mutable accumulator passed through ``_stream_tokens`` so the caller
-    can read back generation state after the iterator is exhausted."""
-
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-
-    cancelled: bool = False
-    stopped: bool = False
-    raw_generated_text: str = ""
-    generated_text: str = ""
-    printable_text: str = ""
-    tools: list[ResponseFunctionToolCall] = Field(default_factory=list)
-    function_tools: list[FunctionTool] = Field(default_factory=list)
-    block_regex: Optional[str] = None
-    enter_code: Optional[str] = None
-    end_code: Optional[str] = None
-    input_tokens: int = 0
-    sentence_batch: list[str] = Field(default_factory=list)
-    turn_id: str | None = None
-    turn_revision: int | None = None
-    speech_stopped_at_s: float | None = None
-    cancel_generation: int | None = None
-
-    @property
-    def interrupted(self) -> bool:
-        """True when generation ended early for any reason."""
-        return self.cancelled or self.stopped
-
-
-class BaseLanguageModelHandler(BaseHandler[LLMIn, LLMOut], ABC):
-    """Abstract base for text-only and vision language model handlers.
-
-    Holds shared pipeline logic (streaming, tool extraction, chat management)
-    and delegates model-specific behaviour to two abstract hooks:
-    ``_load_model`` and ``_generate``.
-    """
-
-    _cancel_criteria: _CancelCriteria
-    streamer: Iterable[str]
-    tokenizer: _Tokenizer
-
-    def setup(
-        self,
-        model_name: str = "Qwen/Qwen3-4B-Instruct-2507",
-        device: str = "cuda",
-        torch_dtype: str = "float16",
-        gen_kwargs: dict[str, Any] = {},
-        user_role: str = "user",
-        cancel_scope: CancelScope | None = None,
-        speculative_turns: SpeculativeTurnTracker | None = None,
-        backend: Literal["transformers", "mlx"] = "transformers",
-        enable_thinking: bool = False,
-        stream_batch_sentences: int = 3,
-        enable_lang_prompt: bool = False,
-        compact_history: bool = False,
-        **_kwargs: Any,
-    ) -> None:
-        self.backend = backend
-        self.cancel_scope = cancel_scope
-        self.speculative_turns = speculative_turns
-        self.device = device
-        self.model_name = model_name
-        self.enable_thinking = enable_thinking
-        self.stream_batch_sentences = max(1, stream_batch_sentences)
-        self.enable_lang_prompt = enable_lang_prompt
-
-        self._load_model(model_name, device, torch_dtype, gen_kwargs)
-
-        self.user_role = user_role
-        # Serializes transformers pipe/model.generate calls between the speech
-        # response path and the background compaction worker. MLX paths use
-        # MLXLockContext instead.
-        self._transformers_lock = Lock()
-        self.compactor = build_compactor(self._build_compaction_generate_fn()) if compact_history else None
-
-    def _turn_is_latest(self, turn_id: str | None, turn_revision: int | None) -> bool:
-        return self.speculative_turns is None or self.speculative_turns.is_latest(turn_id, turn_revision)
-
-    def _turn_output_allowed(self, turn_id: str | None, turn_revision: int | None) -> bool:
-        if self.speculative_turns is None:
-            return True
-        return self.speculative_turns.is_latest_after_reopen_grace(turn_id, turn_revision)
-
-    @abstractmethod
-    def _load_model(
-        self,
-        model_name: str,
-        device: str,
-        torch_dtype: str,
-        gen_kwargs: dict[str, Any],
-    ) -> None:
-        """Load the model, tokenizer, and any backend-specific resources."""
-
-    def _build_compaction_generate_fn(self) -> CompactGenerateFn:
-        """Return a ``(system, user) -> text`` callable for compaction.
-
-        Subclasses must override this when ``compact_history=True`` is used.
-        """
-        raise NotImplementedError(
-            f"{type(self).__name__} does not implement _build_compaction_generate_fn. "
-            "Override it or pass compact_history=False."
-        )
-
-    @abstractmethod
-    def _generate(
-        self,
-        chat: Chat,
-        language_code: Optional[str],
-        gen: int | None,
-        ctx: StreamContext,
-        runtime_config: RuntimeConfig | None = None,
-        response: RealtimeResponseCreateParams | None = None,
-    ) -> Iterator[LLMResponseChunk]:
-        """Run model generation, stream sentence chunks, set ``ctx.input_tokens``."""
-
-    # ------------------------------------------------------------------
-    # Generation lifecycle helpers
-    # ------------------------------------------------------------------
-
-    def _finish_mlx_generation(self, token_iter: Any) -> None:
-        """Close the MLX generator to release resources immediately."""
-        token_iter.close()
-
-    def _finish_transformers_generation(self, thread: Thread) -> None:
-        """Signal the stopping criteria, drain the streamer, and join the thread."""
-        self._cancel_criteria.cancel()
-        try:
-            for _ in self.streamer:
-                pass
-        except Empty:
-            pass
-        thread.join(timeout=5.0)
-
-    # ------------------------------------------------------------------
-    # Shared helpers
-    # ------------------------------------------------------------------
-
-    def _apply_instructions(
-        self,
-        chat: Chat,
-        instructions: Optional[str],
-        raw_tools: list[Any] | None,
-        tool_choice: Optional[str],
-        ctx: StreamContext | None = None,
-        wants_audio: bool = True,
-    ) -> None:
-        if not instructions:
-            return
-
-        raw_tools = raw_tools or []
-
-        function_tools = [FunctionTool(**t.model_dump()) for t in raw_tools if t.type == "function"]
-
-        build_system_prompt = build_voice_system_prompt if wants_audio else build_text_system_prompt
-
-        if function_tools and tool_choice != "none":
-            tool_section = build_tool_system_prompt(function_tools, text_only=not wants_audio)
-            full_instructions = build_system_prompt(instructions, tool_section=tool_section)
-            block_regex = build_block_regex()
-            enter_code = ENTER_CODE
-            end_code = END_CODE
-        else:
-            full_instructions = build_system_prompt(instructions)
-            block_regex = None
-            enter_code = None
-            end_code = None
-
-        chat.add_item(make_system_message(full_instructions))
-
-        if ctx is not None:
-            ctx.function_tools = function_tools
-            ctx.block_regex = block_regex
-            ctx.enter_code = enter_code
-            ctx.end_code = end_code
-
-    def _process_printable_text(
-        self,
-        printable_text: str,
-        language_code: Optional[str],
-        tools: list[ResponseFunctionToolCall],
-        ctx: StreamContext,
-        runtime_config: RuntimeConfig | None = None,
-        response: RealtimeResponseCreateParams | None = None,
-    ) -> tuple[list[LLMResponseChunk], list[ResponseFunctionToolCall], str]:
-        """Extract complete code blocks and return complete sentences to yield.
-
-        Returns ``(chunks_to_yield, updated_tools, remaining_printable_text)``.
-        Each element in *chunks_to_yield* is an :class:`LLMResponseChunk`
-        ready to be yielded to the downstream pipeline.
-
-        Sentences are accumulated in ``ctx.sentence_batch`` and only yielded
-        when the batch reaches ``self.stream_batch_sentences``.
-        """
-        chunks: list[LLMResponseChunk] = []
-
-        if ctx.enter_code and ctx.enter_code in printable_text:
-            idx = printable_text.index(ctx.enter_code)
-            before = printable_text[:idx]
-            code_and_after = printable_text[idx:]
-            if before.strip():
-                for s in sent_tokenize(before):
-                    ctx.sentence_batch.append(s)
-            if ctx.sentence_batch:
-                chunks.append(
-                    LLMResponseChunk(
-                        text=" ".join(ctx.sentence_batch),
-                        language_code=language_code,
-                        runtime_config=runtime_config,
-                        response=response,
-                        turn_id=ctx.turn_id,
-                        turn_revision=ctx.turn_revision,
-                        speech_stopped_at_s=ctx.speech_stopped_at_s,
-                        cancel_generation=ctx.cancel_generation,
-                    )
-                )
-                ctx.sentence_batch = []
-            if ctx.block_regex and ctx.end_code and ctx.end_code in code_and_after:
-                stripped, func_calls = extract_function_calls_from_text(
-                    code_and_after,
-                    ctx.block_regex,
-                )
-                parsed_tools: list[ResponseFunctionToolCall] = []
-                for fc in func_calls:
-                    if tools:
-                        logger.warning(
-                            "Skipping extra tool call '%s'; only one tool call is allowed per response",
-                            fc.function_name,
-                        )
-                        continue
-                    try:
-                        tool_call = fc.to_realtime_function_tool_call(ctx.function_tools)
-                    except ValueError as e:
-                        logger.warning("Skipping invalid tool call: %s", e)
-                        continue
-                    tools.append(tool_call)
-                    parsed_tools.append(tool_call)
-                if parsed_tools:
-                    chunks.append(
-                        LLMResponseChunk(
-                            text="",
-                            language_code=language_code,
-                            tools=parsed_tools,
-                            runtime_config=runtime_config,
-                            response=response,
-                            turn_id=ctx.turn_id,
-                            turn_revision=ctx.turn_revision,
-                            speech_stopped_at_s=ctx.speech_stopped_at_s,
-                            cancel_generation=ctx.cancel_generation,
-                        )
-                    )
-                printable_text = stripped
-            else:
-                printable_text = code_and_after
-            return chunks, tools, printable_text
-
-        if printable_text and not response_wants_audio(response) and ctx.enter_code is None:
-            # Text-only with no tool block: stream the raw text immediately. No TTS
-            # means no need to split into sentences, and sent_tokenize would collapse
-            # newlines / markdown. Streaming (vs buffering to the end) keeps the
-            # response interruptible by a new speech turn.
-            chunks.append(
-                LLMResponseChunk(
-                    text=printable_text,
-                    language_code=language_code,
-                    runtime_config=runtime_config,
-                    response=response,
-                    turn_id=ctx.turn_id,
-                    turn_revision=ctx.turn_revision,
-                    speech_stopped_at_s=ctx.speech_stopped_at_s,
-                    cancel_generation=ctx.cancel_generation,
-                )
-            )
-            return chunks, tools, ""
-
-        if printable_text:
-            sentences = sent_tokenize(printable_text)
-            if len(sentences) > 1:
-                for s in sentences[:-1]:
-                    ctx.sentence_batch.append(s)
-                    if len(ctx.sentence_batch) >= self.stream_batch_sentences:
-                        chunks.append(
-                            LLMResponseChunk(
-                                text=" ".join(ctx.sentence_batch),
-                                language_code=language_code,
-                                runtime_config=runtime_config,
-                                response=response,
-                                turn_id=ctx.turn_id,
-                                turn_revision=ctx.turn_revision,
-                                speech_stopped_at_s=ctx.speech_stopped_at_s,
-                                cancel_generation=ctx.cancel_generation,
-                            )
-                        )
-                        ctx.sentence_batch = []
-                printable_text = sentences[-1]
-
-        return chunks, tools, printable_text
-
-    def _check_stop(self, gen: int | None, ctx: StreamContext) -> bool:
-        """Check whether generation should be aborted and mark the reason on *ctx*."""
-        if gen is not None and self.cancel_scope is not None and self.cancel_scope.is_stale(gen):
-            ctx.cancelled = True
-            logger.info("LLM generation cancelled (interruption)")
-            return True
-        if not self._turn_is_latest(ctx.turn_id, ctx.turn_revision):
-            ctx.cancelled = True
-            logger.info("LLM generation cancelled (stale speculative turn)")
-            return True
-        if self.stop_event.is_set():
-            ctx.stopped = True
-            logger.info("LLM generation stopped (shutdown)")
-            return True
-        return False
-
-    def _stream_tokens(
-        self,
-        token_iter: Iterator[Any],
-        gen: int | None,
-        language_code: Optional[str],
-        ctx: StreamContext,
-        runtime_config: RuntimeConfig | None = None,
-        response: RealtimeResponseCreateParams | None = None,
-    ) -> Iterator[LLMResponseChunk]:
-        """Consume *token_iter*, accumulate text in *ctx*, yield sentence chunks."""
-        # Text-only output keeps every character; only audio strips TTS-unfriendly
-        # symbols via remove_unspeechable.
-        wants_audio = response_wants_audio(response)
-        while True:
-            try:
-                token = next(token_iter)
-            except StopIteration:
-                break
-            except Empty:
-                if self._check_stop(gen, ctx):
-                    break
-                continue
-
-            if self._check_stop(gen, ctx):
-                break
-
-            raw_text: str = token.text if hasattr(token, "text") else token
-            ctx.raw_generated_text += raw_text
-            clean = raw_text if not wants_audio else remove_unspeechable(raw_text)
-            ctx.generated_text += clean
-            ctx.printable_text += clean
-            chunks, ctx.tools, ctx.printable_text = self._process_printable_text(
-                ctx.printable_text,
-                language_code,
-                ctx.tools,
-                ctx,
-                runtime_config,
-                response,
-            )
-            if chunks and not self._turn_output_allowed(ctx.turn_id, ctx.turn_revision):
-                ctx.cancelled = True
-                logger.info("LLM generation cancelled (stale speculative turn)")
-                break
-            yield from chunks
-
-        if ctx.sentence_batch and not ctx.interrupted:
-            if ctx.printable_text.strip():
-                ctx.sentence_batch.append(ctx.printable_text.strip())
-                ctx.printable_text = ""
-            if not self._turn_output_allowed(ctx.turn_id, ctx.turn_revision):
-                ctx.cancelled = True
-                logger.info("LLM generation cancelled (stale speculative turn)")
-                return
-            yield LLMResponseChunk(
-                text=" ".join(ctx.sentence_batch),
-                language_code=language_code,
-                runtime_config=runtime_config,
-                response=response,
-                turn_id=ctx.turn_id,
-                turn_revision=ctx.turn_revision,
-                speech_stopped_at_s=ctx.speech_stopped_at_s,
-                cancel_generation=ctx.cancel_generation,
-            )
-            ctx.sentence_batch = []
-
-    # ------------------------------------------------------------------
-    # Main pipeline entry point
-    # ------------------------------------------------------------------
-
-    def process(self, request: LLMIn) -> Iterator[LLMOut]:
-        ctx = StreamContext()
-
-        if not isinstance(request, GenerateResponseRequest):
-            raise TypeError(f"Unexpected request type: {type(request)}")
-
-        ctx.turn_id = request.turn_id
-        ctx.turn_revision = request.turn_revision
-        ctx.speech_stopped_at_s = request.speech_stopped_at_s
-        if not self._turn_is_latest(ctx.turn_id, ctx.turn_revision):
-            logger.info("Skipping stale LLM request for turn=%s rev=%s", ctx.turn_id, ctx.turn_revision)
-            yield EndOfResponse(turn_id=ctx.turn_id, turn_revision=ctx.turn_revision)
-            return
-
-        runtime_config = request.runtime_config
-        response = request.response
-        original_chat = runtime_config.chat
-        out_of_band = is_out_of_band(response)
-        if out_of_band:
-            try:
-                active_chat = build_active_chat(original_chat, response)
-            except ChatItemError as exc:
-                logger.info("Out-of-band response rejected: %s", exc)
-                yield EndOfResponse(turn_id=ctx.turn_id, turn_revision=ctx.turn_revision, error=str(exc))
-                return
-        else:
-            active_chat = original_chat.copy()
-        language_code = request.language_code
-        instructions = (
-            response.instructions if response and response.instructions else runtime_config.session.instructions
-        )
-        tools = response.tools if response and response.tools else runtime_config.session.tools
-        tool_choice = response.tool_choice if response and response.tool_choice else runtime_config.session.tool_choice
-        self._apply_instructions(
-            active_chat,
-            instructions,
-            tools,
-            str(tool_choice) if tool_choice else None,
-            ctx,
-            response_wants_audio(response),
-        )
-        language_code, lang_name = resolve_auto_language(language_code)
-        if lang_name and self.enable_lang_prompt:
-            active_chat.add_item(make_user_message(f"Please reply to my message in {lang_name}."))
-
-        gen = self.cancel_scope.generation if self.cancel_scope else None
-        ctx.cancel_generation = gen
-        # Images the model sees this turn; only these are stripped on write-back,
-        # so an image a fast client injects mid-generation for the next turn
-        # survives (it is not in this serialized snapshot).
-        consumed_image_ids = active_chat.image_message_ids()
-
-        try:
-            yield from self._generate(active_chat, language_code, gen, ctx, runtime_config, response)
-
-            if ctx.stopped:
-                return
-
-            turn_output_allowed = not ctx.cancelled and self._turn_output_allowed(ctx.turn_id, ctx.turn_revision)
-            # Out-of-band responses still emit output, but never write back to the default
-            # conversation (their context was a throwaway chat).
-            commit_allowed = turn_output_allowed and not out_of_band
-            if commit_allowed:
-                original_chat.add_item(make_assistant_message(ctx.generated_text))
-            if commit_allowed and ctx.tools:
-                for t in ctx.tools:
-                    original_chat.add_item(
-                        RealtimeConversationItemFunctionCall(
-                            type="function_call",
-                            id=t.id,
-                            call_id=t.call_id,
-                            name=t.name,
-                            arguments=t.arguments,
-                            status=t.status,
-                        )
-                    )
-            if commit_allowed:
-                original_chat.strip_images(consumed_image_ids)
-                original_chat.trim_if_needed(self.compactor)
-            logger.debug("Clean text: %s", ctx.generated_text)
-            logger.info(f"Tools: {ctx.tools}")
-
-            if turn_output_allowed and ctx.printable_text.strip():
-                yield LLMResponseChunk(
-                    text=ctx.printable_text.strip(),
-                    language_code=language_code,
-                    runtime_config=runtime_config,
-                    response=response,
-                    turn_id=ctx.turn_id,
-                    turn_revision=ctx.turn_revision,
-                    speech_stopped_at_s=ctx.speech_stopped_at_s,
-                    cancel_generation=ctx.cancel_generation,
-                )
-
-            output_tokens = len(self.tokenizer.encode(ctx.raw_generated_text)) if ctx.raw_generated_text else 0
-            if turn_output_allowed and (ctx.input_tokens or output_tokens):
-                yield TokenUsage(
-                    input_tokens=ctx.input_tokens,
-                    output_tokens=output_tokens,
-                    turn_id=ctx.turn_id,
-                    turn_revision=ctx.turn_revision,
-                )
-        except Exception as exc:
-            # Any generation failure must still terminate the response. Without this
-            # the exception would escape process() and no EndOfResponse would be
-            # emitted, leaving st.in_response stuck and locking every later response.
-            logger.exception("LLM generation failed; ending the current response")
-            yield EndOfResponse(
-                turn_id=ctx.turn_id,
-                turn_revision=ctx.turn_revision,
-                cancel_generation=ctx.cancel_generation,
-                error=f"Language model generation failed: {exc}",
-            )
-            return
-        yield EndOfResponse(
-            turn_id=ctx.turn_id,
-            turn_revision=ctx.turn_revision,
-            cancel_generation=ctx.cancel_generation,
-        )
-
-    def on_session_end(self) -> None:
-        logger.debug("Language model session state reset")
-
-
-# ======================================================================
-# Text-only LLM
-# ======================================================================
-
-
-class LanguageModelHandler(BaseLanguageModelHandler):
-    """Text-only language model handler (transformers or MLX backend)."""
-
-    model: PreTrainedModel
-    tokenizer: PreTrainedTokenizerBase
-    gen_kwargs: dict
-    torch_dtype: torch.dtype
-    pipe: Pipeline
-    streamer: TextIteratorStreamer
-
-    def setup(self, **kwargs: Any) -> None:  # type: ignore[override]
-        super().setup(**kwargs)
-        logger.info(f"LLM Backend: {self.backend}")
-        self.warmup()
-
-    def _load_model(self, model_name: str, device: str, torch_dtype: str, gen_kwargs: dict[str, Any]) -> None:
-
-        logger.info("LLM Language Model Handler setup")
-
-        if self.backend == "mlx":
-            if not HAS_MLX:
-                raise ImportError("mlx_lm is required for the 'mlx' backend. Install with: pip install mlx-lm")
-            self.model, self.tokenizer = mlx_load(model_name)  # type: ignore[assignment, misc]
-            self.gen_kwargs = gen_kwargs
-        else:
-            self.torch_dtype = getattr(torch, torch_dtype)
-            self.tokenizer = AutoTokenizer.from_pretrained(model_name)  # type: ignore[assignment]
-            self.model = AutoModelForCausalLM.from_pretrained(
-                model_name, torch_dtype=torch_dtype, trust_remote_code=True
-            ).to(device)  # type: ignore[arg-type]
-            self.pipe = pipeline(  # type: ignore[call-overload]
-                "text-generation", model=self.model, tokenizer=self.tokenizer, device=device
-            )
-            self.streamer = TextIteratorStreamer(
-                self.tokenizer,  # type: ignore[arg-type]
-                skip_prompt=True,
-                skip_special_tokens=True,
-                timeout=1.0,
-            )
-            self._cancel_criteria = _CancelCriteria()
-            self.gen_kwargs = {
-                "streamer": self.streamer,
-                "return_full_text": False,
-                "stopping_criteria": StoppingCriteriaList([self._cancel_criteria]),
-                **gen_kwargs,
-            }
-
-    def _generate(
-        self,
-        chat: Chat,
-        language_code: Optional[str],
-        gen: int | None,
-        ctx: StreamContext,
-        runtime_config: RuntimeConfig | None = None,
-        response: RealtimeResponseCreateParams | None = None,
-    ) -> Iterator[LLMResponseChunk]:
-        chat_messages = chat.to_transformers_chat()
-        chat_input = self.tokenizer.apply_chat_template(chat_messages, tokenize=True)
-        ctx.input_tokens += len(chat_input if isinstance(chat_input, list) else chat_input["input_ids"])  # type: ignore[index]
-        logger.debug("Prompt token count: %d", ctx.input_tokens)
-
-        chat_prompt = self.tokenizer.apply_chat_template(
-            chat_messages, tokenize=False, add_generation_prompt=True, enable_thinking=False
-        )
-
-        if self.backend == "mlx":
-            with MLXLockContext(handler_name="MLX-LLM", timeout=10.0):
-                token_iter = mlx_stream_generate(
-                    self.model,  # type: ignore[arg-type]
-                    self.tokenizer,  # type: ignore[arg-type]
-                    chat_prompt,  # type: ignore[arg-type]
-                    max_tokens=self.gen_kwargs["max_new_tokens"],
-                )
-                yield from self._stream_tokens(token_iter, gen, language_code, ctx, runtime_config, response)
-                self._finish_mlx_generation(token_iter)
-            try:
-                mx.clear_cache()
-            except Exception:
-                pass
-            torch.mps.empty_cache()
-        else:
-            self._cancel_criteria.reset()
-            lock = self._transformers_lock
-
-            def _locked_pipe() -> None:
-                with lock:
-                    self.pipe(chat_prompt, **self.gen_kwargs)
-
-            thread = Thread(target=_locked_pipe)
-            thread.start()
-            yield from self._stream_tokens(self.streamer, gen, language_code, ctx, runtime_config, response)
-            self._finish_transformers_generation(thread)
-            if self.device == "mps":
-                torch.mps.empty_cache()
-
-    def _build_compaction_generate_fn(self) -> CompactGenerateFn:
-        if self.backend == "mlx":
-            model = self.model
-            tokenizer = self.tokenizer
-            max_tokens = 1024
-
-            def generate_mlx(system: str, user: str) -> str:
-                messages = [{"role": "system", "content": system}, {"role": "user", "content": user}]
-                prompt = tokenizer.apply_chat_template(  # type: ignore[union-attr]
-                    messages, tokenize=False, add_generation_prompt=True
-                )
-                with MLXLockContext(handler_name="MLX-compact", timeout=10.0):
-                    return mlx_generate(model, tokenizer, prompt=prompt, max_tokens=max_tokens, verbose=False)  # type: ignore[arg-type]
-
-            return generate_mlx
-        else:
-            pipe = self.pipe
-            tokenizer = self.tokenizer
-            gen_kwargs = {
-                k: v
-                for k, v in self.gen_kwargs.items()
-                if k not in ("streamer", "stopping_criteria", "max_new_tokens", "return_full_text")
-            }
-            max_new_tokens = 1024
-            lock = self._transformers_lock
-
-            def generate_transformers(system: str, user: str) -> str:
-                messages = [{"role": "system", "content": system}, {"role": "user", "content": user}]
-                prompt = tokenizer.apply_chat_template(  # type: ignore[union-attr]
-                    messages, tokenize=False, add_generation_prompt=True, enable_thinking=False
-                )
-                with lock:
-                    result = pipe(prompt, return_full_text=False, max_new_tokens=max_new_tokens, **gen_kwargs)
-                return result[0]["generated_text"]  # type: ignore[index]
-
-            return generate_transformers
-
-    def warmup(self) -> None:
-        logger.info(f"Warming up {self.__class__.__name__}")
-
-        dummy_input_text = "Repeat the word 'home'."
-        dummy_chat = [{"role": self.user_role, "content": dummy_input_text}]
-        n_steps = 2
-
-        if self.backend == "mlx":
-            for _ in range(n_steps):
-                prompt = self.tokenizer.apply_chat_template(dummy_chat, tokenize=False)
-                mlx_generate(
-                    self.model,  # type: ignore[arg-type]
-                    self.tokenizer,  # type: ignore[arg-type]
-                    prompt=prompt,  # type: ignore[arg-type]
-                    max_tokens=self.gen_kwargs["max_new_tokens"],
-                    verbose=False,
-                )
-        else:
-            warmup_gen_kwargs = {
-                "min_new_tokens": self.gen_kwargs["min_new_tokens"],
-                "max_new_tokens": self.gen_kwargs["max_new_tokens"],
-                **self.gen_kwargs,
-            }
-
-            if self.device == "cuda":
-                start_event = torch.cuda.Event(enable_timing=True)
-                end_event = torch.cuda.Event(enable_timing=True)
-                torch.cuda.synchronize()
-                start_event.record()
-
-            for _ in range(n_steps):
-                thread = Thread(target=self.pipe, args=(dummy_chat,), kwargs=warmup_gen_kwargs)
-                thread.start()
-                for _ in self.streamer:
-                    pass
-
-            if self.device == "cuda":
-                end_event.record()
-                torch.cuda.synchronize()
-                logger.info(
-                    f"{self.__class__.__name__}:  warmed up! time: {start_event.elapsed_time(end_event) * 1e-3:.3f} s"
-                )
-
-
-# ======================================================================
-# Vision Language Model (VLM)
-# ======================================================================
-
-
-class VisionLanguageModelHandler(BaseLanguageModelHandler):
-    """Vision language model handler (transformers or MLX-VLM backend)."""
-
-    model: PreTrainedModel
-    tokenizer: PreTrainedTokenizerBase
-    processor: _Processor
-    gen_kwargs: dict
-    torch_dtype: torch.dtype
-    streamer: TextIteratorStreamer
-
-    def setup(self, **kwargs: Any) -> None:  # type: ignore[override]
-        super().setup(**kwargs)
-        logger.info(f"VLM Backend: {self.backend}")
-
-    def _load_model(self, model_name: str, device: str, torch_dtype: str, gen_kwargs: dict[str, Any]) -> None:
-
-        logger.info("VLM Language Model Handler setup")
-
-        if self.backend == "mlx":
-            if not HAS_MLX_VLM:
-                raise ImportError("mlx-vlm is required for MLX VLM models. Install with: pip install mlx-vlm")
-            self.model, self.processor = mlx_vlm_load(model_name)  # type: ignore[assignment]
-            self.tokenizer = self.processor.tokenizer  # type: ignore[assignment]
-            self.gen_kwargs = gen_kwargs
-        else:
-            self.torch_dtype = getattr(torch, torch_dtype)
-            self.processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)  # type: ignore[assignment]
-            self.tokenizer = self.processor.tokenizer  # type: ignore[assignment]
-            self.model = AutoModelForImageTextToText.from_pretrained(
-                model_name, torch_dtype=self.torch_dtype, trust_remote_code=True
-            ).to(device)  # type: ignore[arg-type]
-            self.streamer = TextIteratorStreamer(
-                self.tokenizer,  # type: ignore[arg-type]
-                skip_prompt=True,
-                skip_special_tokens=True,
-                timeout=1.0,
-            )
-            self._cancel_criteria = _CancelCriteria()
-            self.gen_kwargs = gen_kwargs
-
-    def _generate(
-        self,
-        chat: Chat,
-        language_code: Optional[str],
-        gen: int | None,
-        ctx: StreamContext,
-        runtime_config: RuntimeConfig | None = None,
-        response: RealtimeResponseCreateParams | None = None,
-    ) -> Iterator[LLMResponseChunk]:
-        prepared = chat.to_transformers_chat()
-        if self.backend == "mlx":
-            images, formatted_prompt = self._prepare_mlx_vlm_inputs(prepared)
-            ctx.input_tokens += len(self.tokenizer.encode(formatted_prompt))
-            logger.debug("MLX VLM prompt token count: %d", ctx.input_tokens)
-
-            with MLXLockContext(handler_name="MLX-VLM", timeout=10.0):
-                token_iter = mlx_vlm_stream_generate(  # type: ignore[arg-type]
-                    self.model,
-                    self.processor,
-                    formatted_prompt,
-                    images or None,
-                    max_tokens=self.gen_kwargs.get("max_new_tokens", 1024),
-                    enable_thinking=self.enable_thinking,
-                )
-                yield from self._stream_tokens(token_iter, gen, language_code, ctx, runtime_config, response)
-                self._finish_mlx_generation(token_iter)
-            try:
-                mx.clear_cache()
-            except Exception:
-                pass
-            torch.mps.empty_cache()
-        else:
-            inputs, input_tokens = self._prepare_vlm_inputs(prepared)
-            ctx.input_tokens += input_tokens
-            logger.debug("VLM prompt token count: %d", ctx.input_tokens)
-
-            self._cancel_criteria.reset()
-            generate_kwargs = {
-                **inputs,
-                "max_new_tokens": self.gen_kwargs.get("max_new_tokens", 1024),
-                "streamer": self.streamer,
-                "stopping_criteria": StoppingCriteriaList([self._cancel_criteria]),
-            }
-            lock = self._transformers_lock
-
-            def _locked_generate() -> None:
-                with lock:
-                    self.model.generate(**generate_kwargs)  # type: ignore[union-attr,operator]
-
-            thread = Thread(target=_locked_generate)
-            thread.start()
-            yield from self._stream_tokens(self.streamer, gen, language_code, ctx, runtime_config, response)
-            self._finish_transformers_generation(thread)
-            if self.device == "mps":
-                torch.mps.empty_cache()
-
-    def _build_compaction_generate_fn(self) -> CompactGenerateFn:
-        if self.backend == "mlx":
-            model = self.model
-            processor = self.processor
-            tokenizer = self.tokenizer
-            max_tokens = 1024
-
-            def generate_mlx(system: str, user: str) -> str:
-                messages = [{"role": "system", "content": system}, {"role": "user", "content": user}]
-                formatted_prompt = processor.apply_chat_template(  # type: ignore[union-attr]
-                    messages, tokenize=False, add_generation_prompt=True
-                )
-                with MLXLockContext(handler_name="MLX-VLM-compact", timeout=10.0):
-                    token_iter = mlx_vlm_stream_generate(  # type: ignore[arg-type]
-                        model, processor, formatted_prompt, None, max_tokens=max_tokens
-                    )
-                    return "".join(t.text if hasattr(t, "text") else str(t) for t in token_iter)
-
-            return generate_mlx
-        else:
-            model = self.model
-            processor = self.processor
-            tokenizer = self.tokenizer
-            device = self.device
-            max_new_tokens = 1024
-            lock = self._transformers_lock
-
-            def generate_transformers(system: str, user: str) -> str:
-                messages = [{"role": "system", "content": system}, {"role": "user", "content": user}]
-                prompt = processor.apply_chat_template(  # type: ignore[union-attr]
-                    messages, tokenize=False, add_generation_prompt=True
-                )
-                inputs = processor(text=[prompt], return_tensors="pt", padding=True).to(device)  # type: ignore[operator]
-                input_len = inputs["input_ids"].shape[1]
-                with lock:
-                    output_ids = model.generate(**inputs, max_new_tokens=max_new_tokens)  # type: ignore[union-attr,operator]
-                return str(tokenizer.decode(output_ids[0][input_len:], skip_special_tokens=True))  # type: ignore[union-attr]
-
-            return generate_transformers
-
-    def _prepare_vlm_inputs(self, chat_messages: list[dict[str, Any]]) -> tuple[Any, int]:
-        """Build processor inputs for transformers VLM generation.
-
-        Converts Realtime-style content (``input_text``/``input_image``) to the
-        transformers chat-template format (``text``/``image``) and decodes image
-        URLs to PIL images.
-        """
-        converted_messages = []
-        images = []
-        for msg in chat_messages:
-            content = msg.get("content")
-            if isinstance(content, list):
-                parts = []
-                for p in content:
-                    if p.get("type") == "input_text":
-                        parts.append({"type": "text", "text": p["text"]})
-                    elif p.get("type") == "input_image":
-                        pil_img = image_url_to_pil(p["image_url"])
-                        images.append(pil_img)
-                        parts.append({"type": "image", "image": pil_img})
-                converted_messages.append({"role": msg["role"], "content": parts})
-            else:
-                converted_messages.append(msg)
-
-        text_prompt = self.processor.apply_chat_template(
-            converted_messages,
-            tokenize=False,
-            add_generation_prompt=True,
-        )
-        inputs = self.processor(
-            text=[text_prompt],
-            images=images if images else None,
-            return_tensors="pt",
-            padding=True,
-        ).to(self.device)
-        return inputs, len(inputs["input_ids"][0])
-
-    def _prepare_mlx_vlm_inputs(self, chat_messages: list[dict[str, Any]]) -> tuple[list[Any], str]:
-        """Build formatted prompt and extract PIL images for MLX VLM generation.
-
-        Converts Realtime-style content (``input_text``/``input_image``) to the
-        HF chat-template format and uses the processor's own ``apply_chat_template``
-        for full multi-turn support.
-
-        Returns ``(images, formatted_prompt)`` ready for ``mlx_vlm.stream_generate``.
-        """
-        images = []
-        converted_messages = []
-        for msg in chat_messages:
-            content = msg.get("content")
-            if isinstance(content, list):
-                parts = []
-                for p in content:
-                    if p.get("type") == "input_text":
-                        parts.append({"type": "text", "text": p["text"]})
-                    elif p.get("type") == "input_image":
-                        images.append(image_url_to_pil(p["image_url"]))
-                        parts.append({"type": "image"})
-                converted_messages.append({"role": msg["role"], "content": parts})
-            else:
-                converted_messages.append(msg)
-
-        formatted_prompt = self.processor.apply_chat_template(
-            converted_messages,
-            tokenize=False,
-            add_generation_prompt=True,
-        )
-        return images, formatted_prompt

runtime/src/speech_to_speech/LLM/lm_output_processor.py

@@ -1,148 +0,0 @@
-"""
-LLM Output Processor
-
-Intercepts LLM output to:
-1. Extract tool calls and send them via text_output_queue
-2. Forward clean text to TTS pipeline
-"""
-
-from __future__ import annotations
-
-import logging
-from collections.abc import Iterator
-from queue import Queue
-
-from speech_to_speech.baseHandler import BaseHandler
-from speech_to_speech.pipeline.events import AssistantTextEvent, ResponseFailedEvent, TokenUsageEvent
-from speech_to_speech.pipeline.handler_types import LLMOut, TTSIn
-from speech_to_speech.pipeline.messages import EndOfResponse, LLMResponseChunk, TokenUsage, TTSInput
-from speech_to_speech.pipeline.queue_types import TextEventItem
-from speech_to_speech.pipeline.speculative_turns import SpeculativeTurnTracker
-from speech_to_speech.utils.utils import response_wants_audio
-
-logger = logging.getLogger(__name__)
-
-
-class LMOutputProcessor(BaseHandler[LLMOut, TTSIn]):
-    """
-    Processes LLM output to extract tool calls and forward clean text to TTS.
-
-    Input: :class:`LLMResponseChunk`, :class:`TokenUsage`, or :class:`EndOfResponse` from LLM
-    Output: :class:`TTSInput` or :class:`EndOfResponse` to TTS
-    Side effect: Sends :class:`AssistantTextEvent` / :class:`TokenUsageEvent` to text_output_queue
-    """
-
-    def setup(
-        self,
-        text_output_queue: Queue[TextEventItem] | None = None,
-        speculative_turns: SpeculativeTurnTracker | None = None,
-    ) -> None:
-        """
-        Initialize the processor.
-
-        Args:
-            text_output_queue: Queue to send text messages and tool calls
-        """
-        self.text_output_queue = text_output_queue
-        self.speculative_turns = speculative_turns
-
-    def _turn_output_allowed(self, turn_id: str | None, turn_revision: int | None) -> bool:
-        if self.speculative_turns is None:
-            return True
-        return self.speculative_turns.is_latest_after_reopen_grace(turn_id, turn_revision)
-
-    def process(self, lm_output: LLMOut) -> Iterator[TTSIn]:
-        """
-        Process LLM output: send text/tools to WebSocket, forward clean text to TTS.
-
-        Yields:
-            :class:`TTSInput` or :class:`EndOfResponse` for TTS
-        """
-        if isinstance(lm_output, TokenUsage):
-            if not self._turn_output_allowed(
-                lm_output.turn_id,
-                lm_output.turn_revision,
-            ):
-                logger.debug(
-                    "Dropping stale token usage for turn=%s rev=%s", lm_output.turn_id, lm_output.turn_revision
-                )
-                return
-            if self.text_output_queue is not None:
-                self.text_output_queue.put(
-                    TokenUsageEvent(
-                        input_tokens=lm_output.input_tokens or 0,
-                        output_tokens=lm_output.output_tokens or 0,
-                        turn_id=lm_output.turn_id,
-                        turn_revision=lm_output.turn_revision,
-                    )
-                )
-            return
-
-        if isinstance(lm_output, EndOfResponse):
-            if not self._turn_output_allowed(
-                lm_output.turn_id,
-                lm_output.turn_revision,
-            ):
-                logger.debug(
-                    "Dropping stale end-of-response for turn=%s rev=%s",
-                    lm_output.turn_id,
-                    lm_output.turn_revision,
-                )
-                return
-            # A failed generation (e.g. invalid out-of-band input) closes the response as
-            # "failed" via the text side-channel, then falls through to emit the normal
-            # EndOfResponse so the audio path still re-enables listening / releases the slot.
-            if lm_output.error and self.text_output_queue is not None:
-                self.text_output_queue.put(
-                    ResponseFailedEvent(
-                        message=lm_output.error,
-                        turn_id=lm_output.turn_id,
-                        turn_revision=lm_output.turn_revision,
-                    )
-                )
-            yield EndOfResponse(
-                turn_id=lm_output.turn_id,
-                turn_revision=lm_output.turn_revision,
-                cancel_generation=lm_output.cancel_generation,
-            )
-            return
-
-        if not isinstance(lm_output, LLMResponseChunk):
-            logger.warning("LMOutputProcessor received unexpected type: %s", type(lm_output))
-            return
-
-        if not self._turn_output_allowed(
-            lm_output.turn_id,
-            lm_output.turn_revision,
-        ):
-            logger.debug("Dropping stale LLM chunk for turn=%s rev=%s", lm_output.turn_id, lm_output.turn_revision)
-            return
-
-        logger.debug(f"LM processor: text='{lm_output.text}', tools={lm_output.tools}")
-
-        if self.text_output_queue is not None:
-            event = AssistantTextEvent(
-                text=lm_output.text,
-                turn_id=lm_output.turn_id,
-                turn_revision=lm_output.turn_revision,
-                cancel_generation=lm_output.cancel_generation,
-            )
-            if lm_output.tools:
-                event.tools = lm_output.tools
-                logger.info(f"Sending to clients: text='{lm_output.text}', tools={[t.name for t in lm_output.tools]}")
-            else:
-                logger.debug(f"Sending to clients: text='{lm_output.text}' (no tools)")
-            self.text_output_queue.put(event)
-
-        if lm_output.text and response_wants_audio(lm_output.response):
-            logger.debug(f"Forwarding to TTS: '{lm_output.text}'")
-            yield TTSInput(
-                text=lm_output.text,
-                language_code=lm_output.language_code,
-                runtime_config=lm_output.runtime_config,
-                response=lm_output.response,
-                turn_id=lm_output.turn_id,
-                turn_revision=lm_output.turn_revision,
-                speech_stopped_at_s=lm_output.speech_stopped_at_s,
-                cancel_generation=lm_output.cancel_generation,
-            )

runtime/src/speech_to_speech/LLM/responses_api_language_model.py

@@ -1,201 +0,0 @@
-from __future__ import annotations
-
-import logging
-import time
-from collections.abc import Iterator
-from typing import Any
-
-from openai import Stream
-from openai.types.realtime.realtime_conversation_item_assistant_message import (
-    Content as AssistantContent,
-)
-from openai.types.responses import (
-    ResponseCompletedEvent,
-    ResponseFunctionToolCall,
-    ResponseOutputItemDoneEvent,
-    ResponseOutputMessage,
-    ResponseTextDeltaEvent,
-)
-
-from speech_to_speech.LLM.base_openai_compatible_language_model import (
-    WARMUP_MAX_RETRIES,
-    AssistantMessage,
-    BaseOpenAICompatibleHandler,
-    ProviderEvent,
-    TextDelta,
-    ToolCall,
-    Usage,
-)
-from speech_to_speech.LLM.chat import Chat
-from speech_to_speech.LLM.chat_completions_language_model import (
-    _build_chat_optional_kwargs,
-    _chat_messages,
-    _iter_chat_response_events,
-    _iter_chat_stream_events,
-    _request_chat_completions,
-)
-from speech_to_speech.LLM.compaction_prompt import CompactGenerateFn
-from speech_to_speech.utils.utils import _generate_id
-
-logger = logging.getLogger(__name__)
-
-
-class ResponsesApiModelHandler(BaseOpenAICompatibleHandler):
-    """LLM handler that talks to an OpenAI ``/v1/responses`` server."""
-
-    def warmup(self) -> None:
-        logger.info(f"Warming up {self.__class__.__name__}")
-        start = time.time()
-        self.client.with_options(max_retries=WARMUP_MAX_RETRIES).responses.create(
-            model=self.model_name,
-            input=[
-                {
-                    "type": "message",
-                    "role": "system",
-                    "content": [{"type": "input_text", "text": "You are a helpful assistant"}],
-                },
-                {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Hello"}]},
-            ],
-            timeout=self.request_timeout,
-        )
-        end = time.time()
-        logger.info(f"{self.__class__.__name__}:  warmed up! time: {(end - start):.3f} s")
-
-    def _build_compaction_generate_fn(self) -> CompactGenerateFn:
-        """Return a generate fn that calls the Responses API for compaction."""
-        client = self.client
-        model_name = self.model_name
-        timeout = self.request_timeout
-
-        def generate(system: str, user: str) -> str:
-            response = client.responses.create(
-                model=model_name,
-                input=[
-                    {
-                        "type": "message",
-                        "role": "system",
-                        "content": [{"type": "input_text", "text": system}],
-                    },
-                    {
-                        "type": "message",
-                        "role": "user",
-                        "content": [{"type": "input_text", "text": user}],
-                    },
-                ],
-                timeout=timeout,
-            )
-            return response.output_text
-
-        return generate
-
-    def _build_audio_optional_kwargs(
-        self,
-        response: Any,
-        req_tools: Any,
-        req_tool_choice: Any,
-    ) -> dict[str, Any]:
-        kwargs = _build_chat_optional_kwargs(req_tools, req_tool_choice)
-        max_tokens = getattr(response, "max_output_tokens", None) if response is not None else None
-        kwargs.setdefault("max_tokens", max_tokens or self.audio_max_tokens)
-        kwargs.setdefault("temperature", self.audio_temperature)
-        return kwargs
-
-    def _serialize_audio(self, active_chat: Chat) -> list[dict[str, Any]]:
-        return _chat_messages(active_chat, audio_content_type=self.audio_content_type)
-
-    def _request_audio(
-        self,
-        api_input: list[dict[str, Any]],
-        optional_kwargs: dict[str, Any],
-        runtime_config: Any,
-    ) -> Any:
-        extra_body = self._extra_body
-        session_reasoning_effort = getattr(runtime_config.session, "reasoning_effort", None)
-        if session_reasoning_effort:
-            extra_body = {"reasoning_effort": session_reasoning_effort}
-        return _request_chat_completions(
-            client=self.client,
-            model_name=self.model_name,
-            messages=api_input,
-            stream=self.stream,
-            extra_body=extra_body,
-            timeout=self.request_timeout,
-            optional_kwargs=optional_kwargs,
-        )
-
-    def _iter_audio_events(self, api_response: Any) -> Iterator[ProviderEvent]:
-        if self.stream:
-            yield from _iter_chat_stream_events(api_response)
-        else:
-            yield from _iter_chat_response_events(api_response)
-
-    # ── base hooks ──────────────────────────────────────────────────────────--
-
-    def _serialize(self, active_chat: Chat) -> Any:
-        return active_chat.to_responses_api_chat()
-
-    def _build_optional_kwargs(self, req_tools: Any, req_tool_choice: Any) -> dict[str, Any]:
-        optional_kwargs: dict[str, Any] = {}
-        if req_tools is not None:
-            optional_kwargs["tools"] = req_tools
-        if req_tool_choice is not None:
-            optional_kwargs["tool_choice"] = req_tool_choice
-        return optional_kwargs
-
-    def _request(self, api_input: Any, optional_kwargs: dict[str, Any], runtime_config: Any) -> Any:
-        extra_body = self._extra_body
-        session_reasoning_effort = getattr(runtime_config.session, "reasoning_effort", None)
-        if session_reasoning_effort:
-            extra_body = {"reasoning_effort": session_reasoning_effort}
-        return self.client.responses.create(
-            model=self.model_name,
-            input=api_input,
-            stream=self.stream,
-            extra_body=extra_body,
-            timeout=self.request_timeout,
-            **optional_kwargs,
-        )
-
-    @staticmethod
-    def _assistant_content(content: Any) -> list[AssistantContent]:
-        return [
-            AssistantContent(type="output_text", text=c.text if c.type == "output_text" else c.refusal) for c in content
-        ]
-
-    def _iter_stream_events(self, api_response: Stream) -> Iterator[ProviderEvent]:
-        for raw_event in api_response:
-            if isinstance(raw_event, ResponseTextDeltaEvent):
-                yield TextDelta(text=raw_event.delta)
-            elif isinstance(raw_event, ResponseOutputItemDoneEvent):
-                item = raw_event.item
-                if isinstance(item, ResponseFunctionToolCall):
-                    item.call_id = _generate_id("call")
-                    item.id = _generate_id("fc")
-                    yield ToolCall(item=item)
-                elif isinstance(item, ResponseOutputMessage):
-                    yield AssistantMessage(content=self._assistant_content(item.content))
-            elif isinstance(raw_event, ResponseCompletedEvent):
-                usage = getattr(raw_event.response, "usage", None)
-                if usage:
-                    yield Usage(input_tokens=usage.input_tokens or 0, output_tokens=usage.output_tokens or 0)
-
-    def _iter_response_events(self, api_response: Any) -> Iterator[ProviderEvent]:
-        usage = api_response.usage
-        if usage:
-            yield Usage(input_tokens=usage.input_tokens or 0, output_tokens=usage.output_tokens or 0)
-        for message in api_response.output:
-            if isinstance(message, ResponseFunctionToolCall):
-                message.call_id = _generate_id("call")
-                message.id = _generate_id("fc")
-                yield ToolCall(item=message)
-            elif isinstance(message, ResponseOutputMessage):
-                yield AssistantMessage(content=self._assistant_content(message.content))
-                # Text-only keeps every character; the base applies remove_unspeechable
-                # for audio. Only output_text parts are spoken (refusals are stored).
-                raw = "".join(c.text for c in message.content if c.type == "output_text")
-                yield TextDelta(text=raw)
-            else:
-                logger.warning(f"Not supported message type: {message.type}")
-
-    def on_session_end(self) -> None:
-        logger.debug("OpenAI API language model session state reset")

runtime/src/speech_to_speech/LLM/text_prompt.py

@@ -1,44 +0,0 @@
-"""Text-channel system prompt: lead + session prompt + tail (strongest constraints last)."""
-
-TEXT_SYSTEM_PROMPT_LEAD = """\
-You are a helpful assistant in a text conversation.
-"""
-
-TEXT_SYSTEM_PROMPT_TAIL = """\
-## Text Rules
-- Write clearly and directly. Match length to the request: concise for simple questions, fuller when the task genuinely needs it.
-- Use markdown when it helps (lists, code blocks, tables, emphasis); don't over-format simple answers.
-- This is a written channel: no spoken-style filler and no action/emote text like *laughs*.
-- Use tools when they help fulfill the request. No preamble sentence is required before a tool call.
-- For slow or external tools, just call the tool and use the result; you don't need to announce it.
-- If unsure whether a tool is needed, just answer directly.
-"""
-
-# Skeleton for the assembled system message (placeholders filled in build_text_system_prompt).
-_TEXT_SYSTEM_PROMPT_FULL = """\
-{lead}
-
-Session Prompt:
-{session_prompt}{optional_tools}
-
-{tail}
-"""
-
-
-def build_text_system_prompt(session_prompt: str, *, tool_section: str = "") -> str:
-    """Context → session prompt → optional tool block → strongest text rules last."""
-    tools = tool_section.strip()
-    optional_tools = f"\n\n{tools}" if tools else ""
-    return _TEXT_SYSTEM_PROMPT_FULL.format(
-        lead=TEXT_SYSTEM_PROMPT_LEAD.rstrip(),
-        session_prompt=session_prompt.strip(),
-        optional_tools=optional_tools,
-        tail=TEXT_SYSTEM_PROMPT_TAIL.rstrip(),
-    )
-
-
-# Full text instructions without a separate session block (legacy / rare direct use).
-TEXT_SYSTEM_PROMPT = "{lead}\n\n{tail}".format(
-    lead=TEXT_SYSTEM_PROMPT_LEAD.rstrip(),
-    tail=TEXT_SYSTEM_PROMPT_TAIL.rstrip(),
-)

runtime/src/speech_to_speech/LLM/tool_call/function_call.py

@@ -1,321 +0,0 @@
-#!/usr/bin/env python3
-"""
-Function parser for extracting function names, parameter names, and values from string function calls.
-
-Uses Python's ``tokenize`` and ``ast`` modules so that nested parentheses,
-strings containing ')' characters, tuples, dicts, etc. are handled correctly.
-"""
-
-import ast
-import io
-import json
-import logging
-import re
-import tokenize
-from collections import OrderedDict
-from typing import Any, Dict, List, Tuple
-
-from openai.types.responses import ResponseFunctionToolCall
-from pydantic import BaseModel
-
-from speech_to_speech.LLM.tool_call.function_tool import FunctionTool
-from speech_to_speech.utils.utils import _generate_id
-
-logger = logging.getLogger(__name__)
-
-_POSITIONAL_RE = re.compile(r"^__arg_\d+__$")
-_LENIENT_CALL_RE = re.compile(
-    r"\b[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*\s*"
-    r"\((?:[^()\"']+|\"(?:\\.|[^\"])*\"|'(?:\\.|[^'])*')*\)"
-)
-
-
-# ── AST / tokenize helpers ───────────────────────────────────────────
-
-
-def _split_top_level_calls(source: str) -> List[str]:
-    """Split *source* into individual ``name(...)`` expression strings.
-
-    Uses the tokenizer to walk tokens and track parenthesis depth so that
-    nested parens, strings with ')' chars, etc. are handled correctly.
-    """
-    tokens = list(tokenize.generate_tokens(io.StringIO(source).readline))
-    calls: List[str] = []
-    i = 0
-
-    while i < len(tokens):
-        tok = tokens[i]
-        if tok.type != tokenize.NAME:
-            i += 1
-            continue
-
-        start = i
-        j = i + 1
-
-        # Walk past dotted attribute access (e.g. ``mobile.click``)
-        while j + 1 < len(tokens) and tokens[j].string == "." and tokens[j + 1].type == tokenize.NAME:
-            j += 2
-
-        if j >= len(tokens) or tokens[j].string != "(":
-            i += 1
-            continue
-
-        # Track balanced parens
-        depth = 0
-        end = None
-        k = j
-        while k < len(tokens):
-            t = tokens[k]
-            if t.type == tokenize.OP and t.string == "(":
-                depth += 1
-            elif t.type == tokenize.OP and t.string == ")":
-                depth -= 1
-                if depth == 0:
-                    end = k
-                    break
-            k += 1
-
-        if end is None:
-            i += 1
-            continue
-
-        calls.append(tokenize.untokenize(tokens[start : end + 1]).strip())
-        i = end + 1
-
-    return calls
-
-
-def _split_simple_calls_with_regex(source: str) -> List[str]:
-    """Extract complete simple ``name(args)`` spans from malformed model output.
-
-    This fallback can recover well-formed siblings before a tokenizer error,
-    but not the incomplete call that caused the tokenizer error.
-    """
-    return [match.group(0).strip() for match in _LENIENT_CALL_RE.finditer(source)]
-
-
-def _parse_function_exprs(
-    expressions: List[str],
-    pattern_to_match: list[str],
-    *,
-    skip_invalid: bool = False,
-) -> List["FunctionToolCall"]:
-    results: List[FunctionToolCall] = []
-    for expr in expressions:
-        try:
-            call = _parse_call_expr(expr)
-        except Exception:
-            if skip_invalid:
-                continue
-            raise
-        if pattern_to_match and all(pattern not in call.function_name for pattern in pattern_to_match):
-            continue
-        results.append(call)
-    return results
-
-
-def _extract_function_name(node: ast.expr) -> str:
-    """Return the dotted function name from a Call node's ``func`` attribute."""
-    if isinstance(node, ast.Name):
-        return node.id
-    if isinstance(node, ast.Attribute):
-        base = _extract_function_name(node.value)
-        return f"{base}.{node.attr}" if base else node.attr
-    raise ValueError(f"Unsupported function target: {ast.dump(node)}")
-
-
-def _literal_from_ast(node: ast.AST) -> Any:
-    """Convert an AST node to a Python literal value."""
-    if isinstance(node, ast.Constant):
-        return node.value
-
-    if isinstance(node, ast.Name):
-        return node.id
-
-    if isinstance(node, ast.List):
-        return [_literal_from_ast(elt) for elt in node.elts]
-
-    if isinstance(node, ast.Tuple):
-        return [_literal_from_ast(elt) for elt in node.elts]
-
-    if isinstance(node, ast.Dict):
-        return {
-            _literal_from_ast(key): _literal_from_ast(value)
-            for key, value in zip(node.keys, node.values)
-            if key is not None
-        }
-
-    if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)):
-        value = _literal_from_ast(node.operand)
-        if not isinstance(value, (int, float)):
-            raise ValueError(f"Unsupported unary literal: {ast.dump(node)}")
-        return -value if isinstance(node.op, ast.USub) else value
-
-    raise ValueError(f"Unsupported literal: {ast.dump(node)}")
-
-
-def _parse_call_expr(expr: str) -> "FunctionToolCall":
-    """Parse a single ``name(args...)`` expression string into a FunctionToolCall."""
-    parsed = ast.parse(expr, mode="eval").body
-    if not isinstance(parsed, ast.Call):
-        raise ValueError(f"Expression is not a function call: {expr!r}")
-
-    parameters: "OrderedDict[str, Any]" = OrderedDict()
-
-    for idx, arg in enumerate(parsed.args):
-        parameters[f"__arg_{idx}__"] = _literal_from_ast(arg)
-
-    for kw in parsed.keywords:
-        if kw.arg is None:
-            raise ValueError("**kwargs are not supported")
-        parameters[kw.arg] = _literal_from_ast(kw.value)
-
-    return FunctionToolCall(
-        function_name=_extract_function_name(parsed.func),
-        parameters=parameters,
-        original_string=expr,
-    )
-
-
-# ── Data model ───────────────────────────────────────────────────────
-
-
-class FunctionToolCall(BaseModel):
-    """Represents a parsed function call with its parameters."""
-
-    function_name: str
-    parameters: Dict[str, Any]
-    original_string: str
-    description: str = ""
-
-    def to_realtime_function_tool_call(
-        self,
-        function_tools: list[FunctionTool] | None = None,
-    ) -> ResponseFunctionToolCall:
-        positional = {k for k in self.parameters if _POSITIONAL_RE.match(k)}
-        if positional:
-            logger.warning(
-                "Dropping positional arguments for '%s': %s",
-                self.function_name,
-                positional,
-            )
-        arguments = {k: v for k, v in self.parameters.items() if not _POSITIONAL_RE.match(k)}
-
-        if function_tools is not None:
-            tool = next(
-                (t for t in function_tools if t.name == self.function_name),
-                None,
-            )
-            if tool is None:
-                available = [t.name for t in function_tools]
-                raise ValueError(f"Function '{self.function_name}' not found in available tools: {available}")
-
-            schema = tool.parameters if isinstance(tool.parameters, dict) else {}
-            properties = schema.get("properties", {})
-            required = set(schema.get("required", []))
-
-            undeclared = {k for k in arguments if k not in properties}
-            if undeclared:
-                logger.warning(
-                    "Dropping undeclared parameters for '%s': %s",
-                    self.function_name,
-                    undeclared,
-                )
-                arguments = {k: v for k, v in arguments.items() if k in properties}
-
-            missing = required - set(arguments.keys())
-            if missing:
-                raise ValueError(f"Missing required parameters for '{self.function_name}': {missing}")
-
-        return ResponseFunctionToolCall(
-            name=self.function_name,
-            arguments=json.dumps(arguments),
-            call_id=_generate_id("call"),
-            type="function_call",
-            id=_generate_id("fc"),
-            status="in_progress",
-        )
-
-
-# ── Public API ───────────────────────────────────────────────────────
-
-
-def parse_function_call(function_string: str, pattern_to_match: list[str] = []) -> List[FunctionToolCall]:
-    """Parse a function call string and extract all function calls found.
-
-    Args:
-        function_string: String representation of function calls.
-        pattern_to_match: If non-empty, only calls whose function name
-            contains at least one of these substrings are returned.
-
-    Returns:
-        List of FunctionToolCall objects with parsed information.
-    """
-    function_string = function_string.strip()
-    if not function_string:
-        return []
-
-    try:
-        expressions = _split_top_level_calls(function_string)
-    except tokenize.TokenError:
-        return _parse_function_exprs(
-            _split_simple_calls_with_regex(function_string),
-            pattern_to_match,
-            skip_invalid=True,
-        )
-
-    return _parse_function_exprs(expressions, pattern_to_match)
-
-
-def parse_multiple_functions(function_strings: List[str]) -> List[FunctionToolCall]:
-    """Parse multiple function call strings.
-
-    Args:
-        function_strings: List of function call strings.
-
-    Returns:
-        List of FunctionToolCall objects.
-    """
-    results: List[FunctionToolCall] = []
-    for func_str in function_strings:
-        try:
-            results.extend(parse_function_call(func_str))
-        except Exception:
-            continue
-    return results
-
-
-def extract_function_calls_from_text(text: str, block_regex: str = ".*") -> Tuple[str, List[FunctionToolCall]]:
-    """Extract function calls from delimited code blocks inside *text*.
-
-    The LLM is prompted to wrap tool calls inside code-block delimiters
-    (e.g. ``<code>func(x=1)</code>``).  This function finds those blocks,
-    parses the function calls within them, and returns the remaining text
-    (with blocks stripped) alongside the parsed calls.
-
-    Args:
-        text: Full model output potentially containing code blocks.
-        block_regex: Regex matching the code-block delimiters **and** their
-            content (e.g. ``r"<code>.*?</code>"``).  Only text **inside**
-            matched blocks is scanned for function calls.
-
-    Returns:
-        ``(outside_text, function_calls)`` -- the text with blocks stripped
-        and the parsed function calls found inside the blocks.
-    """
-    if not block_regex:
-        return text, []
-
-    matches = list(re.finditer(block_regex, text, flags=re.DOTALL))
-    if not matches:
-        return text, []
-
-    outside = re.sub(block_regex, "", text, flags=re.DOTALL)
-    inside = " ".join(match.group(0) for match in matches).strip()
-    if not inside:
-        return outside, []
-
-    try:
-        return outside, parse_function_call(inside)
-    except Exception:
-        return outside, []

runtime/src/speech_to_speech/LLM/tool_call/function_tool.py

@@ -1,34 +0,0 @@
-import textwrap
-
-from openai.types.realtime import RealtimeFunctionTool
-
-from speech_to_speech.LLM.tool_call.signature_from_schema import signature_from_schema
-
-
-class FunctionTool(RealtimeFunctionTool):
-    def to_code_prompt(self, include_args_doc: bool = True) -> str:
-        """Generate a code-style prompt string for this function tool.
-
-        Args:
-            include_args_doc: If True, include argument descriptions in the docstring.
-                ⚠️ This lets the model see each argument's purpose but significantly increases
-                token usage (e.g. 906 tokens without vs 3434 with for the default Reachy Mini
-                tool profile). Enable depending on the model's capabilities and context limit.
-        """
-        signature = signature_from_schema(self.parameters)
-
-        tool_doc = self.description or ""
-
-        if isinstance(self.parameters, dict) and include_args_doc:
-            props = self.parameters.get("properties", {})
-            if props:
-                arg_lines = []
-                for arg_name, arg_schema in props.items():
-                    desc = arg_schema.get("description", "") if isinstance(arg_schema, dict) else ""
-                    arg_lines.append(f"{arg_name}: {desc}")
-                args_doc = f"Args:\n{textwrap.indent(chr(10).join(arg_lines), '    ')}"
-                tool_doc += f"\n\n{args_doc}"
-
-        tool_doc = f'"""{tool_doc}\n"""'
-
-        return f"def {self.name}{signature}:\n{textwrap.indent(tool_doc, '    ')}"

runtime/src/speech_to_speech/LLM/tool_call/signature_from_schema.py

@@ -1,108 +0,0 @@
-import inspect
-from typing import Any, Literal, Union
-
-JSON_TYPE_TO_PYTHON_TYPE = {
-    "string": str,
-    "number": float,
-    "boolean": bool,
-    "integer": int,
-    "object": dict,
-    "array": list,
-    "null": type(None),
-}
-
-
-def _dedupe_types(types: list[Any]) -> list[Any]:
-    seen = []
-    for t in types:
-        if t not in seen:
-            seen.append(t)
-    return seen
-
-
-def _annotation_from_spec(spec: dict[str, Any]) -> Any:
-    if not spec or not isinstance(spec, dict):
-        return Any
-
-    # const → Literal[value]
-    if "const" in spec:
-        return Literal[spec["const"]]
-
-    # enum → Literal[val1, val2, ...]
-    if "enum" in spec:
-        values = spec["enum"]
-        if not values:
-            return Any
-        return Literal[tuple(values)]
-
-    # anyOf / oneOf → Union[Type1, Type2, ...]
-    for key in ("anyOf", "oneOf"):
-        if key in spec:
-            variants = [_annotation_from_spec(s) for s in spec[key]]
-            unique = _dedupe_types(variants)
-            if len(unique) == 0:
-                return Any
-            if len(unique) == 1:
-                return unique[0]
-            return Union[tuple(unique)]
-
-    # allOf → merge sub-schemas then resolve
-    if "allOf" in spec:
-        merged = {}
-        for sub in spec["allOf"]:
-            merged.update(sub)
-        return _annotation_from_spec(merged)
-
-    json_type = spec.get("type")
-
-    if json_type is None:
-        return Any
-
-    # type as list, e.g. ["string", "null"] → Union[str, None] (Optional)
-    if isinstance(json_type, list):
-        types = [JSON_TYPE_TO_PYTHON_TYPE.get(t, Any) for t in json_type]
-        unique = _dedupe_types(types)
-        if len(unique) == 0:
-            return Any
-        if len(unique) == 1:
-            return unique[0]
-        return Union[tuple(unique)]
-
-    # array with items → list[ItemType]
-    if json_type == "array" and "items" in spec:
-        item_type = _annotation_from_spec(spec["items"])
-        return list[item_type]  # type: ignore[valid-type]
-
-    return JSON_TYPE_TO_PYTHON_TYPE.get(json_type, Any)
-
-
-def signature_from_schema(schema: object | None) -> inspect.Signature:
-    if not schema or not isinstance(schema, dict):
-        return inspect.Signature()
-
-    props = schema.get("properties", {})
-    required = set(schema.get("required", []))
-    params = []
-
-    for name, spec in props.items():
-        annotation = _annotation_from_spec(spec)
-
-        has_schema_default = "default" in spec if isinstance(spec, dict) else False
-
-        if name in required and not has_schema_default:
-            default = inspect.Parameter.empty
-        elif has_schema_default:
-            default = spec["default"]
-        else:
-            default = None
-
-        params.append(
-            inspect.Parameter(
-                name,
-                inspect.Parameter.POSITIONAL_OR_KEYWORD,
-                default=default,
-                annotation=annotation,
-            )
-        )
-
-    return inspect.Signature(params)

runtime/src/speech_to_speech/LLM/tool_call/tool_prompt.py

@@ -1,111 +0,0 @@
-"""
-Optional system-prompt builder that instructs a local LLM to output tool calls
-inside delimited code blocks (e.g. ``<code>func(arg='val')</code>``).
-
-The prompt is rendered from a Jinja2 template and relies on
-``FunctionTool.to_code_prompt()`` to expose each tool's Python-style signature.
-"""
-
-import re
-
-from jinja2 import Template
-
-from speech_to_speech.LLM.tool_call.function_tool import FunctionTool
-
-# ---------------------------------------------------------------------------
-# Default delimiters
-# ---------------------------------------------------------------------------
-ENTER_CODE = "<code>"
-END_CODE = "</code>"
-
-# ---------------------------------------------------------------------------
-# Jinja2 template
-# ---------------------------------------------------------------------------
-# ``enter_code`` / ``end_code`` are the block delimiters the model must wrap
-# every tool call in.  ``tools`` is a list of FunctionTool instances whose
-# ``.to_code_prompt()`` is called inside the template.
-# ---------------------------------------------------------------------------
-
-TOOL_PROMPT_TEMPLATE = Template(
-    """\
-Available tools:
-
-{% for tool in tools %}
-{{ tool.to_code_prompt() }}
-
-{% endfor %}
-To call a tool, put exactly one named-argument function call inside {{ enter_code }}...{{ end_code }}:
-{{ enter_code }}function_name(required_arg='value'){{ end_code }}
-
-Rules:
-- You may say one brief natural sentence before the tool call; for slow information tools, briefly say that you will check.
-- For expression/background tools, always speak first. For requested expressions, use a short pattern like "Sure, here's my best <emotion>."; otherwise use a fitting empathetic sentence.
-- Do not mention tags, functions, or tools. Keep prose outside tags brief, and do not claim tool results before a tool result is available.
-- Use named arguments only; quote strings. Omit optional args instead of placeholder values like "random", "none", "", or null.
-- Only one tool call may appear in a response.\
-""",
-    keep_trailing_newline=True,
-)
-
-# Text-channel variant: same call format and structural rules, without the
-# voice-specific "speak first" prose.
-TEXT_TOOL_PROMPT_TEMPLATE = Template(
-    """\
-Available tools:
-
-{% for tool in tools %}
-{{ tool.to_code_prompt() }}
-
-{% endfor %}
-To call a tool, put exactly one named-argument function call inside {{ enter_code }}...{{ end_code }}:
-{{ enter_code }}function_name(required_arg='value'){{ end_code }}
-
-Rules:
-- Call a tool directly when it helps fulfill the request; no preamble sentence is required.
-- Do not mention tags, functions, or tools in your prose, and do not claim tool results before a tool result is available.
-- Use named arguments only; quote strings. Omit optional args instead of placeholder values like "random", "none", "", or null.
-- Only one tool call may appear in a response.\
-""",
-    keep_trailing_newline=True,
-)
-
-
-# ---------------------------------------------------------------------------
-# Public helpers
-# ---------------------------------------------------------------------------
-
-
-def build_tool_system_prompt(
-    tools: list[FunctionTool],
-    enter_code: str = ENTER_CODE,
-    end_code: str = END_CODE,
-    *,
-    text_only: bool = False,
-) -> str:
-    """Render the tool-calling system-prompt section.
-
-    Returns an empty string when *tools* is empty so it can be
-    unconditionally appended to a base system prompt.  When *text_only* is set,
-    the written-channel variant is used (no voice "speak first" prose).
-    """
-    if not tools:
-        return ""
-
-    template = TEXT_TOOL_PROMPT_TEMPLATE if text_only else TOOL_PROMPT_TEMPLATE
-    return template.render(
-        tools=tools,
-        enter_code=enter_code,
-        end_code=end_code,
-    )
-
-
-def build_block_regex(
-    enter_code: str = ENTER_CODE,
-    end_code: str = END_CODE,
-) -> str:
-    """Build a regex that matches a single code block (non-greedy).
-
-    >>> build_block_regex("<code>", "</code>")
-    '<code>.*?</code>'
-    """
-    return f"{re.escape(enter_code)}.*?{re.escape(end_code)}"

runtime/src/speech_to_speech/LLM/utils.py

@@ -1,75 +0,0 @@
-import base64
-import io
-import re
-from typing import Optional
-
-import requests  # type: ignore[import-untyped]
-from PIL import Image
-
-SMART_PUNCT_TRANSLATION = str.maketrans(
-    {
-        "\u2018": "'",
-        "\u2019": "'",
-        "\u201c": '"',
-        "\u201d": '"',
-    }
-)
-
-SPEECHABLE_PATTERN = re.compile(
-    r"[^\w\s.,!?;:'\"\-()\/\\@#%&*+=$€£¥₹₽¢\[\]{}<>~`^|…—–\n\r\t]",
-    flags=re.UNICODE,
-)
-
-
-def remove_unspeechable(text: str) -> str:
-    """Keep only speechable characters: letters, digits, punctuation, whitespace.
-    support unicode characters (english, arabic, chinese, japanese, korean, etc.)
-    """
-    text = text.translate(SMART_PUNCT_TRANSLATION)
-    return SPEECHABLE_PATTERN.sub("", text)
-
-
-WHISPER_LANGUAGE_TO_LLM_LANGUAGE = {
-    "en": "english",
-    "fr": "french",
-    "es": "spanish",
-    "zh": "chinese",
-    "ja": "japanese",
-    "ko": "korean",
-    "hi": "hindi",
-    "de": "german",
-    "pt": "portuguese",
-    "pl": "polish",
-    "it": "italian",
-    "nl": "dutch",
-}
-
-
-def resolve_auto_language(language_code: Optional[str]) -> tuple[Optional[str], Optional[str]]:
-    """Strip the ``-auto`` suffix and resolve the human-readable language name.
-
-    Returns ``(clean_code, language_name)``.  ``language_name`` is non-None
-    when the code (with or without ``-auto``) maps to a known language.
-    """
-    if not language_code:
-        return language_code, None
-    if language_code.endswith("-auto"):
-        language_code = language_code[:-5]
-    if language_code not in WHISPER_LANGUAGE_TO_LLM_LANGUAGE:
-        return language_code, None
-    return language_code, WHISPER_LANGUAGE_TO_LLM_LANGUAGE.get(language_code)
-
-
-def image_url_to_pil(image_url: str) -> Image.Image:
-    """Convert an image URL or base64 data URI to a PIL Image.
-
-    Accepts:
-    - 'data:image/...;base64,<b64>' data URIs
-    - 'https://...`` or ``http://...' URLs (fetched with a 10s timeout)
-    """
-    if image_url.startswith("data:"):
-        _, b64_data = image_url.split(",", 1)
-        return Image.open(io.BytesIO(base64.b64decode(b64_data)))
-    resp = requests.get(image_url, timeout=10)
-    resp.raise_for_status()
-    return Image.open(io.BytesIO(resp.content))

runtime/src/speech_to_speech/LLM/voice_prompt.py

@@ -1,48 +0,0 @@
-"""Voice-channel system prompt: lead + session prompt + tail (strongest constraints last)."""
-
-VOICE_SYSTEM_PROMPT_LEAD = """\
-You are in a spoken conversation. The user speaks and hears you.
-The session prompt defines persona, facts, goals, and tool descriptions. These channel rules only control spoken output and tool-use behavior.
-"""
-
-VOICE_SYSTEM_PROMPT_TAIL = """\
-## Voice Rules
-- Keep replies brief by default: usually one spoken sentence, two if needed. Go longer only when asked.
-- Speak naturally. No markdown, bullets, headings, visual formatting, or action/emote text like *laughs*.
-- Treat transcripts as noisy. Correct likely mishearings only if asked or meaning depends on it.
-- Speech is the default. Use at most one tool when it helps fulfill the request or clearly fits the moment.
-- Before a tool call, use a brief natural utterance unless the user asked for silence or tool-only output. For slow information tools, briefly say that you will check.
-- For expression/background tools, speak first. If asked to show an expression, use a short pattern like "Sure, here's my best <emotion>." Otherwise use a fitting empathetic sentence. Never mention tools.
-- After completed expression/background/physical-action tools, do not add a second spoken comment unless the result has user-facing information.
-- Use motion, dance, emotion, and similar tools sparingly when they add empathy, celebration, playfulness, or a requested physical action.
-- If unsure whether a tool is needed, just speak.
-"""
-
-# Skeleton for the assembled system message (placeholders filled in build_voice_system_prompt).
-_VOICE_SYSTEM_PROMPT_FULL = """\
-{lead}
-
-Session Prompt:
-{session_prompt}{optional_tools}
-
-{tail}
-"""
-
-
-def build_voice_system_prompt(session_prompt: str, *, tool_section: str = "") -> str:
-    """Context → session prompt → optional tool block → strongest voice rules last."""
-    tools = tool_section.strip()
-    optional_tools = f"\n\n{tools}" if tools else ""
-    return _VOICE_SYSTEM_PROMPT_FULL.format(
-        lead=VOICE_SYSTEM_PROMPT_LEAD.rstrip(),
-        session_prompt=session_prompt.strip(),
-        optional_tools=optional_tools,
-        tail=VOICE_SYSTEM_PROMPT_TAIL.rstrip(),
-    )
-
-
-# Full voice instructions without a separate session block (legacy / rare direct use).
-VOICE_SYSTEM_PROMPT = "{lead}\n\n{tail}".format(
-    lead=VOICE_SYSTEM_PROMPT_LEAD.rstrip(),
-    tail=VOICE_SYSTEM_PROMPT_TAIL.rstrip(),
-)

runtime/src/speech_to_speech/STT/README.md

@@ -1,163 +0,0 @@
-# STT Summary
-
-This document summarizes the Speech-to-Text (STT) implementations in the `STT/` folder, including language support, language abbreviations, and usage in `s2s_pipeline.py`.
-
-## Available STT Modes (`--stt`)
-
-- `whisper` → `STT/whisper_stt_handler.py`
-- `whisper-mlx` → `STT/lightning_whisper_mlx_handler.py`
-- `mlx-audio-whisper` → `STT/mlx_audio_whisper_handler.py`
-- `faster-whisper` → `STT/faster_whisper_handler.py`
-- `parakeet-tdt` → `STT/parakeet_tdt_handler.py`
-- `paraformer` → `STT/paraformer_handler.py`
-
-## Language Support by Handler
-
-### 1) Whisper (`--stt whisper`)
-
-- Handler: `WhisperSTTHandler`
-- Language input flag: `--language` (from shared Whisper args)
-- Supports fixed language (e.g. `en`) or `auto`
-- Internal supported language list:
-  - `en`, `fr`, `es`, `zh`, `ja`, `ko`, `hi`, `de`, `pt`, `pl`, `it`, `nl`
-- Behavior:
-  - Detects language from token output
-  - If detected language is outside the supported list, it falls back to the previous language
-
-### 2) Lightning Whisper MLX (`--stt whisper-mlx`)
-
-- Handler: `LightningWhisperSTTHandler`
-- Uses same shared `--language` argument as Whisper
-- Internal supported language list:
-  - `en`, `fr`, `es`, `zh`, `ja`, `ko`, `hi`, `de`, `pt`, `pl`, `it`, `nl`
-- Behavior:
-  - If `--language auto`, model auto-detects each utterance
-  - If detected language is unsupported, falls back to last supported language
-
-### 3) MLX Audio Whisper (`--stt mlx-audio-whisper`)
-
-- Handler: `MLXAudioWhisperSTTHandler`
-- Model flag: `--mlx_audio_whisper_model_name`
-- Language still comes from shared `--language` flag (wired in pipeline)
-- Internal supported language list:
-  - `en`, `fr`, `es`, `zh`, `ja`, `ko`, `hi`, `de`, `pt`, `pl`, `it`, `nl`
-- Behavior:
-  - Uses fixed language unless `--language auto`
-  - Falls back to last known supported language when needed
-
-### 4) Faster-Whisper (`--stt faster-whisper`)
-
-- Handler: `FasterWhisperSTTHandler`
-- Language flag: `--faster_whisper_stt_gen_language`
-- Default language: `en`
-- Note:
-  - This handler passes generation kwargs directly to `faster_whisper.WhisperModel.transcribe(...)`
-  - Effective language coverage depends on selected Faster-Whisper/OpenAI Whisper model
-
-### 5) Parakeet TDT (`--stt parakeet-tdt`)
-
-- Handler: `ParakeetTDTSTTHandler`
-- Language flag: `--parakeet_tdt_language` (optional)
-- Supports auto language detection when language not specified
-- Declared supported language list (25 European languages):
-  - `en`, `de`, `fr`, `es`, `it`, `pt`, `nl`, `pl`, `ru`, `uk`, `cs`, `sk`, `hu`, `ro`, `bg`, `hr`, `sl`, `sr`, `da`, `no`, `sv`, `fi`, `et`, `lv`, `lt`
-- Backend behavior:
-  - On macOS/MPS: MLX (`mlx-community/parakeet-tdt-0.6b-v3`)
-  - On CUDA/CPU: nano-parakeet (`nvidia/parakeet-tdt-0.6b-v3`)
-
-### 6) Paraformer (`--stt paraformer`)
-
-- Handler: `ParaformerSTTHandler`
-- Model flag: `--paraformer_stt_model_name`
-- Default model: `paraformer-zh`
-- No dedicated language flag in current args class
-- Practical support:
-  - Depends on selected FunASR model checkpoint
-  - Default setup is Chinese-oriented (`zh`)
-
-## Language Abbreviations (ISO-style codes seen in STT handlers)
-
-| Code | Language |
-|---|---|
-| `en` | English |
-| `fr` | French |
-| `es` | Spanish |
-| `zh` | Chinese |
-| `ja` | Japanese |
-| `ko` | Korean |
-| `hi` | Hindi |
-| `de` | German |
-| `pt` | Portuguese |
-| `pl` | Polish |
-| `it` | Italian |
-| `nl` | Dutch |
-| `ru` | Russian |
-| `uk` | Ukrainian |
-| `cs` | Czech |
-| `sk` | Slovak |
-| `hu` | Hungarian |
-| `ro` | Romanian |
-| `bg` | Bulgarian |
-| `hr` | Croatian |
-| `sl` | Slovenian |
-| `sr` | Serbian |
-| `da` | Danish |
-| `no` | Norwegian |
-| `sv` | Swedish |
-| `fi` | Finnish |
-| `et` | Estonian |
-| `lv` | Latvian |
-| `lt` | Lithuanian |
-| `auto` | Per-utterance automatic language detection |
-
-## Usage Examples
-
-### Whisper (Transformers)
-
-```bash
-python s2s_pipeline.py --stt whisper --language en
-python s2s_pipeline.py --stt whisper --language auto
-```
-
-### Whisper MLX (LightningWhisperMLX)
-
-```bash
-python s2s_pipeline.py --stt whisper-mlx --language auto --device mps
-```
-
-### MLX Audio Whisper
-
-```bash
-python s2s_pipeline.py --stt mlx-audio-whisper \
-  --mlx_audio_whisper_model_name mlx-community/whisper-large-v3-turbo \
-  --language auto
-```
-
-### Faster-Whisper
-
-```bash
-python s2s_pipeline.py --stt faster-whisper \
-  --faster_whisper_stt_model_name large-v3 \
-  --faster_whisper_stt_gen_language en
-```
-
-### Parakeet TDT
-
-```bash
-python s2s_pipeline.py --stt parakeet-tdt --parakeet_tdt_device auto
-python s2s_pipeline.py --stt parakeet-tdt --parakeet_tdt_language de
-```
-
-With live transcription (MLX or CUDA/nano-parakeet backend):
-
-```bash
-python s2s_pipeline.py --stt parakeet-tdt \
-  --enable_live_transcription \
-  --live_transcription_update_interval 0.25
-```
-
-### Paraformer
-
-```bash
-python s2s_pipeline.py --stt paraformer --paraformer_stt_model_name paraformer-zh
-```

runtime/src/speech_to_speech/STT/__init__.py

@@ -1 +0,0 @@
-

runtime/src/speech_to_speech/STT/base_stt_handler.py

@@ -1,212 +0,0 @@
-from __future__ import annotations
-
-import logging
-from collections import Counter, OrderedDict
-from time import perf_counter
-from typing import Any
-
-from speech_to_speech.baseHandler import BaseHandler
-from speech_to_speech.pipeline.handler_types import STTIn, STTOut
-from speech_to_speech.pipeline.messages import PartialTranscription, Transcription, VADAudio
-from speech_to_speech.pipeline.speculative_turns import SpeculativeTurnTracker
-
-logger = logging.getLogger(__name__)
-
-
-class BaseSTTHandler(BaseHandler[STTIn, STTOut]):
-    """Base STT handler with speculative-turn stale input filtering."""
-
-    _MAX_COMPLETED_FINAL_REVISIONS = 2048
-
-    speculative_turns: SpeculativeTurnTracker | None = None
-    final_revision_settle_s: float = 0.0
-
-    def should_process_input(self, item: STTIn) -> bool:
-        mode = getattr(item, "mode", None)
-        turn_id = getattr(item, "turn_id", None)
-        turn_revision = getattr(item, "turn_revision", None)
-        if self._is_completed_final_revision(item):
-            queued_drops = self._drop_stale_queued_inputs()
-            self._log_stale_turn_item(item, "input-after-final", queued_drops=queued_drops)
-            return False
-        if mode == "progressive" and self._has_queued_final_for_revision(item):
-            self._log_stale_turn_item(item, "progressive-before-final")
-            return False
-
-        wait_for_stability = mode == "final"
-        gate_start = perf_counter()
-        is_latest = self._is_latest_turn_item(
-            item,
-            wait_for_pending_reopen=True,
-            wait_for_stability=wait_for_stability,
-        )
-        gate_wait_s = perf_counter() - gate_start
-        if gate_wait_s >= 0.05:
-            logger.info(
-                "%s: STT input gate waited %.3fs for turn=%s rev=%s mode=%s latest=%s age=%.3fs queue=%s",
-                self.__class__.__name__,
-                gate_wait_s,
-                turn_id,
-                turn_revision,
-                mode,
-                is_latest,
-                self._item_age_s(item),
-                self._safe_qsize(),
-            )
-
-        if not is_latest:
-            queued_drops = self._drop_stale_queued_inputs()
-            self._log_stale_turn_item(item, "input", queued_drops=queued_drops)
-            return False
-        return True
-
-    def should_emit_output(self, output: STTOut) -> bool:
-        if isinstance(output, PartialTranscription) and self._is_completed_final_revision(output):
-            self._log_stale_turn_item(output, "output-after-final")
-            return False
-
-        if not self._is_latest_turn_item(output, wait_for_pending_reopen=True, wait_for_stability=False):
-            self._log_stale_turn_item(output, "output")
-            return False
-        return True
-
-    def before_emit_output(self, output: STTOut) -> None:
-        if isinstance(output, Transcription):
-            self._mark_completed_final_revision(output)
-
-    def _is_latest_turn_item(
-        self,
-        item: object,
-        *,
-        wait_for_pending_reopen: bool,
-        wait_for_stability: bool,
-    ) -> bool:
-        if self.speculative_turns is None:
-            return True
-        turn_id = getattr(item, "turn_id", None)
-        turn_revision = getattr(item, "turn_revision", None)
-        if turn_id is None or turn_revision is None:
-            return True
-
-        if wait_for_stability:
-            is_latest = self.speculative_turns.is_latest_after_stability_window(
-                turn_id,
-                turn_revision,
-                self.final_revision_settle_s,
-            )
-        elif wait_for_pending_reopen:
-            is_latest = self.speculative_turns.is_latest_after_pending_reopen(turn_id, turn_revision)
-        else:
-            is_latest = self.speculative_turns.is_latest(turn_id, turn_revision)
-        return is_latest
-
-    def _drop_stale_queued_inputs(self) -> int:
-        if self.speculative_turns is None or not hasattr(self.queue_in, "mutex") or not hasattr(self.queue_in, "queue"):
-            return 0
-
-        dropped = 0
-        with self.queue_in.mutex:
-            kept: list[Any] = []
-            while self.queue_in.queue:
-                queued_item = self.queue_in.queue.popleft()
-                if isinstance(queued_item, VADAudio) and (
-                    self._is_completed_final_revision(queued_item)
-                    or (queued_item.mode == "progressive" and self._has_queued_final_for_revision_locked(queued_item))
-                    or not self._is_latest_turn_item(
-                        queued_item,
-                        wait_for_pending_reopen=False,
-                        wait_for_stability=False,
-                    )
-                ):
-                    dropped += 1
-                else:
-                    kept.append(queued_item)
-            self.queue_in.queue.extend(kept)
-            if dropped:
-                self.queue_in.not_full.notify_all()
-        return dropped
-
-    def _log_stale_turn_item(self, item: object, stage: str, *, queued_drops: int = 0) -> None:
-        turn_id = getattr(item, "turn_id", None)
-        turn_revision = getattr(item, "turn_revision", None)
-        if turn_id is None or turn_revision is None:
-            return
-
-        if not hasattr(self, "_stale_drop_counts"):
-            self._stale_drop_counts: Counter[tuple[str, str, int]] = Counter()
-        key = (stage, turn_id, turn_revision)
-        self._stale_drop_counts[key] += 1
-
-        message = "%s: dropping stale STT %s for turn=%s rev=%s age=%.3fs"
-        args: tuple[object, ...] = (
-            self.__class__.__name__,
-            stage,
-            turn_id,
-            turn_revision,
-            self._item_age_s(item),
-        )
-        if queued_drops:
-            message += " (+%d queued)"
-            args = (*args, queued_drops)
-
-        if self._stale_drop_counts[key] == 1:
-            logger.info(message, *args)
-        else:
-            logger.debug(message, *args)
-
-    def _item_age_s(self, item: object) -> float:
-        created_at_s = getattr(item, "created_at_s", None)
-        if not isinstance(created_at_s, float):
-            return 0.0
-        return max(0.0, perf_counter() - created_at_s)
-
-    def _safe_qsize(self) -> int | str:
-        try:
-            return self.queue_in.qsize()
-        except NotImplementedError:
-            return "unknown"
-
-    def _has_queued_final_for_revision(self, item: object) -> bool:
-        if not hasattr(self.queue_in, "mutex") or not hasattr(self.queue_in, "queue"):
-            return False
-        with self.queue_in.mutex:
-            return self._has_queued_final_for_revision_locked(item)
-
-    def _has_queued_final_for_revision_locked(self, item: object) -> bool:
-        key = self._revision_key(item)
-        if key is None:
-            return False
-        return any(
-            isinstance(queued_item, VADAudio) and queued_item.mode == "final" and self._revision_key(queued_item) == key
-            for queued_item in self.queue_in.queue
-        )
-
-    def _revision_key(self, item: object) -> tuple[str, int] | None:
-        turn_id = getattr(item, "turn_id", None)
-        turn_revision = getattr(item, "turn_revision", None)
-        if not isinstance(turn_id, str) or not isinstance(turn_revision, int):
-            return None
-        return (turn_id, turn_revision)
-
-    def _completed_final_revisions(self) -> OrderedDict[tuple[str, int], None]:
-        if not hasattr(self, "_completed_final_revision_keys"):
-            self._completed_final_revision_keys: OrderedDict[tuple[str, int], None] = OrderedDict()
-        return self._completed_final_revision_keys
-
-    def _is_completed_final_revision(self, item: object) -> bool:
-        key = self._revision_key(item)
-        return key is not None and key in self._completed_final_revisions()
-
-    def _mark_completed_final_revision(self, output: Transcription) -> None:
-        key = self._revision_key(output)
-        if key is None:
-            return
-        completed = self._completed_final_revisions()
-        completed[key] = None
-        completed.move_to_end(key)
-        while len(completed) > self._MAX_COMPLETED_FINAL_REVISIONS:
-            completed.popitem(last=False)
-
-    def on_session_end(self) -> None:
-        if hasattr(self, "_completed_final_revision_keys"):
-            self._completed_final_revision_keys.clear()

runtime/src/speech_to_speech/STT/faster_whisper_handler.py

@@ -1,68 +0,0 @@
-from __future__ import annotations
-
-import logging
-import os
-from typing import Any, Iterator
-
-from faster_whisper import WhisperModel
-from rich.console import Console
-
-from speech_to_speech.pipeline.handler_types import STTIn, STTOut
-from speech_to_speech.pipeline.messages import Transcription
-from speech_to_speech.STT.base_stt_handler import BaseSTTHandler
-
-console = Console()
-
-logger = logging.getLogger(__name__)
-
-
-class FasterWhisperSTTHandler(BaseSTTHandler):
-    """
-    Handles the Speech To Text generation using a Whisper model.
-    """
-
-    def setup(
-        self,
-        model_name: str = "tiny.en",
-        device: str = "auto",
-        compute_type: str = "auto",
-        gen_kwargs: dict[str, Any] = {},
-    ) -> None:
-        self.gen_kwargs = self.adapt_gen_kwargs(gen_kwargs)
-
-        os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
-        self.model = WhisperModel(model_name, device=device, compute_type=compute_type)
-
-    def process(self, vad_audio: STTIn) -> Iterator[STTOut]:
-        logger.debug("infering faster whisper...")
-
-        segments, info = self.model.transcribe(vad_audio.audio, **self.gen_kwargs)
-        output_text = []
-
-        for segment in segments:
-            logger.debug("[%.2fs -> %.2fs] %s" % (segment.start, segment.end, segment.text))
-            output_text.append(segment.text)
-
-        pred_text = " ".join(output_text).strip()
-
-        logger.debug("finished whisper inference")
-        if pred_text:
-            console.print(f"[yellow]USER: {pred_text}")
-
-            yield Transcription(
-                text=pred_text,
-                turn_id=vad_audio.turn_id,
-                turn_revision=vad_audio.turn_revision,
-                speech_stopped_at_s=vad_audio.created_at_s,
-            )
-        else:
-            logger.debug("no text detected. skipping...")
-
-    def cleanup(self) -> None:
-        print("Stopping FasterWhisperSTTHandler")
-        del self.model
-
-    def adapt_gen_kwargs(self, gen_kwargs: dict[str, Any]) -> dict[str, Any]:
-        gen_kwargs["without_timestamps"] = not gen_kwargs.pop("return_timestamps", True)
-
-        return gen_kwargs

runtime/src/speech_to_speech/STT/lightning_whisper_mlx_handler.py

@@ -1,103 +0,0 @@
-from __future__ import annotations
-
-import logging
-from typing import Any, Iterator, Optional
-
-import numpy as np
-import torch
-from lightning_whisper_mlx import LightningWhisperMLX
-from rich.console import Console
-
-from speech_to_speech.pipeline.handler_types import STTIn, STTOut
-from speech_to_speech.pipeline.messages import Transcription
-from speech_to_speech.STT.base_stt_handler import BaseSTTHandler
-
-logger = logging.getLogger(__name__)
-
-console = Console()
-
-SUPPORTED_LANGUAGES = [
-    "en",
-    "fr",
-    "es",
-    "zh",
-    "ja",
-    "ko",
-    "hi",
-    "de",
-    "pt",
-    "pl",
-    "it",
-    "nl",
-]
-
-
-class LightningWhisperSTTHandler(BaseSTTHandler):
-    """
-    Handles the Speech To Text generation using a Whisper model.
-    """
-
-    def setup(
-        self,
-        model_name: str = "distil-large-v3",
-        device: str = "mps",
-        torch_dtype: str = "float16",
-        compile_mode: Optional[str] = None,
-        language: Optional[str] = None,
-        gen_kwargs: dict[str, Any] = {},
-    ) -> None:
-        if len(model_name.split("/")) > 1:
-            model_name = model_name.split("/")[-1]
-        self.device = device
-        self.model = LightningWhisperMLX(model=model_name, batch_size=6, quant=None)
-        self.start_language = language
-        self.last_language = language
-
-        self.warmup()
-
-    def warmup(self) -> None:
-        logger.info(f"Warming up {self.__class__.__name__}")
-
-        # 2 warmup steps for no compile or compile mode with CUDA graphs capture
-        n_steps = 1
-        dummy_input = np.array([0] * 512)
-
-        for _ in range(n_steps):
-            _ = self.model.transcribe(dummy_input)["text"].strip()
-
-    def process(self, vad_audio: STTIn) -> Iterator[STTOut]:
-        logger.debug("infering whisper...")
-
-        audio = vad_audio.audio
-        if self.start_language != "auto":
-            transcription_dict = self.model.transcribe(audio, language=self.start_language)
-        else:
-            transcription_dict = self.model.transcribe(audio)
-            language_code = transcription_dict["language"]
-            if language_code not in SUPPORTED_LANGUAGES:
-                logger.warning(f"Whisper detected unsupported language: {language_code}")
-                if self.last_language in SUPPORTED_LANGUAGES:  # reprocess with the last language
-                    transcription_dict = self.model.transcribe(audio, language=self.last_language)
-                else:
-                    transcription_dict = {"text": "", "language": "en"}
-            else:
-                self.last_language = language_code
-
-        pred_text = transcription_dict["text"].strip()
-        language_code = transcription_dict["language"]
-        torch.mps.empty_cache()
-
-        logger.debug("finished whisper inference")
-        console.print(f"[yellow]USER: {pred_text}")
-        logger.debug(f"Language Code Whisper: {language_code}")
-
-        if self.start_language == "auto":
-            language_code += "-auto"
-
-        yield Transcription(
-            text=pred_text,
-            language_code=language_code,
-            turn_id=vad_audio.turn_id,
-            turn_revision=vad_audio.turn_revision,
-            speech_stopped_at_s=vad_audio.created_at_s,
-        )

runtime/src/speech_to_speech/STT/mlx_audio_whisper_handler.py

@@ -1,174 +0,0 @@
-from __future__ import annotations
-
-import logging
-from typing import Any, Iterator, Optional
-
-import numpy as np
-from rich.console import Console
-
-from speech_to_speech.pipeline.handler_types import STTIn, STTOut
-from speech_to_speech.pipeline.messages import Transcription
-from speech_to_speech.STT.base_stt_handler import BaseSTTHandler
-from speech_to_speech.utils.mlx_lock import MLXLockContext
-
-logger = logging.getLogger(__name__)
-
-console = Console()
-
-DEFAULT_LANGUAGE = "en"
-
-SUPPORTED_LANGUAGES = [
-    "en",
-    "fr",
-    "es",
-    "zh",
-    "ja",
-    "ko",
-    "hi",
-    "de",
-    "pt",
-    "pl",
-    "it",
-    "nl",
-]
-
-
-class MLXAudioWhisperSTTHandler(BaseSTTHandler):
-    """
-    Handles the Speech To Text generation using MLX Audio's Whisper implementation.
-    Optimized for Apple Silicon using the MLX framework.
-    """
-
-    def setup(
-        self,
-        model_name: str = "mlx-community/whisper-large-v3-turbo",
-        language: Optional[str] = None,
-        gen_kwargs: dict[str, Any] = {},
-    ) -> None:
-        from mlx_audio.stt.generate import load_model
-        from transformers import WhisperProcessor
-
-        self.model_name = model_name
-        self.start_language = language
-        # "auto" is a request to detect, not a language code, so it must never leak into
-        # last_language -- it would fail every SUPPORTED_LANGUAGES check downstream.
-        self.last_language = language if language != "auto" else None
-        self.gen_kwargs = gen_kwargs
-
-        # Load the model directly
-        logger.info(f"Loading model {model_name}...")
-        self.model = load_model(model_name)
-
-        # Check if processor was loaded, if not, load it manually from original model
-        if self.model._processor is None:
-            logger.info("Processor not found in MLX model, loading from original Whisper model...")
-            # Map MLX model names to their original Whisper counterparts
-            processor_model_map = {
-                "mlx-community/whisper-large-v3-turbo": "openai/whisper-large-v3",
-                "mlx-community/whisper-large-v3": "openai/whisper-large-v3",
-                "mlx-community/whisper-medium": "openai/whisper-medium",
-                "mlx-community/whisper-small": "openai/whisper-small",
-                "mlx-community/whisper-base": "openai/whisper-base",
-                "mlx-community/whisper-tiny": "openai/whisper-tiny",
-            }
-
-            # Get the appropriate processor model name
-            processor_model = processor_model_map.get(model_name, "openai/whisper-large-v3")
-            logger.info(f"Loading processor from {processor_model}...")
-
-            try:
-                self.model._processor = WhisperProcessor.from_pretrained(processor_model)
-                logger.info("Processor loaded successfully")
-            except Exception as e:
-                logger.error(f"Failed to load processor: {e}")
-                raise
-
-        logger.info(f"Model {model_name} loaded successfully")
-
-        self.warmup()
-
-    def warmup(self) -> None:
-        logger.info(f"Warming up {self.__class__.__name__}")
-
-        # Warmup with a dummy input
-        dummy_audio = np.zeros(16000, dtype=np.float32)
-
-        try:
-            # Pre-warm the model by running a transcription
-            with MLXLockContext(handler_name=self.__class__.__name__):
-                _ = self.model.generate(dummy_audio, verbose=False)
-            logger.info("Model warmed up and ready")
-        except Exception as e:
-            logger.warning(f"Warmup failed: {e}")
-
-    def _forced_language(self) -> Optional[str]:
-        """The language explicitly requested by the user, if any."""
-        if self.start_language and self.start_language != "auto":
-            return self.start_language
-        return None
-
-    def _resolve_language(self, result: Any) -> str:
-        """Pick the language code to report, updating the sticky fallback on success."""
-        forced = self._forced_language()
-        if forced is not None:
-            # generate() ran with this language, so it is authoritative.
-            self.last_language = forced
-            return forced
-
-        detected = getattr(result, "language", None)
-        if isinstance(detected, str) and detected:
-            if detected in SUPPORTED_LANGUAGES:
-                self.last_language = detected
-                return detected
-            logger.warning("Detected unsupported language: %s", detected)
-
-        last_language = self.last_language
-        if last_language is not None and last_language in SUPPORTED_LANGUAGES:
-            return last_language
-        return DEFAULT_LANGUAGE
-
-    def process(self, vad_audio: STTIn) -> Iterator[STTOut]:
-        logger.debug("inferring mlx-audio whisper...")
-
-        assert isinstance(vad_audio.audio, np.ndarray), "Audio must be a numpy array"
-        audio_input = vad_audio.audio.astype(np.float32)
-
-        # Prepare generation kwargs - only pass valid parameters
-        gen_kwargs = {}
-
-        # Add language if specified
-        forced_language = self._forced_language()
-        if forced_language is not None:
-            gen_kwargs["language"] = forced_language
-
-        try:
-            # MLX models share a single Metal command queue, so concurrent inference from
-            # the STT/LLM/TTS threads aborts the process with
-            # "Completed handler provided after commit call". Every other MLX path in the
-            # pipeline serializes through this lock; this one must too.
-            with MLXLockContext(handler_name=self.__class__.__name__):
-                result = self.model.generate(audio_input, verbose=False, **gen_kwargs)
-
-            # Extract text from result
-            pred_text = result.text.strip() if hasattr(result, "text") else str(result).strip()
-            language_code = self._resolve_language(result)
-
-        except Exception as e:
-            logger.error(f"MLX Audio Whisper inference failed: {e}")
-            pred_text = ""
-            language_code = self.last_language if self.last_language else DEFAULT_LANGUAGE
-
-        logger.debug("finished mlx-audio whisper inference")
-        console.print(f"[yellow]USER: {pred_text}")
-        logger.debug(f"Language Code: {language_code}")
-
-        if self.start_language == "auto":
-            language_code += "-auto"
-
-        yield Transcription(
-            text=pred_text,
-            language_code=language_code,
-            turn_id=vad_audio.turn_id,
-            turn_revision=vad_audio.turn_revision,
-            speech_stopped_at_s=vad_audio.created_at_s,
-        )

runtime/src/speech_to_speech/STT/paraformer_handler.py

@@ -1,79 +0,0 @@
-from __future__ import annotations
-
-import logging
-from typing import Any, Iterator
-
-import numpy as np
-import torch
-from rich.console import Console
-
-from speech_to_speech.pipeline.handler_types import STTIn, STTOut
-from speech_to_speech.pipeline.messages import PartialTranscription, Transcription
-from speech_to_speech.STT.base_stt_handler import BaseSTTHandler
-
-logging.basicConfig(
-    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-)
-logger = logging.getLogger(__name__)
-
-console = Console()
-
-
-class ParaformerSTTHandler(BaseSTTHandler):
-    """
-    Handles the Speech To Text generation using a Paraformer model.
-    The default for this model is set to Chinese.
-    This model was contributed by @wuhongsheng.
-    """
-
-    def setup(
-        self,
-        model_name: str = "paraformer-zh",
-        device: str = "cuda",
-        gen_kwargs: dict[str, Any] = {},
-    ) -> None:
-        print(model_name)
-        if len(model_name.split("/")) > 1:
-            model_name = model_name.split("/")[-1]
-        self.device = device
-        try:
-            from funasr import AutoModel
-        except ModuleNotFoundError as exc:
-            raise ModuleNotFoundError(
-                "Paraformer STT requires the optional 'paraformer' extra. "
-                "Install it with `pip install speech-to-speech[paraformer]`."
-            ) from exc
-        self.model = AutoModel(model=model_name, device=device)
-        self.warmup()
-
-    def warmup(self) -> None:
-        logger.info(f"Warming up {self.__class__.__name__}")
-
-        # 2 warmup steps for no compile or compile mode with CUDA graphs capture
-        n_steps = 1
-        dummy_input = np.array([0] * 512, dtype=np.float32)
-        for _ in range(n_steps):
-            _ = self.model.generate(dummy_input)[0]["text"].strip().replace(" ", "")
-
-    def process(self, vad_audio: STTIn) -> Iterator[STTOut]:
-        logger.debug("infering paraformer...")
-
-        pred_text = self.model.generate(vad_audio.audio)[0]["text"].strip().replace(" ", "")
-        torch.mps.empty_cache()
-
-        logger.debug("finished paraformer inference")
-        console.print(f"[yellow]USER: {pred_text}")
-
-        if vad_audio.mode == "progressive":
-            yield PartialTranscription(
-                text=pred_text,
-                turn_id=vad_audio.turn_id,
-                turn_revision=vad_audio.turn_revision,
-            )
-        else:
-            yield Transcription(
-                text=pred_text,
-                turn_id=vad_audio.turn_id,
-                turn_revision=vad_audio.turn_revision,
-                speech_stopped_at_s=vad_audio.created_at_s,
-            )

runtime/src/speech_to_speech/STT/parakeet_tdt_handler.py

@@ -1,648 +0,0 @@
-"""
-Parakeet TDT Speech-to-Text Handler
-
-Supports NVIDIA Parakeet TDT model for high-quality multilingual ASR.
-- On Apple Silicon (MPS): Uses mlx-audio with mlx-community/parakeet-tdt-0.6b-v3
-- On CUDA/CPU: Uses nano-parakeet (pure PyTorch) with nvidia/parakeet-tdt-0.6b-v3
-
-Model supports 25 European languages with automatic language detection.
-"""
-
-from __future__ import annotations
-
-import logging
-from contextlib import contextmanager
-from sys import platform
-from threading import Lock
-from time import perf_counter
-from typing import Any, Iterator, Optional
-
-import numpy as np
-from rich.console import Console
-from rich.text import Text
-
-from speech_to_speech.pipeline.handler_types import STTIn, STTOut
-from speech_to_speech.pipeline.messages import PartialTranscription, Transcription
-from speech_to_speech.STT.base_stt_handler import BaseSTTHandler
-from speech_to_speech.STT.smart_progressive_streaming import PartialTranscription as ProgressiveStreamPartial
-from speech_to_speech.utils.mlx_lock import MLXLockContext
-
-try:
-    from lingua import Language, LanguageDetectorBuilder
-
-    LINGUA_AVAILABLE = True
-except ImportError:
-    LINGUA_AVAILABLE = False
-
-logger = logging.getLogger(__name__)
-console = Console()
-
-# Parakeet TDT v3 supports 25 European languages
-SUPPORTED_LANGUAGES = [
-    "en",
-    "de",
-    "fr",
-    "es",
-    "it",
-    "pt",
-    "nl",
-    "pl",
-    "ru",
-    "uk",
-    "cs",
-    "sk",
-    "hu",
-    "ro",
-    "bg",
-    "hr",
-    "sl",
-    "sr",
-    "da",
-    "no",
-    "sv",
-    "fi",
-    "et",
-    "lv",
-    "lt",
-]
-
-# Lingua uses "nb" (Bokmål) for Norwegian instead of "no"
-_LINGUA_CODE_MAP = {"no": "nb"}
-
-if LINGUA_AVAILABLE:
-    _lingua_iso_to_code = {
-        lang.iso_code_639_1.name.lower(): lang for lang in Language.all() if lang.iso_code_639_1 is not None
-    }
-    _lingua_languages = [
-        _lingua_iso_to_code[_LINGUA_CODE_MAP.get(code, code)]
-        for code in SUPPORTED_LANGUAGES
-        if _LINGUA_CODE_MAP.get(code, code) in _lingua_iso_to_code
-    ]
-
-    def _build_lingua_detector():
-        # Preloading can take multiple seconds on some hardware, including the
-        # deployed server. Pay that cost at startup instead of on the first user
-        # request, where it would look like slow STT.
-        return LanguageDetectorBuilder.from_languages(*_lingua_languages).with_preloaded_language_models().build()
-
-    _lingua_detector = _build_lingua_detector()
-
-
-class ParakeetTDTSTTHandler(BaseSTTHandler):
-    """
-    Handles Speech-to-Text using NVIDIA Parakeet TDT model.
-
-    On Apple Silicon (MPS): Uses mlx-audio with the MLX-converted model.
-    On CUDA/CPU: Uses nano-parakeet (pure PyTorch) for NeMo-free inference.
-
-    Parakeet TDT 0.6B v3 is a 600M parameter multilingual ASR model
-    supporting 25 European languages with automatic language detection.
-    """
-
-    def setup(
-        self,
-        model_name: Optional[str] = None,
-        device: str = "auto",
-        compute_type: str = "float16",
-        language: Optional[str] = None,
-        gen_kwargs: dict[str, Any] = {},
-        enable_live_transcription: bool = False,
-        live_transcription_update_interval: float = 0.5,
-    ) -> None:
-        """
-        Initialize the Parakeet TDT model.
-
-        Args:
-            model_name: Model identifier. Defaults are:
-                - MPS: "mlx-community/parakeet-tdt-0.6b-v3"
-                - CUDA/CPU: "nvidia/parakeet-tdt-0.6b-v3"
-            device: Device to use ("auto", "cuda", "mps", "cpu")
-            compute_type: Compute precision ("float16", "float32")
-            language: Target language code (optional, model auto-detects)
-            gen_kwargs: Additional generation kwargs
-        """
-        self.gen_kwargs = gen_kwargs
-        self.start_language = language
-        self.last_language = language if language else "en"
-        self.enable_live_transcription = enable_live_transcription
-        self.live_transcription_update_interval = live_transcription_update_interval
-        self.compute_lock = Lock()
-        self.sample_rate = 16000
-
-        # Determine device
-        if device == "auto":
-            if platform == "darwin":
-                self.device = "mps"
-            else:
-                import torch
-
-                self.device = "cuda" if torch.cuda.is_available() else "cpu"
-        else:
-            self.device = device
-
-        # Set default model based on device
-        if model_name is None:
-            if self.device == "mps":
-                model_name = "mlx-community/parakeet-tdt-0.6b-v3"
-            else:
-                model_name = "nvidia/parakeet-tdt-0.6b-v3"
-
-        self.model_name = model_name
-        self.compute_type = compute_type
-
-        logger.info(f"Loading Parakeet TDT model: {model_name} on {self.device}")
-
-        if self.device == "mps":
-            self._setup_mlx(model_name)
-        else:
-            self._setup_nano_parakeet(model_name)
-
-        # Setup streaming handler if live transcription is enabled
-        self.streaming_handler = None
-        if self.enable_live_transcription:
-            from speech_to_speech.STT.smart_progressive_streaming import (
-                SmartProgressiveStreamingHandler,
-            )
-
-            self.streaming_handler = SmartProgressiveStreamingHandler(
-                self.model,
-                emission_interval=self.live_transcription_update_interval,
-                max_window_size=15.0,
-                sentence_buffer=2.0,
-            )
-            self.processing_final = False  # Track if we're processing final audio
-            logger.info(f"Live transcription enabled for Parakeet TDT ({self.backend})")
-        self._live_transcription_active = False
-        self._live_turn_key: tuple[str | None, int | None] | None = None
-
-        self.warmup()
-
-    def _setup_mlx(self, model_name: str) -> None:
-        """Setup for Apple Silicon using mlx-audio."""
-        try:
-            from mlx_audio.stt.generate import load_model
-
-            self.backend = "mlx"
-            self.model = load_model(model_name)
-            logger.info("MLX Audio Parakeet model loaded successfully")
-        except ImportError as e:
-            raise ImportError(
-                "mlx-audio is required for Parakeet TDT on Apple Silicon. Install with: pip install mlx-audio"
-            ) from e
-
-    def _setup_nano_parakeet(self, model_name: str) -> None:
-        """Setup for CUDA/CPU using nano-parakeet."""
-        try:
-            import torch
-            from nano_parakeet import from_pretrained
-
-            self.backend = "nano_parakeet"
-
-            if self.device == "cuda" and not torch.cuda.is_available():
-                logger.warning("CUDA requested but not available. Falling back to CPU for nano-parakeet.")
-                self.device = "cpu"
-
-            self.model = from_pretrained(model_name=model_name, device=self.device)
-
-            logger.info(f"nano-parakeet model loaded successfully on {self.device}")
-        except ImportError as e:
-            raise ImportError(
-                "nano-parakeet is required for Parakeet TDT on CUDA/CPU. Install with: pip install nano-parakeet"
-            ) from e
-
-    def warmup(self) -> None:
-        """Warm up the model with a dummy input."""
-        logger.info(f"Warming up {self.__class__.__name__}")
-
-        # Create 1 second of silence at 16kHz
-        dummy_audio = np.zeros(16000, dtype=np.float32)
-
-        try:
-            if self.backend == "mlx":
-                import mlx.core as mx
-
-                # Convert to mx.array and call decode_chunk directly
-                audio_mx = mx.array(dummy_audio, dtype=mx.float32)
-                _ = self.model.decode_chunk(audio_mx, verbose=False)
-            elif self.backend == "nano_parakeet":
-                _ = self.model.transcribe(dummy_audio)
-            else:
-                _ = self.model.transcribe([dummy_audio], batch_size=1, verbose=False)
-
-            logger.info("Model warmed up and ready")
-        except Exception as e:
-            logger.warning(f"Warmup failed: {e}")
-
-    def process(self, vad_audio: STTIn) -> Iterator[STTOut]:
-        """
-        Process audio and generate transcription.
-
-        Yields:
-            :class:`PartialTranscription` or :class:`Transcription`
-        """
-        process_start_s = perf_counter()
-        is_progressive = vad_audio.mode == "progressive"
-        audio_input = vad_audio.audio
-
-        # Ensure audio is float32 numpy array
-        if not isinstance(audio_input, np.ndarray):
-            audio_input = np.array(audio_input, dtype=np.float32)
-        else:
-            audio_input = audio_input.astype(np.float32)
-        audio_duration_s = len(audio_input) / getattr(self, "sample_rate", 16000)
-        item_age_s = self._item_age_s(vad_audio)
-
-        self._prepare_live_transcription_turn(vad_audio.turn_id, vad_audio.turn_revision)
-
-        # Handle progressive updates: yield tagged partial for TranscriptionNotifier
-        if self.enable_live_transcription and is_progressive:
-            # Ignore progressive updates if we're already processing final audio
-            if self.processing_final:
-                logger.debug("Skipping stale progressive update (final audio already received)")
-                return
-
-            # Try to acquire lock with short timeout - skip if busy
-            lock_scope_start_s = perf_counter()
-            with self._compute_lock_context(handler_name="ParakeetSTT-Progressive", timeout=0.01) as acquired:
-                if acquired:
-                    try:
-                        inference_start_s = perf_counter()
-                        progressive_text = self._show_progressive_transcription(audio_input)
-                        inference_s = perf_counter() - inference_start_s
-                        if inference_s >= 0.25:
-                            logger.info(
-                                "Parakeet progressive STT timing turn=%s rev=%s audio=%.3fs age=%.3fs "
-                                "lock_scope=%.3fs inference=%.3fs chars=%d",
-                                vad_audio.turn_id,
-                                vad_audio.turn_revision,
-                                audio_duration_s,
-                                item_age_s,
-                                perf_counter() - lock_scope_start_s,
-                                inference_s,
-                                len(progressive_text),
-                            )
-                        if progressive_text:
-                            yield PartialTranscription(
-                                text=progressive_text,
-                                turn_id=vad_audio.turn_id,
-                                turn_revision=vad_audio.turn_revision,
-                            )
-                            return
-                    except Exception as e:
-                        logger.debug(f"Progressive transcription failed: {e}")
-                else:
-                    logger.debug("Skipping progressive update (compute busy)")
-            return
-
-        # Handle final transcription (send to LLM)
-        logger.info(
-            "Parakeet final STT start turn=%s rev=%s audio=%.3fs age=%.3fs",
-            vad_audio.turn_id,
-            vad_audio.turn_revision,
-            audio_duration_s,
-            item_age_s,
-        )
-        inference_s = 0.0
-        lock_scope_s = 0.0
-        try:
-            if self.enable_live_transcription:
-                # Mark that we're processing final audio (ignore stale progressive updates)
-                self.processing_final = True
-
-            # Acquire lock with longer timeout for final transcription
-            lock_scope_start_s = perf_counter()
-            with self._compute_lock_context(handler_name="ParakeetSTT-Final", timeout=5.0) as acquired:
-                lock_scope_s = perf_counter() - lock_scope_start_s
-                if not acquired:
-                    logger.error("Failed to acquire compute lock for final transcription")
-                    pred_text = ""
-                    language_code = self.last_language
-                else:
-                    inference_start_s = perf_counter()
-                    if self.backend == "mlx":
-                        pred_text, language_code = self._process_mlx_final(audio_input)
-                    else:
-                        pred_text, language_code = self._process_nano_parakeet(audio_input)
-                    inference_s = perf_counter() - inference_start_s
-                    lock_scope_s = perf_counter() - lock_scope_start_s
-
-            # Validate and update language
-            if language_code and language_code in SUPPORTED_LANGUAGES:
-                self.last_language = language_code
-            else:
-                language_code = self.last_language
-
-        except Exception as e:
-            logger.error(f"Parakeet TDT inference failed: {e}")
-            pred_text = ""
-            language_code = self.last_language
-
-        total_s = perf_counter() - process_start_s
-        logger.info(
-            "Parakeet final STT done turn=%s rev=%s total=%.3fs lock_scope=%.3fs inference=%.3fs chars=%d",
-            vad_audio.turn_id,
-            vad_audio.turn_revision,
-            total_s,
-            lock_scope_s,
-            inference_s,
-            len(pred_text),
-        )
-        logger.debug("Finished Parakeet TDT inference")
-        self._clear_live_transcription_line()
-        if pred_text.strip():
-            console.print(f"[yellow]USER: {pred_text.strip()}")
-            if language_code:
-                console.print(f"[dim]Language: {language_code}[/dim]")
-
-        # Reset per-utterance live transcription state only after final STT
-        # completes. The streaming handler carries fixed sentence timing within
-        # an utterance, and stale timing must not leak into the next turn.
-        if self.enable_live_transcription:
-            self.processing_final = False
-            self._reset_live_transcription_state(clear_turn=True)
-
-        yield Transcription(
-            text=pred_text,
-            language_code=language_code,
-            turn_id=vad_audio.turn_id,
-            turn_revision=vad_audio.turn_revision,
-            speech_stopped_at_s=vad_audio.created_at_s,
-        )
-
-    @property
-    def timing_log_level(self) -> int:
-        return logging.INFO
-
-    def should_log_timing(self, output: STTOut) -> bool:
-        return isinstance(output, Transcription) and self.last_time > self.min_time_to_debug
-
-    def _detect_language_from_text(self, text: str) -> Optional[str]:
-        """
-        Detect language from transcribed text using lingua-py.
-
-        Args:
-            text: Transcribed text string
-
-        Returns:
-            Detected language code or None if detection fails
-        """
-        if not LINGUA_AVAILABLE:
-            logger.warning("lingua-py not available, cannot detect language from text")
-            return None
-
-        # Skip very short utterances where language ID is still too noisy.
-        if not text or len(text.strip()) < 20:
-            return None
-
-        detected = _lingua_detector.detect_language_of(text)
-        if detected is None:
-            return None
-
-        code = detected.iso_code_639_1.name.lower()
-        # Map back lingua-specific codes to our supported codes
-        return {v: k for k, v in _LINGUA_CODE_MAP.items()}.get(code, code)
-
-    @contextmanager
-    def _compute_lock_context(self, handler_name: str, timeout: float) -> Iterator[bool]:
-        if self.backend == "mlx":
-            with MLXLockContext(handler_name=handler_name, timeout=timeout) as acquired:
-                yield acquired
-            return
-
-        lock_start_s = perf_counter()
-        acquired = self.compute_lock.acquire(timeout=timeout)
-        wait_s = perf_counter() - lock_start_s
-        hold_start_s: float | None = None
-        if acquired:
-            if wait_s >= 0.25:
-                logger.info("%s: compute lock acquired after %.2fs", handler_name, wait_s)
-            else:
-                logger.debug("%s: compute lock acquired after %.3fs", handler_name, wait_s)
-            hold_start_s = perf_counter()
-        else:
-            logger.warning("%s: Failed to acquire compute lock after %.3fs (timeout=%s)", handler_name, wait_s, timeout)
-        try:
-            yield acquired
-        finally:
-            if acquired:
-                assert hold_start_s is not None
-                self.compute_lock.release()
-                hold_s = perf_counter() - hold_start_s
-                if hold_s >= 0.25:
-                    logger.info("%s: compute lock released after holding %.2fs", handler_name, hold_s)
-                else:
-                    logger.debug("%s: compute lock released after holding %.3fs", handler_name, hold_s)
-
-    def _show_progressive_transcription(self, audio_input: np.ndarray) -> str:
-        """Run progressive transcription, print to console, and return the text."""
-        result = self.streaming_handler.transcribe_incremental(audio_input)
-        rich_text = Text()
-        if result.fixed_text:
-            rich_text.append("Live: ", style="dim")
-            rich_text.append(result.fixed_text, style="yellow")
-            if result.active_text:
-                rich_text.append(" ", style="dim")
-
-        if result.active_text:
-            if not result.fixed_text:
-                rich_text.append("Live: ", style="dim")
-            rich_text.append(result.active_text, style="cyan dim")
-
-        progressive_text = self._build_progressive_text(result)
-        if progressive_text:
-            self._print_live_transcription(rich_text, progressive_text)
-
-        return progressive_text
-
-    def _print_live_transcription(self, rich_text: Text, progressive_text: str) -> None:
-        is_terminal = bool(getattr(console, "is_terminal", False))
-        if is_terminal:
-            self._write_live_control("\r\x1b[2K")
-            if rich_text:
-                console.print(self._truncate_live_transcription(rich_text), end="")
-            else:
-                fallback = Text("Live: ", style="dim")
-                fallback.append(progressive_text, style="cyan dim")
-                console.print(self._truncate_live_transcription(fallback), end="")
-            self._write_live_control("\r")
-            self._live_transcription_active = True
-            return
-
-        if rich_text:
-            console.print(rich_text)
-        else:
-            console.print(f"[dim]Live: [/dim]{progressive_text}")
-
-    def _clear_live_transcription_line(self) -> None:
-        if not getattr(self, "_live_transcription_active", False):
-            return
-        self._write_live_control("\r\x1b[2K")
-        self._live_transcription_active = False
-
-    def _write_live_control(self, sequence: str) -> None:
-        file = getattr(console, "file", None)
-        if file is None:
-            return
-        file.write(sequence)
-        file.flush()
-
-    def _truncate_live_transcription(self, text: Text) -> Text:
-        text = text.copy()
-        width = getattr(console, "width", 80)
-        try:
-            max_width = max(1, int(width) - 1)
-        except (TypeError, ValueError):
-            max_width = 79
-        text.truncate(max_width, overflow="ellipsis")
-        return text
-
-    def _prepare_live_transcription_turn(self, turn_id: str | None, turn_revision: int | None) -> None:
-        if not self.enable_live_transcription:
-            return
-        turn_key = (turn_id, turn_revision)
-        if getattr(self, "_live_turn_key", None) == turn_key:
-            return
-        self._reset_live_transcription_state(clear_turn=False)
-        self._live_turn_key = turn_key
-
-    def _reset_live_transcription_state(self, clear_turn: bool) -> None:
-        self._clear_live_transcription_line()
-        streaming_handler = getattr(self, "streaming_handler", None)
-        if streaming_handler is not None:
-            streaming_handler.reset()
-        if clear_turn:
-            self._live_turn_key = None
-
-    def _build_progressive_text(self, result: ProgressiveStreamPartial) -> str:
-        parts = []
-        if result.fixed_text:
-            parts.append(result.fixed_text.strip())
-        if result.active_text:
-            parts.append(result.active_text.strip())
-        return " ".join(part for part in parts if part).strip()
-
-    def _process_mlx_final(self, audio_input: np.ndarray) -> tuple[str, str]:
-        """Process final audio using MLX backend with streaming handler."""
-        # If we have fixed sentences from progressive updates, only transcribe the new part
-        if (
-            self.streaming_handler is not None
-            and hasattr(self.streaming_handler, "fixed_sentences")
-            and self.streaming_handler.fixed_sentences
-        ):
-            self._clear_live_transcription_line()
-
-            # Get fixed text from previous progressive updates
-            fixed_text = " ".join(self.streaming_handler.fixed_sentences).strip()
-
-            # Calculate where fixed part ends in audio
-            fixed_end_time = self.streaming_handler.fixed_end_time
-            sample_rate = 16000
-            fixed_end_sample = int(fixed_end_time * sample_rate)
-            if fixed_end_sample > len(audio_input):
-                logger.warning(
-                    "Ignoring stale progressive fixed text: fixed_end_sample=%d exceeds final audio samples=%d",
-                    fixed_end_sample,
-                    len(audio_input),
-                )
-                pred_text, language_code = self._process_mlx(audio_input)
-                return pred_text, language_code
-
-            # Only transcribe the part after fixed sentences
-            if fixed_end_sample < len(audio_input):
-                remaining_audio = audio_input[fixed_end_sample:]
-
-                import mlx.core as mx
-
-                audio_mx = mx.array(remaining_audio, dtype=mx.float32)
-                result = self.model.decode_chunk(audio_mx, verbose=False)
-
-                if hasattr(result, "text"):
-                    new_text = result.text.strip()
-                else:
-                    new_text = str(result).strip()
-
-                # Combine fixed + new
-                pred_text = f"{fixed_text} {new_text}".strip() if new_text else fixed_text
-            else:
-                # All audio already transcribed in progressive updates
-                pred_text = fixed_text
-
-            # Reset streaming handler for next utterance
-            self.streaming_handler.reset()
-        else:
-            # No progressive updates, transcribe everything
-            pred_text, language_code = self._process_mlx(audio_input)
-            return pred_text, language_code
-
-        # Determine language
-        if self.start_language and self.start_language != "auto":
-            language_code = self.start_language
-        else:
-            detected_lang = self._detect_language_from_text(pred_text)
-            if detected_lang:
-                language_code = detected_lang
-            else:
-                language_code = self.last_language
-
-        return pred_text, language_code
-
-    def _process_mlx(self, audio_input: np.ndarray) -> tuple[str, str]:
-        """Process audio using MLX backend."""
-        import mlx.core as mx
-
-        # Convert numpy array to mx.array
-        audio_mx = mx.array(audio_input, dtype=mx.float32)
-
-        # Call decode_chunk directly with the audio array
-        result = self.model.decode_chunk(audio_mx, verbose=False)
-
-        # Extract text from result
-        if hasattr(result, "text"):
-            pred_text = result.text.strip()
-        else:
-            pred_text = str(result).strip()
-
-        # Determine language:
-        # 1. Use fixed language if specified by user
-        # 2. Try to detect from transcribed text using langdetect
-        # 3. Fall back to last known language
-        if self.start_language and self.start_language != "auto":
-            language_code = self.start_language
-        else:
-            # Detect language from transcribed text
-            detected_lang = self._detect_language_from_text(pred_text)
-            if detected_lang:
-                language_code = detected_lang
-            else:
-                language_code = self.last_language
-
-        return pred_text, language_code
-
-    def _process_nano_parakeet(self, audio_input: np.ndarray) -> tuple[str, str]:
-        """Process audio using nano-parakeet backend."""
-        pred_text = self.model.transcribe(audio_input).strip()
-
-        if self.start_language and self.start_language != "auto":
-            language_code = self.start_language
-        else:
-            detected_lang = self._detect_language_from_text(pred_text)
-            if detected_lang:
-                language_code = detected_lang
-            else:
-                language_code = self.last_language
-
-        return pred_text, language_code
-
-    def cleanup(self) -> None:
-        """Clean up model resources."""
-        logger.info(f"Cleaning up {self.__class__.__name__}")
-        if hasattr(self, "model"):
-            del self.model
-
-    def on_session_end(self) -> None:
-        super().on_session_end()
-        self.last_language = self.start_language if self.start_language else "en"
-        if self.enable_live_transcription:
-            self.processing_final = False
-            self._reset_live_transcription_state(clear_turn=True)
-        logger.debug("Parakeet TDT session state reset")

runtime/src/speech_to_speech/STT/smart_progressive_streaming.py

@@ -1,343 +0,0 @@
-#!/usr/bin/env python3
-"""
-Smart Progressive Streaming Handler
-
-Provides frequent partial transcriptions (every 500ms) with:
-- Growing window up to 15s for accuracy
-- Sentence-boundary-aware window sliding for audio > 15s
-- Fixed sentences + active transcription
-"""
-
-from dataclasses import dataclass
-from types import SimpleNamespace
-from typing import Any, Generator
-
-import numpy as np
-
-
-@dataclass
-class PartialTranscription:
-    """Result from progressive streaming."""
-
-    fixed_text: str  # Sentences that won't change
-    active_text: str  # Current partial transcription
-    timestamp: float  # Current position in audio
-    is_final: bool  # True if this is the last update
-
-
-class SmartProgressiveStreamingHandler:
-    """
-    Smart progressive streaming with sentence-aware window management.
-
-    Strategy:
-    1. Emit partial transcriptions every 500ms
-    2. Use growing window (up to 15s) for better accuracy
-    3. When audio > 15s, slide window using sentence boundaries:
-       - Keep completed sentences as "fixed"
-       - Only re-transcribe the "active" portion
-    """
-
-    def __init__(
-        self,
-        model: Any,
-        emission_interval: float = 0.5,
-        max_window_size: float = 15.0,
-        sentence_buffer: float = 2.0,
-    ) -> None:
-        """
-        Args:
-            model: Parakeet model with sentence alignment
-            emission_interval: Emit partial transcription every N seconds (default 500ms)
-            max_window_size: Maximum window size before sliding (default 15s)
-            sentence_buffer: Keep last N seconds of sentences in active window (default 2s)
-        """
-        self.model = model
-        self.emission_interval = emission_interval
-        self.max_window_size = max_window_size
-        self.sentence_buffer = sentence_buffer
-        self.sample_rate = 16000
-
-        # State for incremental streaming
-        self.reset()
-
-    def reset(self) -> None:
-        """Reset state for new streaming session."""
-        self.fixed_sentences: list[str] = []
-        self.fixed_end_time: float = 0.0
-        self.last_transcribed_length: int = 0
-
-    def _decode_window(self, audio_window: np.ndarray) -> Any:
-        """
-        Decode an audio window and return an object with:
-          - text: full transcript for the window
-          - sentences: list of objects with .text and .end (seconds)
-        """
-        if hasattr(self.model, "decode_chunk"):
-            import mlx.core as mx
-
-            audio_mx = mx.array(audio_window, dtype=mx.float32)
-            return self.model.decode_chunk(audio_mx, verbose=False)
-
-        # nano-parakeet: use timestamps from transcribe
-        result = self.model.transcribe(audio_window, timestamps=True)
-        segments = result.timestamp.get("segment", []) if hasattr(result, "timestamp") else []
-        sentences = [SimpleNamespace(text=seg.get("segment", "").strip(), end=seg.get("end", 0.0)) for seg in segments]
-        return SimpleNamespace(text=getattr(result, "text", ""), sentences=sentences)
-
-    def transcribe_incremental(self, audio: np.ndarray) -> PartialTranscription:
-        """
-        Transcribe audio incrementally (for live streaming).
-
-        Call this repeatedly with growing audio buffer.
-        Returns a single PartialTranscription for current state.
-        """
-        # Skip if not enough new audio
-        current_length = len(audio)
-        if current_length < self.sample_rate * 0.5:  # Need at least 500ms
-            return PartialTranscription(
-                fixed_text=" ".join(self.fixed_sentences),
-                active_text="",
-                timestamp=current_length / self.sample_rate,
-                is_final=False,
-            )
-
-        # Skip if no new audio since last transcription
-        if current_length == self.last_transcribed_length:
-            return PartialTranscription(
-                fixed_text=" ".join(self.fixed_sentences),
-                active_text="",
-                timestamp=current_length / self.sample_rate,
-                is_final=False,
-            )
-
-        self.last_transcribed_length = current_length
-
-        # Extract window for transcription (from last fixed sentence to end)
-        window_start_samples = int(self.fixed_end_time * self.sample_rate)
-        audio_window = audio[window_start_samples:]
-
-        # Transcribe current window
-        result = self._decode_window(audio_window)
-
-        # Check if window exceeds max_window_size
-        window_duration = len(audio_window) / self.sample_rate
-
-        if window_duration >= self.max_window_size and len(result.sentences) > 1:
-            # Window is too large - fix some sentences
-            cutoff_time = window_duration - self.sentence_buffer
-
-            # Find sentences to fix
-            new_fixed_sentences = []
-            new_fixed_end_time = self.fixed_end_time
-
-            for sentence in result.sentences:
-                sentence_abs_time = self.fixed_end_time + sentence.end
-
-                if sentence.end < cutoff_time:
-                    # Fix this sentence
-                    new_fixed_sentences.append(sentence.text.strip())
-                    new_fixed_end_time = sentence_abs_time
-                else:
-                    break
-
-            if new_fixed_sentences:
-                self.fixed_sentences.extend(new_fixed_sentences)
-                self.fixed_end_time = new_fixed_end_time
-
-                # Re-transcribe from new fixed point
-                window_start_samples = int(self.fixed_end_time * self.sample_rate)
-                audio_window = audio[window_start_samples:]
-                result = self._decode_window(audio_window)
-
-        # Build output
-        fixed_text = " ".join(self.fixed_sentences)
-        active_text = result.text.strip()
-        timestamp = len(audio) / self.sample_rate
-
-        return PartialTranscription(fixed_text=fixed_text, active_text=active_text, timestamp=timestamp, is_final=False)
-
-    def transcribe_progressive(self, audio: np.ndarray) -> Generator[PartialTranscription, None, None]:
-        """
-        Transcribe audio with smart progressive emissions.
-
-        Yields PartialTranscription with:
-        - fixed_text: Completed sentences (won't change)
-        - active_text: Current partial transcription
-        - timestamp: Current position
-        - is_final: True for last update
-        """
-        position = 0  # Start of current window (in samples)
-        fixed_sentences = []  # List of completed sentence texts
-        fixed_end_time = 0.0  # End time of last fixed sentence
-
-        while position < len(audio):
-            # Determine current window end
-            remaining = (len(audio) - position) / self.sample_rate
-
-            if remaining <= self.emission_interval:
-                # Last chunk - process everything remaining
-                window_end = len(audio)
-                is_final = True
-            else:
-                # Regular chunk
-                window_end = min(len(audio), position + int(self.emission_interval * self.sample_rate))
-                is_final = False
-
-            # Extract window for transcription
-            # Window includes: fixed_end_time to current position
-            # (we re-transcribe from last fixed sentence to get better context)
-            window_start_samples = int(fixed_end_time * self.sample_rate)
-            audio_window = audio[window_start_samples:window_end]
-
-            # Transcribe current window
-            result = self._decode_window(audio_window)
-
-            # Check if window exceeds max_window_size
-            window_duration = (window_end - window_start_samples) / self.sample_rate
-
-            if window_duration >= self.max_window_size and len(result.sentences) > 1:
-                # Window is too large - need to fix some sentences
-                # Strategy: Keep sentences up to (max_window - sentence_buffer) as fixed
-                cutoff_time = window_duration - self.sentence_buffer
-
-                # Find sentences that are completed and before cutoff
-                new_fixed_sentences = []
-                new_fixed_end_time = fixed_end_time
-
-                for sentence in result.sentences:
-                    sentence_abs_time = fixed_end_time + sentence.end
-
-                    if sentence.end < cutoff_time:
-                        # This sentence is complete and old enough to fix
-                        new_fixed_sentences.append(sentence.text.strip())
-                        new_fixed_end_time = sentence_abs_time
-                    else:
-                        # Stop here - keep remaining sentences as active
-                        break
-
-                if new_fixed_sentences:
-                    fixed_sentences.extend(new_fixed_sentences)
-                    fixed_end_time = new_fixed_end_time
-
-                    # Re-transcribe from new fixed_end_time
-                    window_start_samples = int(fixed_end_time * self.sample_rate)
-                    audio_window = audio[window_start_samples:window_end]
-                    result = self._decode_window(audio_window)
-
-            # Build output
-            fixed_text = " ".join(fixed_sentences)
-            active_text = result.text.strip()
-            timestamp = window_end / self.sample_rate
-
-            yield PartialTranscription(
-                fixed_text=fixed_text, active_text=active_text, timestamp=timestamp, is_final=is_final
-            )
-
-            # Move to next position
-            position = window_end
-
-            if is_final:
-                break
-
-
-def demo_smart_progressive() -> None:
-    """Demonstrate smart progressive streaming."""
-    import time
-
-    import soundfile as sf
-    from mlx_audio.stt.generate import load_model
-
-    print("=" * 80)
-    print("SMART PROGRESSIVE STREAMING DEMO")
-    print("=" * 80)
-    print("\nFeatures:")
-    print("  • Partial updates every 500ms")
-    print("  • Growing window up to 15s")
-    print("  • Sentence-aware window sliding for long audio")
-    print()
-
-    # Load model
-    print("Loading model...")
-    model = load_model("mlx-community/parakeet-tdt-0.6b-v3")
-
-    # Load audio
-    audio, sr = sf.read("reachy-voice-test.wav")
-    if len(audio.shape) > 1:
-        audio = audio.mean(axis=1)
-    if sr != 16000:
-        from scipy import signal
-
-        audio = signal.resample(audio, int(len(audio) * 16000 / sr))
-    audio = audio.astype(np.float32)
-
-    # Test cases
-    test_cases = [
-        (audio, "Short (5.8s)"),
-        (np.concatenate([audio] * 3), "Medium (17.5s) - exceeds 15s window"),
-        (np.concatenate([audio] * 5), "Long (29.1s) - multiple sentence fixings"),
-    ]
-
-    for test_audio, label in test_cases:
-        duration = len(test_audio) / 16000
-
-        print(f"\n{'=' * 80}")
-        print(f"TEST: {label} - Duration: {duration:.1f}s")
-        print("=" * 80)
-
-        handler = SmartProgressiveStreamingHandler(
-            model,
-            emission_interval=0.5,  # 500ms updates
-            max_window_size=15.0,  # Max 15s window
-            sentence_buffer=2.0,  # Keep 2s of sentences in active window
-        )
-
-        print("\nProgressive transcriptions (showing every 4th update for readability):")
-        print()
-
-        update_count = 0
-        start_time = time.perf_counter()
-
-        for result in handler.transcribe_progressive(test_audio):
-            update_count += 1
-
-            # Show every other update (every 1s) to keep output manageable
-            if update_count % 2 == 0 or result.is_final:
-                elapsed = time.perf_counter() - start_time
-                marker = "FINAL" if result.is_final else f"Update {update_count}"
-
-                print(f"[{result.timestamp:5.1f}s | {elapsed:.3f}s elapsed] {marker}:")
-
-                if result.fixed_text:
-                    print(f"  Fixed:  {result.fixed_text[:70]}...")
-                print(f"  Active: {result.active_text[:70]}...")
-                print()
-
-        total_time = time.perf_counter() - start_time
-        print(f"Total updates: {update_count}")
-        print(f"Total time: {total_time:.3f}s ({total_time / duration:.3f}s per audio second)")
-
-    print("\n" + "=" * 80)
-    print("KEY INSIGHTS")
-    print("=" * 80)
-    print("""
-1. Frequent updates (500ms):
-   - User sees transcription building in real-time
-   - Low latency feel
-
-2. Growing window (up to 15s):
-   - Better accuracy with more context
-   - No arbitrary chunking for short audio
-
-3. Sentence-aware sliding (for audio > 15s):
-   - Completed sentences become "fixed"
-   - Only active portion re-transcribed
-   - Maintains accuracy while managing window size
-
-4. Performance:
-   - Processing happens DURING user speaking
-   - Final result available immediately when user stops
-    """)
-
-
-if __name__ == "__main__":
-    demo_smart_progressive()

runtime/src/speech_to_speech/STT/transcription_notifier.py

@@ -1,103 +0,0 @@
-from __future__ import annotations
-
-import logging
-from queue import Queue
-from threading import Event
-from typing import Iterator, Union
-
-from speech_to_speech.api.openai_realtime.runtime_config import RuntimeConfig
-from speech_to_speech.baseHandler import BaseHandler
-from speech_to_speech.LLM.chat import make_user_message
-from speech_to_speech.pipeline.events import PartialTranscriptionEvent, TranscriptionCompletedEvent
-from speech_to_speech.pipeline.handler_types import LLMIn, STTOut
-from speech_to_speech.pipeline.messages import GenerateResponseRequest, PartialTranscription, Transcription
-from speech_to_speech.pipeline.queue_types import TextEventItem
-
-logger = logging.getLogger(__name__)
-
-
-class TranscriptionNotifier(BaseHandler[STTOut, Union[STTOut, LLMIn]]):
-    """Sits between STT and LLM.
-
-    For **realtime mode** (no ``runtime_config``): emits transcription events
-    on ``text_output_queue`` for protocol translation but yields nothing -- the
-    ``RealtimeService`` builds ``GenerateResponseRequest`` directly.
-
-    For **legacy mode** (``runtime_config`` provided): appends the user
-    message to ``runtime_config.chat`` and yields a
-    ``GenerateResponseRequest`` so the LLM handler receives a uniform input
-    type regardless of pipeline mode.
-    """
-
-    def setup(
-        self,
-        text_output_queue: Queue[TextEventItem] | None = None,
-        runtime_config: RuntimeConfig | None = None,
-        should_listen: Event | None = None,
-    ) -> None:
-        self.text_output_queue = text_output_queue
-        self.runtime_config = runtime_config
-        self.should_listen = should_listen
-
-    def process(self, transcription: STTOut) -> Iterator[Union[STTOut, LLMIn]]:
-        if isinstance(transcription, PartialTranscription):
-            if self.text_output_queue and transcription.text:
-                self.text_output_queue.put(
-                    PartialTranscriptionEvent(
-                        delta=str(transcription.text),
-                        turn_id=transcription.turn_id,
-                        turn_revision=transcription.turn_revision,
-                    )
-                )
-                logger.debug("Partial transcription: %s", str(transcription.text)[:80])
-            return
-
-        if isinstance(transcription, Transcription):
-            text = transcription.text
-            language_code = transcription.language_code
-            turn_id = transcription.turn_id
-            turn_revision = transcription.turn_revision
-            speech_stopped_at_s = transcription.speech_stopped_at_s
-        else:
-            text = transcription
-            language_code = None
-            turn_id = None
-            turn_revision = None
-            speech_stopped_at_s = None
-
-        transcript = str(text)
-        # Always close the client-visible transcription item. Empty final STT
-        # results should not trigger the LLM, but clients may already have
-        # received partial deltas and still need a completed event.
-        if self.text_output_queue is not None:
-            self.text_output_queue.put(
-                TranscriptionCompletedEvent(
-                    transcript=transcript,
-                    language_code=language_code,
-                    turn_id=turn_id,
-                    turn_revision=turn_revision,
-                    speech_stopped_at_s=speech_stopped_at_s,
-                )
-            )
-
-        if not transcript:
-            logger.debug("Transcription completed with empty transcript")
-            if self.should_listen is not None:
-                self.should_listen.set()
-                logger.debug("Empty transcription completed; listening re-enabled")
-            return
-
-        if language_code:
-            logger.info("Transcription completed (language=%s): %s", language_code, transcript)
-        else:
-            logger.info("Transcription completed: %s", transcript)
-
-        if self.runtime_config is not None:
-            self.runtime_config.chat.add_item(make_user_message(transcript))
-            yield GenerateResponseRequest(
-                runtime_config=self.runtime_config,
-                language_code=language_code,
-                turn_id=turn_id,
-                turn_revision=turn_revision,
-                speech_stopped_at_s=speech_stopped_at_s,
-            )

runtime/src/speech_to_speech/STT/whisper_stt_handler.py

@@ -1,147 +0,0 @@
-from __future__ import annotations
-
-import logging
-from copy import copy
-from typing import Any, Iterator, Optional
-
-import numpy as np
-import torch
-from rich.console import Console
-from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor
-
-from speech_to_speech.pipeline.handler_types import STTIn, STTOut
-from speech_to_speech.pipeline.messages import Transcription
-from speech_to_speech.STT.base_stt_handler import BaseSTTHandler
-
-logger = logging.getLogger(__name__)
-console = Console()
-
-SUPPORTED_LANGUAGES = [
-    "en",
-    "fr",
-    "es",
-    "zh",
-    "ja",
-    "ko",
-    "hi",
-    "de",
-    "pt",
-    "pl",
-    "it",
-    "nl",
-]
-
-
-class WhisperSTTHandler(BaseSTTHandler):
-    """
-    Handles the Speech To Text generation using a Whisper model.
-    """
-
-    def setup(
-        self,
-        model_name: str = "distil-whisper/distil-large-v3",
-        device: str = "cuda",
-        torch_dtype: str = "float16",
-        compile_mode: Optional[str] = None,
-        language: Optional[str] = None,
-        gen_kwargs: dict[str, Any] = {},
-    ) -> None:
-        self.device = device
-        self.torch_dtype = getattr(torch, torch_dtype)
-        self.compile_mode = compile_mode
-        self.gen_kwargs = gen_kwargs
-        self.start_language = language
-        self.last_language = language if language != "auto" else None
-        if self.last_language is not None:
-            self.gen_kwargs["language"] = self.last_language
-
-        self.processor = AutoProcessor.from_pretrained(model_name)
-        self.model = AutoModelForSpeechSeq2Seq.from_pretrained(
-            model_name,
-            torch_dtype=self.torch_dtype,
-        ).to(device)
-
-        # compile
-        if self.compile_mode:
-            self.model.generation_config.cache_implementation = "static"
-            self.model.forward = torch.compile(self.model.forward, mode=self.compile_mode, fullgraph=True)
-        self.warmup()
-
-    def prepare_model_inputs(self, audio: np.ndarray) -> torch.Tensor:
-        input_features = self.processor(audio, sampling_rate=16000, return_tensors="pt").input_features
-        input_features = input_features.to(self.device, dtype=self.torch_dtype)
-
-        return input_features
-
-    def warmup(self) -> None:
-        logger.info(f"Warming up {self.__class__.__name__}")
-
-        # 2 warmup steps for no compile or compile mode with CUDA graphs capture
-        n_steps = 1 if self.compile_mode == "default" else 2
-        dummy_input = torch.randn(
-            (1, self.model.config.num_mel_bins, 3000),
-            dtype=self.torch_dtype,
-            device=self.device,
-        )
-        if self.compile_mode not in (None, "default"):
-            # generating more tokens than previously will trigger CUDA graphs capture
-            # one should warmup with a number of generated tokens above max tokens targeted for subsequent generation
-            # hence, having min_new_tokens < max_new_tokens in the future doesn't make sense
-            warmup_gen_kwargs = {
-                "min_new_tokens": self.gen_kwargs["max_new_tokens"],  # Yes, assign max_new_tokens to min_new_tokens
-                "max_new_tokens": self.gen_kwargs["max_new_tokens"],
-                **self.gen_kwargs,
-            }
-        else:
-            warmup_gen_kwargs = self.gen_kwargs
-
-        if self.device == "cuda":
-            start_event = torch.cuda.Event(enable_timing=True)
-            end_event = torch.cuda.Event(enable_timing=True)
-            torch.cuda.synchronize()
-            start_event.record()
-
-        for _ in range(n_steps):
-            _ = self.model.generate(dummy_input, **warmup_gen_kwargs)
-
-        if self.device == "cuda":
-            end_event.record()
-            torch.cuda.synchronize()
-
-            logger.info(
-                f"{self.__class__.__name__}:  warmed up! time: {start_event.elapsed_time(end_event) * 1e-3:.3f} s"
-            )
-
-    def process(self, vad_audio: STTIn) -> Iterator[STTOut]:
-        logger.debug("infering whisper...")
-
-        input_features = self.prepare_model_inputs(vad_audio.audio)
-        pred_ids = self.model.generate(input_features, **self.gen_kwargs)
-        language_code = self.processor.tokenizer.decode(pred_ids[0, 1])[2:-2]  # remove "<|" and "|>"
-
-        if language_code not in SUPPORTED_LANGUAGES:  # reprocess with the last language
-            logger.warning("Whisper detected unsupported language: %s", language_code)
-            gen_kwargs = copy(self.gen_kwargs)
-            gen_kwargs["language"] = self.last_language
-            language_code = self.last_language
-            pred_ids = self.model.generate(input_features, **gen_kwargs)
-        else:
-            self.last_language = language_code
-
-        pred_text = self.processor.batch_decode(pred_ids, skip_special_tokens=True, decode_with_timestamps=False)[0]
-        language_code = self.processor.tokenizer.decode(pred_ids[0, 1])[2:-2]  # remove "<|" and "|>"
-
-        logger.debug("finished whisper inference")
-        console.print(f"[yellow]USER: {pred_text}")
-        logger.debug(f"Language Code Whisper: {language_code}")
-
-        if self.start_language == "auto":
-            language_code += "-auto"
-
-        yield Transcription(
-            text=pred_text,
-            language_code=language_code,
-            turn_id=vad_audio.turn_id,
-            turn_revision=vad_audio.turn_revision,
-            speech_stopped_at_s=vad_audio.created_at_s,
-        )

runtime/src/speech_to_speech/TTS/README.md

@@ -1,155 +0,0 @@
-# TTS Summary
-
-## Available TTS modes (`--tts`)
-
-Runtime-supported values in `s2s_pipeline.py`:
-
-- `chatTTS` → `chatTTS_handler.py`
-- `facebookMMS` → `facebookmms_handler.py`
-- `pocket` → `pocket_tts_handler.py`
-- `kokoro` → `kokoro_handler.py`
-- `qwen3` → `qwen3_tts_handler.py`
-
-Deprecated TTS implementations, including MeloTTS, live in [`../../../archive/TTS`](../../../archive/TTS) and are no longer wired into `s2s_pipeline.py`.
-
-## Usage
-
-### 1) ChatTTS (`--tts chatTTS`)
-
-Primary args prefix: `--chat_tts_*`
-
-```bash
-python s2s_pipeline.py \
-  --tts chatTTS \
-  --chat_tts_device cuda \
-  --chat_tts_stream true \
-  --chat_tts_chunk_size 512
-```
-
-### 2) Facebook MMS (`--tts facebookMMS`)
-
-Primary args prefix: `--facebook_mms_*` plus `--tts_language`
-
-```bash
-python s2s_pipeline.py \
-  --tts facebookMMS \
-  --facebook_mms_device cuda \
-  --tts_language en
-```
-
-This handler maps STT language codes (e.g. `en`, `fr`, `es`) to MMS model suffixes (e.g. `eng`, `fra`, `spa`) and reloads the model on language changes.
-
-### 3) Pocket TTS (`--tts pocket`)
-
-Primary args prefix: `--pocket_tts_*`
-
-```bash
-python s2s_pipeline.py \
-  --tts pocket \
-  --pocket_tts_voice jean \
-  --pocket_tts_device cpu \
-  --pocket_tts_sample_rate 16000
-```
-
-Available preset voices include:
-`alba`, `marius`, `javert`, `jean`, `fantine`, `cosette`, `eponine`, `azelma`.
-
-### 4) Kokoro (`--tts kokoro`)
-
-Primary args prefix: `--kokoro_*`
-
-```bash
-python s2s_pipeline.py \
-  --tts kokoro \
-  --kokoro_device auto \
-  --kokoro_voice bm_fable \
-  --kokoro_lang_code b
-```
-
-Behavior:
-- Uses MLX backend on Apple Silicon (`mlx-community/Kokoro-82M-bf16`)
-- Uses native kokoro pipeline otherwise (`hexgrad/Kokoro-82M`)
-- Can auto-switch voice/language based on STT language code mapping
-
-### 5) Qwen3-TTS (`--tts qwen3`)
-
-Primary args prefix: `--qwen3_tts_*`
-
-```bash
-python s2s_pipeline.py \
-  --tts qwen3 \
-  --qwen3_tts_model_name Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-  --qwen3_tts_device cuda \
-  --qwen3_tts_backend ggml \
-  --qwen3_tts_speaker Aiden \
-  --qwen3_tts_language auto \
-  --qwen3_tts_non_streaming_mode True
-```
-
-Behavior:
-- Uses `faster-qwen3-tts` on non-macOS platforms, defaulting to the GGML backend. Pass `--qwen3_tts_backend torch` to use the CUDA-graphs backend instead.
-- Uses `mlx-audio` on Apple Silicon and auto-maps `Qwen/...` model IDs to `mlx-community/...`, defaulting to the `6bit` MLX variant unless the model name already pins a suffix.
-- Supports MLX quantization overrides on Apple Silicon via `--qwen3_tts_mlx_quantization bf16|4bit|6bit|8bit`.
-- Keeps the existing voice-clone/custom-voice/voice-design handler flow intact.
-- Defaults to the CustomVoice model with speaker `Aiden`, so no reference audio is required. Voice-clone/base models can still use `--qwen3_tts_ref_audio`.
-
-Install notes for Linux GGML:
-- The default PyPI `qwentts-cpp-python` wheel targets CUDA 12.8.
-- If that wheel does not match your CUDA runtime, install one of the Hugging Face wheelhouse builds before installing `speech-to-speech`.
-
-```bash
-pip install "qwentts-cpp-python==0.3.1+cu130" \
-  -f https://huggingface.co/datasets/andito/qwentts-cpp-python-wheels/tree/main/whl/cu130
-pip install speech-to-speech
-```
-
-Available wheelhouse directories include `cu124`, `cu128`, `cu130`, and `cpu`.
-
-Example for Apple Silicon using the default 6-bit MLX variant:
-
-```bash
-python s2s_pipeline.py \
-  --tts qwen3 \
-  --qwen3_tts_model_name Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-  --qwen3_tts_speaker Aiden
-```
-
-You can override the default and select `bf16`, `4bit`, or `8bit` explicitly:
-
-```bash
-python s2s_pipeline.py \
-  --tts qwen3 \
-  --qwen3_tts_model_name Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-  --qwen3_tts_mlx_quantization 4bit \
-  --qwen3_tts_speaker Aiden
-```
-
-To benchmark the Apple Silicon MLX variants side by side:
-
-```bash
-.venv/bin/python benchmark_tts.py \
-  --handlers qwen3 \
-  --iterations 3 \
-  --qwen3_mlx_quantizations bf16 4bit 6bit 8bit
-```
-
-This will run separate benchmark entries for `qwen3[bf16]`, `qwen3[4bit]`, `qwen3[6bit]`, and `qwen3[8bit]`.
-
-## Setup
-
-### Low-latency GPU setup
-
-```bash
-python s2s_pipeline.py \
-  --stt whisper \
-  --tts pocket
-```
-
-### Apple Silicon setup
-
-```bash
-python s2s_pipeline.py \
-  --local_mac_optimal_settings
-```
-
-`--tts pocket` and `--tts kokoro` are also valid options on macOS.

runtime/src/speech_to_speech/TTS/__init__.py

@@ -1 +0,0 @@
-# TTS module

runtime/src/speech_to_speech/TTS/chatTTS_handler.py

@@ -1,119 +0,0 @@
-from __future__ import annotations
-
-import logging
-from threading import Event
-from typing import Any, Iterator
-
-import ChatTTS
-import librosa
-import numpy as np
-import torch
-from rich.console import Console
-
-from speech_to_speech.baseHandler import BaseHandler
-from speech_to_speech.pipeline.cancel_scope import CancelScope
-from speech_to_speech.pipeline.handler_types import TTSIn, TTSOut
-from speech_to_speech.pipeline.messages import AUDIO_RESPONSE_DONE, EndOfResponse
-from speech_to_speech.pipeline.speculative_turns import SpeculativeTurnTracker
-
-logging.basicConfig(
-    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-)
-
-logger = logging.getLogger(__name__)
-
-console = Console()
-
-
-class ChatTTSHandler(BaseHandler[TTSIn, TTSOut]):
-    def setup(
-        self,
-        should_listen: Event,
-        device: str = "cuda",
-        gen_kwargs: dict[str, Any] = {},  # Unused
-        stream: bool = True,
-        chunk_size: int = 512,
-        cancel_scope: CancelScope | None = None,
-        speculative_turns: SpeculativeTurnTracker | None = None,
-    ) -> None:
-        self.should_listen = should_listen
-        self.cancel_scope = cancel_scope
-        self.speculative_turns = speculative_turns
-        self.device = device
-        self.model = ChatTTS.Chat()
-        self.model.load(compile=False)  # Doesn't work for me with True
-        self.chunk_size = chunk_size
-        self.stream = stream
-        rnd_spk_emb = self.model.sample_random_speaker()
-        self.params_infer_code = ChatTTS.Chat.InferCodeParams(
-            spk_emb=rnd_spk_emb,
-        )
-        self.warmup()
-
-    def warmup(self) -> None:
-        logger.info(f"Warming up {self.__class__.__name__}")
-        _ = self.model.infer("text")
-
-    def process(self, tts_input: TTSIn) -> Iterator[TTSOut]:
-        speculative_turns = getattr(self, "speculative_turns", None)
-        if isinstance(tts_input, EndOfResponse):
-            if speculative_turns and not speculative_turns.is_latest_after_reopen_grace(
-                tts_input.turn_id,
-                tts_input.turn_revision,
-            ):
-                return
-            yield AUDIO_RESPONSE_DONE
-            return
-
-        if speculative_turns and not speculative_turns.is_latest_after_reopen_grace(
-            tts_input.turn_id,
-            tts_input.turn_revision,
-        ):
-            logger.debug("Dropping stale TTS input for turn=%s rev=%s", tts_input.turn_id, tts_input.turn_revision)
-            return
-        if speculative_turns:
-            speculative_turns.commit(tts_input.turn_id, tts_input.turn_revision)
-
-        text = tts_input.text
-
-        _cancel_gen = self.cancel_scope.generation if self.cancel_scope else None
-        console.print(f"[green]ASSISTANT: {text}")
-        if self.device == "mps":
-            import time
-
-            start = time.time()
-            torch.mps.synchronize()  # Waits for all kernels in all streams on the MPS device to complete.
-            torch.mps.empty_cache()  # Frees all memory allocated by the MPS device.
-            _ = time.time() - start  # Removing this line makes it fail more often. I'm looking into it.
-
-        wavs_gen = self.model.infer(text, params_infer_code=self.params_infer_code, stream=self.stream)
-
-        if self.stream:
-            wavs = [np.array([])]
-            for gen in wavs_gen:
-                if (
-                    _cancel_gen is not None
-                    and self.cancel_scope is not None
-                    and self.cancel_scope.is_stale(_cancel_gen)
-                ):
-                    logger.info("TTS generation cancelled (interruption)")
-                    return
-                if gen[0] is None or len(gen[0]) == 0:
-                    return
-                audio_chunk = librosa.resample(gen[0], orig_sr=24000, target_sr=16000)
-                audio_chunk = (audio_chunk * 32768).astype(np.int16)[0]
-                while len(audio_chunk) > self.chunk_size:
-                    yield audio_chunk[: self.chunk_size]  # Return the first chunk_size samples of the audio data
-                    audio_chunk = audio_chunk[self.chunk_size :]  # Remove the samples that have already been returned
-                yield np.pad(audio_chunk, (0, self.chunk_size - len(audio_chunk)))
-        else:
-            wavs = wavs_gen
-            if len(wavs[0]) == 0:
-                return
-            audio_chunk = librosa.resample(wavs[0], orig_sr=24000, target_sr=16000)
-            audio_chunk = (audio_chunk * 32768).astype(np.int16)
-            for i in range(0, len(audio_chunk), self.chunk_size):
-                yield np.pad(
-                    audio_chunk[i : i + self.chunk_size],
-                    (0, self.chunk_size - len(audio_chunk[i : i + self.chunk_size])),
-                )

runtime/src/speech_to_speech/TTS/facebookmms_handler.py

@@ -1,212 +0,0 @@
-from __future__ import annotations
-
-import logging
-from threading import Event
-from typing import Any, Iterator
-
-import librosa
-import numpy as np
-import torch
-from rich.console import Console
-from transformers import AutoTokenizer, VitsModel
-
-from speech_to_speech.baseHandler import BaseHandler
-from speech_to_speech.pipeline.cancel_scope import CancelScope
-from speech_to_speech.pipeline.handler_types import TTSIn, TTSOut
-from speech_to_speech.pipeline.messages import AUDIO_RESPONSE_DONE, EndOfResponse
-from speech_to_speech.pipeline.speculative_turns import SpeculativeTurnTracker
-
-logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.DEBUG)
-
-logger = logging.getLogger(__name__)
-
-console = Console()
-
-WHISPER_LANGUAGE_TO_FACEBOOK_LANGUAGE = {
-    "en": "eng",  # English
-    "fr": "fra",  # French
-    "es": "spa",  # Spanish
-    "ko": "kor",  # Korean
-    "hi": "hin",  # Hindi
-    "ar": "ara",  # Arabic
-    "hy": "hyw",  # Armenian
-    "az": "azb",  # Azerbaijani
-    "bu": "bul",  # Bulgarian
-    "ca": "cat",  # Catalan
-    "nl": "nld",  # Dutch
-    "fi": "fin",  # Finnish
-    "de": "deu",  # German
-    "el": "ell",  # Greek
-    "he": "heb",  # Hebrew
-    "hu": "hun",  # Hungarian
-    "is": "isl",  # Icelandic
-    "id": "ind",  # Indonesian
-    "ka": "kan",  # Kannada
-    "kk": "kaz",  # Kazakh
-    "lv": "lav",  # Latvian
-    "zl": "zlm",  # Malay
-    "ma": "mar",  # Marathi
-    "fa": "fas",  # Persian
-    "po": "pol",  # Polish
-    "pt": "por",  # Portuguese
-    "ro": "ron",  # Romanian
-    "ru": "rus",  # Russian
-    "sw": "swh",  # Swahili
-    "sv": "swe",  # Swedish
-    "tg": "tgl",  # Tagalog
-    "ta": "tam",  # Tamil
-    "th": "tha",  # Thai
-    "tu": "tur",  # Turkish
-    "uk": "ukr",  # Ukrainian
-    "ur": "urd",  # Urdu
-    "vi": "vie",  # Vietnamese
-    "cy": "cym",  # Welsh
-}
-
-
-class FacebookMMSTTSHandler(BaseHandler[TTSIn, TTSOut]):
-    def setup(
-        self,
-        should_listen: Event,
-        device: str = "cuda",
-        torch_dtype: str = "float32",
-        language: str = "en",
-        stream: bool = True,
-        chunk_size: int = 512,
-        cancel_scope: CancelScope | None = None,
-        speculative_turns: SpeculativeTurnTracker | None = None,
-        **kwargs: Any,
-    ) -> None:
-        self.should_listen = should_listen
-        self.cancel_scope = cancel_scope
-        self.speculative_turns = speculative_turns
-        self.device = device
-        self.torch_dtype = getattr(torch, torch_dtype)
-        self.stream = stream
-        self.chunk_size = chunk_size
-        self.language = language
-
-        self._initial_language = self.language
-        self.load_model(self.language)
-        self.warmup()
-
-    def load_model(self, language_code: str) -> None:
-        try:
-            model_name = f"facebook/mms-tts-{WHISPER_LANGUAGE_TO_FACEBOOK_LANGUAGE[language_code]}"
-            logger.info(f"Loading model: {model_name}")
-            self.model = VitsModel.from_pretrained(model_name).to(self.device)  # type: ignore[arg-type]
-            self.tokenizer = AutoTokenizer.from_pretrained(model_name)
-            self.language = language_code
-        except KeyError:
-            logger.warning(f"Unsupported language: {language_code}. Falling back to English.")
-            self.load_model("en")
-
-    def warmup(self) -> None:
-        logger.info(f"Warming up {self.__class__.__name__}")
-        self.generate_audio("Hello, this is a test")
-
-    def generate_audio(self, text: str) -> torch.Tensor | None:
-        if not text:
-            logger.warning("Received empty text input")
-            return None
-
-        try:
-            logger.debug(f"Tokenizing text: {text}")
-            logger.debug(f"Current language: {self.language}")
-            logger.debug(f"Tokenizer: {self.tokenizer}")
-
-            inputs = self.tokenizer(text, return_tensors="pt", padding=True, truncation=True)
-            input_ids = inputs.input_ids.to(self.device).long()
-            attention_mask = inputs.attention_mask.to(self.device)
-
-            logger.debug(f"Input IDs shape: {input_ids.shape}, dtype: {input_ids.dtype}")
-            logger.debug(f"Input IDs: {input_ids}")
-
-            if input_ids.numel() == 0:
-                logger.error("Input IDs tensor is empty")
-                return None
-
-            with torch.no_grad():
-                output = self.model(input_ids=input_ids, attention_mask=attention_mask)
-
-            logger.debug(f"Output waveform shape: {output.waveform.shape}")
-            return output.waveform
-        except Exception as e:
-            logger.error(f"Error in generate_audio: {str(e)}")
-            logger.exception("Full traceback:")
-            return None
-
-    def process(self, tts_input: TTSIn) -> Iterator[TTSOut]:
-        speculative_turns = getattr(self, "speculative_turns", None)
-        if isinstance(tts_input, EndOfResponse):
-            if speculative_turns and not speculative_turns.is_latest_after_reopen_grace(
-                tts_input.turn_id,
-                tts_input.turn_revision,
-            ):
-                return
-            yield AUDIO_RESPONSE_DONE
-            return
-
-        if speculative_turns and not speculative_turns.is_latest_after_reopen_grace(
-            tts_input.turn_id,
-            tts_input.turn_revision,
-        ):
-            logger.debug("Dropping stale TTS input for turn=%s rev=%s", tts_input.turn_id, tts_input.turn_revision)
-            return
-        if speculative_turns:
-            speculative_turns.commit(tts_input.turn_id, tts_input.turn_revision)
-
-        gen = self.cancel_scope.generation if self.cancel_scope else None
-        language_code = tts_input.language_code
-        text = tts_input.text
-
-        console.print(f"[green]ASSISTANT: {text}")
-        logger.debug(f"Processing text: {text}")
-        logger.debug(f"Language code: {language_code}")
-
-        if language_code is not None and self.language != language_code:
-            try:
-                logger.info(f"Switching language from {self.language} to {language_code}")
-                self.load_model(language_code)
-            except KeyError:
-                console.print(
-                    f"[red]Language {language_code} not supported by Facebook MMS. Using {self.language} instead."
-                )
-                logger.warning(f"Unsupported language: {language_code}")
-
-        audio_output = self.generate_audio(text)
-
-        if audio_output is None or audio_output.numel() == 0:
-            logger.warning("No audio output generated")
-            return
-
-        audio_numpy = audio_output.cpu().numpy().squeeze()
-        logger.debug(f"Raw audio shape: {audio_numpy.shape}, dtype: {audio_numpy.dtype}")
-
-        audio_resampled = librosa.resample(audio_numpy, orig_sr=self.model.config.sampling_rate, target_sr=16000)
-        logger.debug(f"Resampled audio shape: {audio_resampled.shape}, dtype: {audio_resampled.dtype}")
-
-        audio_int16 = (audio_resampled * 32768).astype(np.int16)
-        logger.debug(f"Final audio shape: {audio_int16.shape}, dtype: {audio_int16.dtype}")
-
-        if self.stream:
-            for i in range(0, len(audio_int16), self.chunk_size):
-                if gen is not None and self.cancel_scope is not None and self.cancel_scope.is_stale(gen):
-                    logger.info("TTS generation cancelled (interruption)")
-                    return
-                chunk = audio_int16[i : i + self.chunk_size]
-                yield np.pad(chunk, (0, self.chunk_size - len(chunk)))
-        else:
-            for i in range(0, len(audio_int16), self.chunk_size):
-                if gen is not None and self.cancel_scope is not None and self.cancel_scope.is_stale(gen):
-                    logger.info("TTS generation cancelled (interruption)")
-                    return
-                yield np.pad(
-                    audio_int16[i : i + self.chunk_size],
-                    (0, self.chunk_size - len(audio_int16[i : i + self.chunk_size])),
-                )
-
-    def on_session_end(self) -> None:
-        if self.language != self._initial_language:
-            self.load_model(self._initial_language)
-        logger.debug("Facebook MMS TTS session state reset")

runtime/src/speech_to_speech/TTS/kokoro_handler.py

@@ -1,418 +0,0 @@
-"""
-Kokoro TTS Handler
-
-Supports NVIDIA Kokoro TTS model for high-quality multilingual speech synthesis.
-- On Apple Silicon (MPS): Uses mlx-audio with mlx-community/Kokoro-82M-bf16
-- On CUDA/CPU: Uses native kokoro library with hexgrad/Kokoro-82M
-
-Model supports 8 languages with multiple voices per language.
-"""
-
-from __future__ import annotations
-
-import logging
-from sys import platform
-from threading import Event
-from typing import Any, Iterator, Optional
-
-import numpy as np
-from rich.console import Console
-
-from speech_to_speech.baseHandler import BaseHandler
-from speech_to_speech.pipeline.cancel_scope import CancelScope
-from speech_to_speech.pipeline.handler_types import TTSIn, TTSOut
-from speech_to_speech.pipeline.messages import AUDIO_RESPONSE_DONE, EndOfResponse
-from speech_to_speech.pipeline.speculative_turns import SpeculativeTurnTracker
-from speech_to_speech.utils.mlx_lock import MLXLockContext
-
-logger = logging.getLogger(__name__)
-console = Console()
-
-# Language code mapping from Whisper/langdetect language codes to Kokoro lang codes
-WHISPER_LANGUAGE_TO_KOKORO_LANG = {
-    "en": "b",  # British English
-    "ja": "j",  # Japanese
-    "zh": "z",  # Chinese
-    "fr": "f",  # French
-    "es": "e",  # Spanish
-    "it": "i",  # Italian
-    "pt": "p",  # Portuguese
-    "hi": "h",  # Hindi
-    # Additional European languages from Parakeet - map to closest Kokoro language
-    "de": "b",  # German -> British English (no German voice available)
-    "nl": "b",  # Dutch -> British English
-    "pl": "b",  # Polish -> British English
-    "ru": "b",  # Russian -> British English
-    "uk": "b",  # Ukrainian -> British English
-}
-
-# Default voices for each Kokoro language code
-# These are native voices that sound natural for each language
-# Voice naming: first letter = language, second letter = gender (f=female, m=male)
-# Available voices per language:
-#   a (American): af_alloy, af_aoede, af_bella, af_heart, af_jessica, af_kore, af_nicole, af_nova, af_river, af_sarah, af_sky
-#                 am_adam, am_echo, am_eric, am_fenrir, am_liam, am_michael, am_onyx, am_puck, am_santa
-#   b (British):  bf_alice, bf_emma, bf_isabella, bf_lily, bm_daniel, bm_fable, bm_george, bm_lewis
-#   e (Spanish):  ef_dora, em_alex, em_santa
-#   f (French):   ff_siwis
-#   h (Hindi):    hf_alpha, hf_beta, hm_omega, hm_psi
-#   i (Italian):  if_sara, im_nicola
-#   j (Japanese): jf_alpha, jf_gongitsune, jf_nezumi, jf_tebukuro, jm_kumo
-#   p (Portuguese): pf_dora, pm_alex, pm_santa
-#   z (Chinese):  zf_xiaobei, zf_xiaoni, zf_xiaoxiao, zf_xiaoyi, zm_yunjian, zm_yunxi, zm_yunxia, zm_yunyang
-KOKORO_LANG_DEFAULT_VOICES = {
-    "a": "af_heart",  # American English female
-    "b": "bm_fable",  # British English male
-    "e": "ef_dora",  # Spanish female
-    "f": "ff_siwis",  # French female
-    "h": "hf_alpha",  # Hindi female
-    "i": "if_sara",  # Italian female
-    "j": "jf_alpha",  # Japanese female
-    "p": "pf_dora",  # Portuguese female
-    "z": "zf_xiaobei",  # Chinese female
-}
-
-
-class KokoroTTSHandler(BaseHandler[TTSIn, TTSOut]):
-    """
-    Handles Text-to-Speech using Kokoro TTS model.
-
-    On Apple Silicon (MPS): Uses mlx-audio with the MLX-converted model.
-    On CUDA/CPU: Uses native kokoro library for optimal performance.
-
-    Kokoro 82M is a 82M parameter multilingual TTS model
-    supporting 8 languages with multiple voices per language.
-    """
-
-    def setup(
-        self,
-        should_listen: Event,
-        model_name: Optional[str] = None,
-        device: str = "auto",
-        voice: str = "bm_fable",
-        lang_code: str = "b",
-        speed: float = 1.0,
-        blocksize: int = 512,
-        gen_kwargs: dict[str, Any] | None = None,
-        cancel_scope: CancelScope | None = None,
-        speculative_turns: SpeculativeTurnTracker | None = None,
-    ) -> None:
-        """
-        Initialize the Kokoro TTS model.
-
-        Args:
-            model_name: Model identifier. Defaults are:
-                - MPS: "mlx-community/Kokoro-82M-bf16"
-                - CUDA/CPU: "hexgrad/Kokoro-82M"
-            device: Device to use ("auto", "cuda", "mps", "cpu")
-            voice: Voice identifier (e.g. "bm_fable")
-            lang_code: Language code (e.g. "b" for British English)
-            speed: Speech speed multiplier
-            blocksize: Audio chunk size for streaming
-            gen_kwargs: Unused, for pipeline compatibility
-        """
-        self.should_listen = should_listen
-        self.voice = voice
-        self.lang_code = lang_code
-        self.speed = speed
-        self.blocksize = blocksize
-        self.cancel_scope = cancel_scope
-        self.speculative_turns = speculative_turns
-
-        # Determine device
-        if device == "auto":
-            if platform == "darwin":
-                self.device = "mps"
-            else:
-                import torch
-
-                self.device = "cuda" if torch.cuda.is_available() else "cpu"
-        else:
-            self.device = device
-
-        # Set default model based on device
-        if model_name is None:
-            if self.device == "mps":
-                model_name = "mlx-community/Kokoro-82M-bf16"
-            else:
-                model_name = "hexgrad/Kokoro-82M"
-
-        self.model_name = model_name
-
-        logger.info(f"Loading Kokoro model: {model_name} on {self.device}")
-
-        if self.device == "mps":
-            self._setup_mlx(model_name)
-        else:
-            self._setup_kokoro(model_name)
-
-        self._initial_voice = self.voice
-        self._initial_lang_code = self.lang_code
-
-        self.warmup()
-
-    def _setup_mlx(self, model_name: str) -> None:
-        """Setup for Apple Silicon using mlx-audio."""
-        try:
-            from mlx_audio.tts.utils import load_model
-
-            self.backend = "mlx"
-            self.model = load_model(model_name)
-            logger.info("MLX Audio Kokoro model loaded successfully")
-
-            # Get or create the pipeline for our language and preload the voice
-            # This avoids the voice being reloaded on every generate() call
-            self._pipeline = self.model._get_pipeline(self.lang_code)
-            self._voice_tensor = self._pipeline.load_voice(self.voice)
-            logger.info(f"Preloaded voice: {self.voice}")
-
-            # Preload voices for common languages to avoid download delays during inference
-            self._preload_multilingual_voices()
-        except ImportError as e:
-            message = str(e)
-            if "misaki" in message or "espeakng_loader" in message:
-                raise ImportError(
-                    "Kokoro TTS on Apple Silicon requires additional mlx-audio TTS dependencies. "
-                    f"Missing dependency: {message}. "
-                    "Install with: pip install misaki espeakng-loader"
-                ) from e
-            raise ImportError(
-                "mlx-audio is required for Kokoro TTS on Apple Silicon. Install with: pip install mlx-audio"
-            ) from e
-
-    def _setup_kokoro(self, model_name: str) -> None:
-        """Setup for CUDA/CPU using native kokoro library."""
-        try:
-            from kokoro import KPipeline
-
-            self.backend = "kokoro"
-            self.pipeline = KPipeline(lang_code=self.lang_code)
-            logger.info("Native Kokoro pipeline loaded successfully")
-        except ImportError as e:
-            raise ImportError(
-                "kokoro is required for Kokoro TTS on CUDA/CPU. "
-                "Install with: pip install kokoro>=0.9.2 soundfile\n"
-                "Also ensure espeak-ng is installed: apt-get install espeak-ng (Linux) or brew install espeak-ng (macOS)"
-            ) from e
-
-    def _preload_multilingual_voices(self) -> None:
-        """Preload voices for common languages to avoid download delays during inference."""
-        # Only preload a few commonly used language voices to avoid excessive startup time
-        # Users speaking other languages will experience a one-time download delay
-        # preload_langs = ["a", "b", "e", "f", "h", "i", "j", "p", "z"] # English, Spanish, French, Hindi, Italian, Japanese, Portuguese
-        preload_langs = ["a", "e", "f"]  # English, Spanish, French
-
-        for lang_code in preload_langs:
-            voice = KOKORO_LANG_DEFAULT_VOICES.get(lang_code)
-            if voice:
-                try:
-                    logger.info(f"Preloading voice for language '{lang_code}': {voice}")
-                    pipeline = self.model._get_pipeline(lang_code)
-                    pipeline.load_voice(voice)
-                except Exception as e:
-                    logger.warning(f"Failed to preload voice {voice}: {e}")
-
-    def warmup(self) -> None:
-        """Warm up the model with a dummy inference."""
-        logger.info(f"Warming up {self.__class__.__name__}")
-
-        # Run a short dummy inference to warm up the model
-        if self.backend == "mlx":
-            for _ in self.model.generate(
-                text="Hello",
-                voice=self.voice,
-                speed=self.speed,
-                lang_code=self.lang_code,
-            ):
-                pass
-        else:
-            for _ in self.pipeline("Hello", voice=self.voice, speed=self.speed):
-                pass
-
-        logger.info(f"{self.__class__.__name__} warmed up")
-
-    def process(self, tts_input: TTSIn) -> Iterator[TTSOut]:
-        """
-        Process text input and generate audio output.
-
-        Yields:
-            Audio chunks as numpy int16 arrays
-        """
-        speculative_turns = getattr(self, "speculative_turns", None)
-        if isinstance(tts_input, EndOfResponse):
-            if speculative_turns and not speculative_turns.is_latest_after_reopen_grace(
-                tts_input.turn_id,
-                tts_input.turn_revision,
-            ):
-                return
-            yield AUDIO_RESPONSE_DONE
-            return
-
-        if speculative_turns and not speculative_turns.is_latest_after_reopen_grace(
-            tts_input.turn_id,
-            tts_input.turn_revision,
-        ):
-            logger.debug("Dropping stale TTS input for turn=%s rev=%s", tts_input.turn_id, tts_input.turn_revision)
-            return
-        if speculative_turns:
-            speculative_turns.commit(tts_input.turn_id, tts_input.turn_revision)
-
-        runtime_config = tts_input.runtime_config
-        response = tts_input.response
-        language_code = tts_input.language_code
-        text = tts_input.text
-
-        voice: Optional[str] = None
-        if response and response.audio and response.audio.output and response.audio.output.voice:
-            voice = str(response.audio.output.voice)
-        if not voice and runtime_config:
-            audio_cfg = runtime_config.session.audio
-            audio_output = audio_cfg.output if audio_cfg is not None else None
-            voice = str(audio_output.voice) if audio_output is not None and audio_output.voice else None
-        if voice:
-            self.voice = voice
-
-        if self.backend == "mlx":
-            yield from self._process_mlx(text, language_code)
-        else:
-            yield from self._process_kokoro(text, language_code)
-
-    def _process_mlx(self, llm_sentence: str, language_code: Optional[str] = None) -> Iterator[np.ndarray]:
-        """Process using MLX backend with Apple Silicon optimizations."""
-        from scipy.signal import resample_poly
-
-        gen = self.cancel_scope.generation if self.cancel_scope else None
-        with MLXLockContext(handler_name="KokoroTTS", timeout=10.0):
-            if language_code is not None:
-                new_lang_code = WHISPER_LANGUAGE_TO_KOKORO_LANG.get(language_code, self.lang_code)
-                if new_lang_code != self.lang_code:
-                    new_voice = KOKORO_LANG_DEFAULT_VOICES.get(new_lang_code, self.voice)
-                    logger.info(
-                        f"Language change detected: {self.lang_code} -> {new_lang_code}, voice: {self.voice} -> {new_voice}"
-                    )
-                    try:
-                        new_pipeline = self.model._get_pipeline(new_lang_code)
-                        new_voice_tensor = new_pipeline.load_voice(new_voice)
-                        self.lang_code = new_lang_code
-                        self.voice = new_voice
-                        self._pipeline = new_pipeline
-                        self._voice_tensor = new_voice_tensor
-                    except Exception as e:
-                        logger.warning(
-                            f"Failed to switch language/voice: {e}. Keeping current language: {self.lang_code}"
-                        )
-
-            console.print(f"[green]ASSISTANT: {llm_sentence}")
-
-            # Generate audio using the preloaded pipeline directly
-            # This avoids the voice reload that happens in model.generate()
-            for result in self._pipeline(
-                text=llm_sentence,
-                voice=self.voice,
-                speed=self.speed,
-            ):
-                if result.audio is None:
-                    continue
-                # result.audio is an mx.array with shape (1, samples), convert to numpy and squeeze
-                audio = np.array(result.audio, dtype=np.float32).squeeze(0)
-
-                # Trim silence from start and end of audio
-                # Kokoro generates ~250ms of silence at the start and variable silence at the end
-                threshold = 0.01
-                abs_audio = np.abs(audio)
-                # Find first and last samples above threshold
-                above_threshold = abs_audio > threshold
-                if np.any(above_threshold):
-                    start_idx = int(np.argmax(above_threshold))
-                    end_idx = len(audio) - int(np.argmax(above_threshold[::-1]))
-                    # Add small padding (5ms = 120 samples at 24kHz) to avoid cutting speech
-                    padding = 120
-                    start_idx = max(0, start_idx - padding)
-                    end_idx = min(len(audio), end_idx + padding)
-                    audio = audio[start_idx:end_idx]
-
-                # Kokoro outputs at 24kHz, resample to 16kHz for the pipeline
-                # Using scipy's polyphase resampling (fast and high quality)
-                # 16000/24000 = 2/3, so up=2, down=3
-                audio = resample_poly(audio, up=2, down=3)
-
-                # Convert to int16 format
-                audio = (audio * 32768).astype(np.int16)
-
-                # Yield audio in fixed-size chunks
-                for i in range(0, len(audio), self.blocksize):
-                    if gen is not None and self.cancel_scope is not None and self.cancel_scope.is_stale(gen):
-                        logger.info("TTS generation cancelled (interruption)")
-                        return
-                    chunk = audio[i : i + self.blocksize]
-                    # Pad the last chunk if necessary
-                    if len(chunk) < self.blocksize:
-                        chunk = np.pad(chunk, (0, self.blocksize - len(chunk)))
-                    logger.debug(f"TTS yielding audio chunk: {len(chunk)} samples")
-                    yield chunk
-
-    def _process_kokoro(self, llm_sentence: str, language_code: Optional[str] = None) -> Iterator[np.ndarray]:
-        """Process using native kokoro library."""
-        from scipy.signal import resample_poly
-
-        gen = self.cancel_scope.generation if self.cancel_scope else None
-        if language_code is not None:
-            new_lang_code = WHISPER_LANGUAGE_TO_KOKORO_LANG.get(language_code, self.lang_code)
-            if new_lang_code != self.lang_code:
-                new_voice = KOKORO_LANG_DEFAULT_VOICES.get(new_lang_code, self.voice)
-                logger.info(
-                    f"Language change detected: {self.lang_code} -> {new_lang_code}, voice: {self.voice} -> {new_voice}"
-                )
-                self.lang_code = new_lang_code
-                self.voice = new_voice
-                from kokoro import KPipeline
-
-                self.pipeline = KPipeline(lang_code=self.lang_code)
-
-        console.print(f"[green]ASSISTANT: {llm_sentence}")
-
-        # Generate audio using Kokoro
-        # The pipeline yields tuples of (graphemes, phonemes, audio)
-        for gs, ps, audio in self.pipeline(llm_sentence, voice=self.voice, speed=self.speed):
-            if audio is None:
-                continue
-
-            # Ensure audio is numpy array
-            if not isinstance(audio, np.ndarray):
-                audio = np.array(audio, dtype=np.float32)
-            else:
-                audio = audio.astype(np.float32)
-
-            # Kokoro outputs at 24kHz, resample to 16kHz for the pipeline
-            # Using scipy's polyphase resampling (fast and high quality)
-            # 16000/24000 = 2/3, so up=2, down=3
-            audio = resample_poly(audio, up=2, down=3)
-
-            # Convert to int16 format
-            audio = (audio * 32768).astype(np.int16)
-
-            # Yield audio in fixed-size chunks
-            for i in range(0, len(audio), self.blocksize):
-                if gen is not None and self.cancel_scope is not None and self.cancel_scope.is_stale(gen):
-                    logger.info("TTS generation cancelled (interruption)")
-                    return
-                chunk = audio[i : i + self.blocksize]
-                # Pad the last chunk if necessary
-                if len(chunk) < self.blocksize:
-                    chunk = np.pad(chunk, (0, self.blocksize - len(chunk)))
-                yield chunk
-
-    def on_session_end(self) -> None:
-        self.voice = self._initial_voice
-        self.lang_code = self._initial_lang_code
-        if self.backend == "mlx":
-            try:
-                self._pipeline = self.model._get_pipeline(self.lang_code)
-                self._voice_tensor = self._pipeline.load_voice(self.voice)
-            except Exception as e:
-                logger.warning(f"Failed to restore initial voice/language on session end: {e}")
-        else:
-            from kokoro import KPipeline
-
-            self.pipeline = KPipeline(lang_code=self.lang_code)
-        logger.debug("Kokoro TTS session state reset")

runtime/src/speech_to_speech/TTS/pocket_tts_handler.py

@@ -1,227 +0,0 @@
-from __future__ import annotations
-
-import logging
-from threading import Event
-from time import perf_counter
-from typing import Any, Iterator
-
-import numpy as np
-from rich.console import Console
-
-from speech_to_speech.baseHandler import BaseHandler
-from speech_to_speech.pipeline.cancel_scope import CancelScope
-from speech_to_speech.pipeline.handler_types import TTSIn, TTSOut
-from speech_to_speech.pipeline.messages import AUDIO_RESPONSE_DONE, EndOfResponse
-from speech_to_speech.pipeline.speculative_turns import SpeculativeTurnTracker
-
-logger = logging.getLogger(__name__)
-console = Console()
-
-
-class PocketTTSHandler(BaseHandler[TTSIn, TTSOut]):
-    """
-    Handler for Pocket TTS model from Kyutai Labs.
-    Supports streaming audio generation with voice cloning.
-    """
-
-    def setup(
-        self,
-        should_listen: Event,
-        device: str = "cpu",
-        voice: str = "alba",  # Default voice from catalog
-        sample_rate: int = 16000,  # Match the pipeline's audio output (LocalAudioStreamer uses 16kHz)
-        blocksize: int = 512,
-        max_tokens: int = 50,
-        gen_kwargs: dict[str, Any] | None = None,  # For compatibility with pipeline, not used
-        cancel_scope: CancelScope | None = None,
-        speculative_turns: SpeculativeTurnTracker | None = None,
-    ) -> None:
-        """
-        Initialize Pocket TTS handler.
-
-        Args:
-            should_listen: Event to control when to start listening again
-            device: Device to run model on ('cpu', 'cuda', 'mps')
-            voice: Voice to use. Can be:
-                - A preset name: 'alba', 'marius', 'javert', 'jean', 'fantine', 'cosette', 'eponine', 'azelma'
-                - A local audio file path
-                - A Hugging Face path like "hf://kyutai/tts-voices/..."
-            sample_rate: Output sample rate (pocket-tts generates at 24kHz and will be resampled to this rate). Default 16kHz matches the pipeline's audio output.
-            blocksize: Size of audio blocks to yield
-            max_tokens: Maximum tokens to generate
-        """
-        self.should_listen = should_listen
-        self.cancel_scope = cancel_scope
-        self.speculative_turns = speculative_turns
-        self.device = device
-        self.voice = voice
-        self.sample_rate = sample_rate
-        self.blocksize = blocksize
-        self.max_tokens = max_tokens
-
-        # Suppress verbose logging from pocket_tts library
-        logging.getLogger("pocket_tts").setLevel(logging.WARNING)
-        logging.getLogger("pocket_tts.models.tts_model").setLevel(logging.WARNING)
-        logging.getLogger("pocket_tts.utils.utils").setLevel(logging.WARNING)
-
-        # Import and load model
-        from pocket_tts import TTSModel
-
-        logger.info("Loading Pocket TTS model")
-        self.model = TTSModel.load_model()
-
-        # Move model to specified device
-        if device == "cuda":
-            self.model = self.model.cuda()
-        elif device == "mps":
-            self.model = self.model.to("mps")
-        elif device != "cpu":
-            self.model = self.model.to(device)
-
-        logger.info(f"Pocket TTS model moved to {device}")
-
-        # Load voice state
-        logger.info(f"Loading voice: {voice}")
-        self.voice_state = self.model.get_state_for_audio_prompt(voice)
-
-        logger.info(f"Pocket TTS model sample rate: {self.model.sample_rate}")
-
-    @property
-    def min_time_to_debug(self) -> float:
-        """
-        Override to suppress logging for individual audio chunks.
-        Pocket TTS yields many small chunks (~10-20ms each), which would flood logs.
-        Only log if a chunk takes unusually long (>100ms).
-        """
-        return 0.1  # 100ms threshold
-
-    def process(self, tts_input: TTSIn) -> Iterator[TTSOut]:
-        speculative_turns = getattr(self, "speculative_turns", None)
-        if isinstance(tts_input, EndOfResponse):
-            if speculative_turns and not speculative_turns.is_latest_after_reopen_grace(
-                tts_input.turn_id,
-                tts_input.turn_revision,
-            ):
-                return
-            yield AUDIO_RESPONSE_DONE
-            return
-
-        if speculative_turns and not speculative_turns.is_latest_after_reopen_grace(
-            tts_input.turn_id,
-            tts_input.turn_revision,
-        ):
-            logger.debug("Dropping stale TTS input for turn=%s rev=%s", tts_input.turn_id, tts_input.turn_revision)
-            return
-        if speculative_turns:
-            speculative_turns.commit(tts_input.turn_id, tts_input.turn_revision)
-
-        gen = self.cancel_scope.generation if self.cancel_scope else None
-        language_code = tts_input.language_code
-        text = tts_input.text
-        logger.debug(f"Received language code: {language_code}")
-
-        console.print(f"[green]ASSISTANT: {text}")
-
-        # Generate audio stream
-        logger.debug(f"Generating audio for: {text[:50]}...")
-
-        pipeline_start = perf_counter()
-        first_chunk = True
-
-        # Calculate target chunk size in original sample rate
-        # We need enough samples so that after resampling we get blocksize samples
-        needs_resampling = self.model.sample_rate != self.sample_rate
-        if needs_resampling:
-            from scipy.signal import resample_poly
-
-            # Target chunk size in original sample rate (before resampling)
-            resample_ratio = self.model.sample_rate / self.sample_rate
-            target_chunk_size = int(self.blocksize * resample_ratio)
-            # Calculate up/down factors for polyphase resampling
-            # For 24kHz -> 16kHz: 16000/24000 = 2/3
-            from math import gcd
-
-            g = gcd(self.sample_rate, self.model.sample_rate)
-            self._resample_up = self.sample_rate // g
-            self._resample_down = self.model.sample_rate // g
-        else:
-            target_chunk_size = self.blocksize
-
-        # Buffer to accumulate audio until we have enough for one block
-        audio_buffer = []
-        buffer_size = 0
-
-        for audio_chunk in self.model.generate_audio_stream(
-            self.voice_state,
-            text,
-            max_tokens=self.max_tokens,
-            copy_state=True,  # Don't modify the original voice state
-        ):
-            if gen is not None and self.cancel_scope is not None and self.cancel_scope.is_stale(gen):
-                logger.info("TTS generation cancelled (interruption)")
-                return
-            if first_chunk:
-                logger.debug(f"Time to first audio: {perf_counter() - pipeline_start:.3f}s")
-                first_chunk = False
-
-            # Convert from torch tensor to numpy and add to buffer
-            audio_np = audio_chunk.cpu().numpy()
-            audio_buffer.append(audio_np)
-            buffer_size += len(audio_np)
-
-            # Process buffer when we have enough samples for at least one block
-            while buffer_size >= target_chunk_size:
-                # Concatenate buffer
-                concatenated = np.concatenate(audio_buffer)
-
-                # Extract enough samples for one block
-                chunk_to_process = concatenated[:target_chunk_size]
-                remainder = concatenated[target_chunk_size:]
-
-                # Resample this chunk if needed
-                if needs_resampling:
-                    chunk_resampled = resample_poly(
-                        chunk_to_process,
-                        up=self._resample_up,
-                        down=self._resample_down,
-                    )
-                else:
-                    chunk_resampled = chunk_to_process
-
-                # Convert to int16 format expected by audio output
-                audio_int16 = (chunk_resampled * 32768).astype(np.int16)
-
-                # Ensure exact blocksize (pad or trim if resampling caused slight size differences)
-                if len(audio_int16) < self.blocksize:
-                    audio_int16 = np.pad(audio_int16, (0, self.blocksize - len(audio_int16)))
-                elif len(audio_int16) > self.blocksize:
-                    audio_int16 = audio_int16[: self.blocksize]
-
-                yield audio_int16
-
-                # Update buffer with remainder
-                audio_buffer = [remainder] if len(remainder) > 0 else []
-                buffer_size = len(remainder)
-
-        # Process any remaining audio in buffer
-        if audio_buffer and buffer_size > 0:
-            concatenated = np.concatenate(audio_buffer)
-
-            # Resample the remainder
-            if needs_resampling:
-                concatenated = resample_poly(
-                    concatenated,
-                    up=self._resample_up,
-                    down=self._resample_down,
-                )
-
-            # Convert to int16
-            audio_int16 = (concatenated * 32768).astype(np.int16)
-
-            # Yield in blocks
-            for i in range(0, len(audio_int16), self.blocksize):
-                chunk = audio_int16[i : i + self.blocksize]
-                # Pad last chunk if needed
-                if len(chunk) < self.blocksize:
-                    chunk = np.pad(chunk, (0, self.blocksize - len(chunk)))
-                yield chunk

runtime/src/speech_to_speech/TTS/qwen3_tts_handler.py

@@ -1,903 +0,0 @@
-"""
-Qwen3 TTS Handler
-
-- On Apple Silicon: Uses mlx-audio with MLX-converted Qwen3-TTS models.
-- On CUDA/CPU: Uses faster-qwen3-tts for low-latency streaming.
-"""
-
-from __future__ import annotations
-
-import logging
-import math
-import re
-import tempfile
-import unicodedata
-from collections.abc import Callable
-from pathlib import Path
-from sys import platform
-from threading import Event
-from time import perf_counter
-from typing import Any, Iterator, Optional
-
-import numpy as np
-import torch
-from openai.types.realtime.realtime_response_create_params import RealtimeResponseCreateParams
-from rich.console import Console
-
-from speech_to_speech.api.openai_realtime.runtime_config import RuntimeConfig
-from speech_to_speech.baseHandler import BaseHandler
-from speech_to_speech.pipeline.cancel_scope import CancelScope
-from speech_to_speech.pipeline.control import SESSION_END, is_control_message
-from speech_to_speech.pipeline.handler_types import TTSIn, TTSOut
-from speech_to_speech.pipeline.messages import AUDIO_RESPONSE_DONE, PIPELINE_END, EndOfResponse, TTSInput
-from speech_to_speech.pipeline.speculative_turns import SpeculativeTurnTracker
-from speech_to_speech.utils.mlx_lock import MLXLockContext
-
-logger = logging.getLogger(__name__)
-console = Console()
-
-DEFAULT_MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
-DEFAULT_MLX_MODEL = "mlx-community/Qwen3-TTS-12Hz-1.7B-CustomVoice-6bit"
-DEFAULT_REF_TEXT = "I'm confused why some people have super short timelines, yet at the same time are bullish on scaling up reinforcement learning atop LLMs. If we're actually close to a human-like learner, then this whole approach of training on verifiable outcomes."
-DEFAULT_FASTER_STREAMING_CHUNK_SIZE = 8
-DEFAULT_MLX_STREAMING_CHUNK_SIZE = 4
-DEFAULT_QWEN3_TTS_MAX_NEW_TOKENS = 1536
-MIN_QWEN3_TTS_UTTERANCE_TOKENS = 360
-VALID_MLX_QUANTIZATION_SUFFIXES = ("bf16", "4bit", "6bit", "8bit")
-VALID_FASTER_BACKENDS = ("ggml", "torch")
-MLX_STREAMING_TOKENS_PER_SECOND = 12.5
-PIPELINE_SR = 16000
-ESTIMATED_QWEN3_WORDS_PER_SECOND = 2.6
-ESTIMATED_QWEN3_CHARS_PER_SECOND = 14.0
-QWEN3_TOKEN_SAFETY_MARGIN = 1.35
-QWEN3_BASE_PROMPT_SECONDS = 1.0
-QWEN3_PUNCTUATION_PAUSE_SECONDS = 0.5
-QWEN3_LANGUAGE_ALIASES = {
-    "zh": "chinese",
-    "zh-cn": "chinese",
-    "zh-hans": "chinese",
-    "zh-hant": "chinese",
-    "cmn": "chinese",
-    "en": "english",
-    "en-us": "english",
-    "en-gb": "english",
-    "ja": "japanese",
-    "jp": "japanese",
-    "ko": "korean",
-    "kr": "korean",
-    "de": "german",
-    "fr": "french",
-    "ru": "russian",
-    "pt": "portuguese",
-    "pt-br": "portuguese",
-    "pt-pt": "portuguese",
-    "es": "spanish",
-    "it": "italian",
-}
-
-
-class Qwen3TTSHandler(BaseHandler[TTSIn, TTSOut]):
-    """
-    Handles Text-to-Speech using Qwen3-TTS.
-
-    Backend selection:
-      - Apple Silicon (Darwin): mlx-audio
-      - Other platforms: faster-qwen3-tts
-
-    Supports three generation modes depending on the loaded model:
-      - Voice cloning (ref_audio + ref_text)
-      - Custom voice (preset speakers)
-      - Voice design (instruct prompt)
-    """
-
-    def setup(
-        self,
-        should_listen: Event,
-        model_name: str = DEFAULT_MODEL,
-        device: str = "cuda",
-        dtype: str | torch.dtype = "auto",
-        attn_implementation: str = "eager",
-        backend: str = "ggml",
-        ref_audio: str | Path | None = None,
-        ref_text: str = DEFAULT_REF_TEXT,
-        language: str = "auto",
-        speaker: Optional[str] = "Aiden",
-        instruct: Optional[str] = None,
-        xvec_only: bool = False,
-        parity_mode: bool = False,
-        non_streaming_mode: bool | None = True,
-        mlx_quantization: Optional[str] = None,
-        streaming_chunk_size: int | None = None,
-        max_new_tokens: int = DEFAULT_QWEN3_TTS_MAX_NEW_TOKENS,
-        blocksize: int = 512,
-        gen_kwargs: dict[str, Any] | None = None,
-        cancel_scope: CancelScope | None = None,
-        speculative_turns: SpeculativeTurnTracker | None = None,
-    ) -> None:
-        self.cancel_scope = cancel_scope
-        self.speculative_turns = speculative_turns
-        self.should_listen = should_listen
-        self.requested_device = device
-        self.ref_audio = ref_audio
-        self.ref_text = ref_text
-        self.language = self._normalize_language(language)
-        self.speaker = speaker
-        self.instruct = instruct
-        self.xvec_only = xvec_only
-        self.parity_mode = parity_mode
-        self.non_streaming_mode = non_streaming_mode
-        self.faster_backend = self._normalize_faster_backend(backend)
-        self.mlx_quantization = self._normalize_mlx_quantization(mlx_quantization)
-        self.max_new_tokens = max_new_tokens
-        self.blocksize = blocksize
-        self.dtype: torch.dtype | None | str = None
-        self.gen_kwargs = gen_kwargs or {}
-        self._mlx_ref_audio_cache: dict[str, Any] = {}
-        self._mlx_temp_ref_audio_files: set[str] = set()
-
-        self.backend = "mlx" if platform == "darwin" else "faster_qwen3_tts"
-        self.streaming_chunk_size = self._resolve_streaming_chunk_size(streaming_chunk_size)
-
-        if self.backend == "mlx":
-            self.device = "mps"
-            self.model_name = self._resolve_mlx_model_name(model_name)
-            logger.info(f"Loading Qwen3-TTS model: {self.model_name} via mlx-audio on Apple Silicon")
-            if self.non_streaming_mode is not None:
-                logger.debug(
-                    "qwen3_tts_non_streaming_mode=%s is ignored on Apple Silicon because "
-                    "mlx-audio does not expose non_streaming_mode yet.",
-                    self.non_streaming_mode,
-                )
-            model_quantization = self._model_name_quantization_suffix(self.model_name)
-            if model_quantization and model_quantization != "bf16":
-                logger.info(
-                    "Using MLX quantized Qwen3-TTS variant: %s",
-                    model_quantization,
-                )
-            self._setup_mlx(self.model_name)
-        else:
-            self.device = device
-            self.model_name = model_name
-            logger.info(
-                "Loading Qwen3-TTS model: %s via faster-qwen3-tts (%s backend)",
-                self.model_name,
-                self.faster_backend,
-            )
-            self._setup_faster(
-                model_name=self.model_name,
-                dtype=dtype,
-                attn_implementation=attn_implementation,
-                backend=self.faster_backend,
-            )
-
-        logger.info(
-            "Using Qwen3-TTS streaming chunk size %d (~%.0fms audio per chunk) on %s",
-            self.streaming_chunk_size,
-            self.streaming_chunk_size / MLX_STREAMING_TOKENS_PER_SECOND * 1000,
-            self.backend,
-        )
-
-        self._initial_speaker = self.speaker
-        self._initial_ref_audio = self.ref_audio
-
-        self.warmup()
-
-    def _setup_faster(self, model_name: str, dtype: Any, attn_implementation: str, backend: str) -> None:
-        try:
-            import torch
-        except ImportError as e:
-            raise ImportError("torch is required. Install with: pip install torch") from e
-
-        if dtype == "auto":
-            self.dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
-        elif isinstance(dtype, str):
-            self.dtype = getattr(torch, dtype)
-        else:
-            self.dtype = dtype
-
-        try:
-            from faster_qwen3_tts import FasterQwen3TTS
-        except ImportError as e:
-            raise ImportError(
-                "faster-qwen3-tts is required for Qwen3 TTS on non-macOS platforms. "
-                "Install with: pip install 'faster-qwen3-tts[ggml]'"
-            ) from e
-
-        self.model = FasterQwen3TTS.from_pretrained(
-            model_name,
-            device=self.device,
-            dtype=self.dtype,
-            attn_implementation=attn_implementation,
-            backend=backend,
-        )
-        logger.info("Qwen3-TTS model loaded")
-
-    def _setup_mlx(self, model_name: str) -> None:
-        try:
-            from mlx_audio.tts.utils import load_model
-
-            self.model = load_model(model_name)
-        except ImportError as e:
-            message = str(e)
-            if any(
-                dep in message
-                for dep in (
-                    "misaki",
-                    "spacy",
-                    "phonemizer",
-                    "espeakng_loader",
-                )
-            ):
-                raise ImportError(
-                    "Qwen3-TTS on Apple Silicon requires mlx-audio and its TTS dependencies. "
-                    f"Missing dependency: {message}. "
-                    "Install with: pip install mlx-audio misaki spacy phonemizer-fork espeakng-loader"
-                ) from e
-            raise ImportError(
-                "mlx-audio is required for Qwen3 TTS on Apple Silicon. Install with: pip install mlx-audio"
-            ) from e
-
-        logger.info("MLX Audio Qwen3-TTS model loaded")
-
-    def _normalize_faster_backend(self, backend: Any) -> str:
-        value = str(backend or "ggml").strip().lower()
-        if value not in VALID_FASTER_BACKENDS:
-            raise ValueError(
-                f"Unsupported qwen3_tts_backend value {backend!r}. Supported values: {', '.join(VALID_FASTER_BACKENDS)}"
-            )
-        return value
-
-    def _normalize_mlx_quantization(self, mlx_quantization: Any) -> Optional[str]:
-        if mlx_quantization is None:
-            return None
-
-        value = str(mlx_quantization).strip().lower()
-        if value in ("", "none", "default"):
-            return None
-        if value not in VALID_MLX_QUANTIZATION_SUFFIXES:
-            raise ValueError(
-                "Unsupported qwen3_tts_mlx_quantization value "
-                f"{mlx_quantization!r}. Supported values: {', '.join(VALID_MLX_QUANTIZATION_SUFFIXES)}"
-            )
-        return value
-
-    def _apply_mlx_quantization_suffix(self, model_name: str) -> str:
-        if self.mlx_quantization is None:
-            return model_name
-
-        desired_suffix = f"-{self.mlx_quantization}"
-        for suffix in VALID_MLX_QUANTIZATION_SUFFIXES:
-            current_suffix = f"-{suffix}"
-            if model_name.endswith(current_suffix):
-                return model_name[: -len(current_suffix)] + desired_suffix
-
-        return f"{model_name}{desired_suffix}"
-
-    def _model_name_quantization_suffix(self, model_name: str) -> Optional[str]:
-        if not model_name:
-            return None
-
-        for suffix in VALID_MLX_QUANTIZATION_SUFFIXES:
-            if model_name.endswith(f"-{suffix}"):
-                return suffix
-
-        return None
-
-    def _resolve_mlx_model_name(self, model_name: str) -> str:
-        if not model_name:
-            return self._apply_mlx_quantization_suffix(DEFAULT_MLX_MODEL)
-        if model_name.startswith("mlx-community/"):
-            if self.mlx_quantization is None and self._model_name_quantization_suffix(model_name) is None:
-                return f"{model_name}-6bit"
-            return self._apply_mlx_quantization_suffix(model_name)
-        if model_name.startswith("Qwen/"):
-            mapped = model_name.replace("Qwen/", "mlx-community/", 1)
-            if self._model_name_quantization_suffix(mapped) is None:
-                if self.mlx_quantization is None:
-                    return f"{mapped}-6bit"
-                mapped = f"{mapped}-bf16"
-            return self._apply_mlx_quantization_suffix(mapped)
-        return model_name
-
-    def _resolve_streaming_chunk_size(self, streaming_chunk_size: int | None) -> int:
-        if streaming_chunk_size is not None:
-            return max(1, int(streaming_chunk_size))
-        if self.backend == "mlx":
-            return DEFAULT_MLX_STREAMING_CHUNK_SIZE
-        return DEFAULT_FASTER_STREAMING_CHUNK_SIZE
-
-    def _normalize_language(self, language: str | None) -> str:
-        if language is None:
-            return "auto"
-        normalized = str(language).strip().replace("_", "-").lower()
-        if not normalized:
-            return "auto"
-        return QWEN3_LANGUAGE_ALIASES.get(normalized, normalized)
-
-    def _infer_model_type_from_name(self) -> str:
-        name = (self.model_name or "").lower()
-        if "voicedesign" in name:
-            return "voice_design"
-        if "customvoice" in name:
-            return "custom_voice"
-        return "base"
-
-    def _resolve_audio_path(self, audio: Any) -> Path | None:
-        if not isinstance(audio, (str, Path)) or not audio:
-            return None
-
-        candidate = Path(audio).expanduser()
-        repo_root = Path(__file__).resolve().parents[1]
-        search_paths = []
-
-        if candidate.is_absolute():
-            search_paths.append(candidate)
-        else:
-            search_paths.append(Path.cwd() / candidate)
-            search_paths.append(repo_root / candidate)
-
-        seen = set()
-        for path in search_paths:
-            normalized = str(path)
-            if normalized in seen:
-                continue
-            seen.add(normalized)
-            if path.exists():
-                return path.resolve()
-
-        return None
-
-    def _prepare_mlx_ref_audio(self, ref_audio: Any) -> Any:
-        if self.backend != "mlx" or ref_audio is None:
-            return ref_audio
-
-        if not isinstance(ref_audio, (str, Path)):
-            return ref_audio
-
-        resolved_path = self._resolve_audio_path(ref_audio)
-        if resolved_path is None:
-            raise FileNotFoundError(
-                "Qwen3-TTS on Apple Silicon requires qwen3_tts_ref_audio to point to "
-                f"a readable audio file. Got: {ref_audio!r}"
-            )
-
-        cache_key = str(resolved_path)
-        cached_path = self._mlx_ref_audio_cache.get(cache_key)
-        if cached_path and Path(cached_path).exists():
-            return cached_path
-
-        try:
-            import soundfile as sf
-            from scipy.signal import resample_poly
-
-            waveform, sample_rate = sf.read(str(resolved_path), always_2d=False, dtype="float32")
-            waveform = np.asarray(waveform, dtype=np.float32)
-            if waveform.ndim > 1:
-                waveform = waveform.mean(axis=1)
-            target_sample_rate = getattr(self.model, "sample_rate", 24000)
-            if sample_rate != target_sample_rate:
-                gcd = np.gcd(int(sample_rate), int(target_sample_rate))
-                waveform = resample_poly(
-                    waveform,
-                    up=int(target_sample_rate) // gcd,
-                    down=int(sample_rate) // gcd,
-                )
-                sample_rate = target_sample_rate
-
-            with tempfile.NamedTemporaryFile(
-                prefix="qwen3_ref_",
-                suffix=".wav",
-                delete=False,
-            ) as temp_file:
-                normalized_path = temp_file.name
-
-            sf.write(
-                normalized_path,
-                waveform,
-                sample_rate,
-                format="WAV",
-                subtype="PCM_16",
-            )
-        except Exception as e:
-            raise RuntimeError(f"Failed to normalize Qwen3-TTS reference audio {resolved_path}: {e}") from e
-
-        self._mlx_ref_audio_cache[cache_key] = normalized_path
-        self._mlx_temp_ref_audio_files.add(normalized_path)
-        return normalized_path
-
-    def _apply_session_voice_override(
-        self,
-        model_type: str,
-        runtime_config: RuntimeConfig | None = None,
-        response: RealtimeResponseCreateParams | None = None,
-    ) -> None:
-        session_voice: Optional[str] = None
-        if response and response.audio and response.audio.output:
-            resp_voice = response.audio.output.voice
-            session_voice = str(resp_voice) if resp_voice else None
-        if not session_voice and runtime_config is not None:
-            audio = runtime_config.session.audio
-            output = audio.output if audio is not None else None
-            sess_voice = output.voice if output is not None else None
-            session_voice = str(sess_voice) if sess_voice else None
-        if not session_voice:
-            return
-
-        if model_type == "custom_voice":
-            supported_speakers = self._supported_speakers()
-            if supported_speakers is not None:
-                speakers_by_lower = {speaker.lower(): speaker for speaker in supported_speakers}
-                speaker = speakers_by_lower.get(session_voice.lower())
-                if speaker is None:
-                    supported_text = ", ".join(sorted(supported_speakers, key=str.lower)) or "unknown"
-                    logger.warning(
-                        "Ignoring Qwen3-TTS session voice override %r because it is not a supported "
-                        "CustomVoice speaker. Supported speakers: %s",
-                        session_voice,
-                        supported_text,
-                    )
-                    return
-                session_voice = speaker
-
-            self.speaker = session_voice
-            self.ref_audio = None
-            return
-
-        if self._resolve_audio_path(session_voice) is not None:
-            self.ref_audio = session_voice
-            return
-
-        logger.warning(
-            "Ignoring Qwen3-TTS session voice override because it is not an audio file path: %r",
-            session_voice,
-        )
-
-    def warmup(self) -> None:
-        logger.info(f"Warming up {self.__class__.__name__}")
-
-        if self.backend == "faster_qwen3_tts":
-            if self.parity_mode:
-                logger.info("Qwen3-TTS parity mode enabled: skipping backend warmup")
-            else:
-                try:
-                    self.model.warmup(prefill_len=100)
-                except Exception as e:
-                    logger.warning("Qwen3-TTS backend warmup failed: %s", e)
-
-        try:
-            for _ in self._warmup_process("Hello, this is a warmup."):
-                pass
-            logger.info(f"{self.__class__.__name__} warmed up")
-        except Exception as e:
-            logger.warning(f"Warmup generation failed: {e}")
-
-    def _model_type(self) -> str:
-        if self.backend == "mlx":
-            config = getattr(self.model, "config", None)
-            return getattr(config, "tts_model_type", None) or self._infer_model_type_from_name()
-
-        inner = getattr(getattr(self.model, "model", None), "model", None)
-        return getattr(inner, "tts_model_type", None) or self._infer_model_type_from_name()
-
-    def _supported_speakers(self) -> list[str] | None:
-        for candidate in (
-            self.model,
-            getattr(self.model, "model", None),
-            getattr(getattr(self.model, "model", None), "model", None),
-        ):
-            get_speakers = getattr(candidate, "get_supported_speakers", None)
-            if callable(get_speakers):
-                speakers = get_speakers()
-                if speakers is None:
-                    return None
-                return [str(speaker) for speaker in speakers if speaker]
-        return None
-
-    def _resolve_speaker(self) -> Optional[str]:
-        if self.speaker:
-            return self.speaker
-
-        for candidate in (
-            self.model,
-            getattr(getattr(self.model, "model", None), "model", None),
-        ):
-            get_speakers = getattr(candidate, "get_supported_speakers", None)
-            if callable(get_speakers):
-                speakers = list(get_speakers() or [])
-                if speakers:
-                    return speakers[0]
-
-        return None
-
-    def _to_int16(self, audio: np.ndarray) -> np.ndarray:
-        return np.clip(audio * 32768, -32768, 32767).astype(np.int16)
-
-    def _estimate_max_new_tokens(self, text: Optional[str]) -> int:
-        text = (text or "").strip()
-        chunk_size = max(1, int(getattr(self, "streaming_chunk_size", 1)))
-        configured_cap = max(1, int(getattr(self, "max_new_tokens", DEFAULT_QWEN3_TTS_MAX_NEW_TOKENS)))
-
-        if not text:
-            return min(configured_cap, MIN_QWEN3_TTS_UTTERANCE_TOKENS)
-
-        word_count = len(re.findall(r"\w+", text, flags=re.UNICODE))
-        char_count = len(re.sub(r"\s+", "", text))
-        word_seconds = word_count / ESTIMATED_QWEN3_WORDS_PER_SECOND if word_count else 0.0
-        char_seconds = char_count / ESTIMATED_QWEN3_CHARS_PER_SECOND if char_count else 0.0
-        punctuation_count = sum(unicodedata.category(ch).startswith("P") for ch in text)
-        punctuation_seconds = punctuation_count * QWEN3_PUNCTUATION_PAUSE_SECONDS
-        estimated_seconds = max(word_seconds, char_seconds) + punctuation_seconds + QWEN3_BASE_PROMPT_SECONDS
-        estimated_tokens = math.ceil(estimated_seconds * MLX_STREAMING_TOKENS_PER_SECOND * QWEN3_TOKEN_SAFETY_MARGIN)
-        aligned_tokens = max(
-            chunk_size,
-            math.ceil(estimated_tokens / chunk_size) * chunk_size,
-        )
-        requested_tokens = max(MIN_QWEN3_TTS_UTTERANCE_TOKENS, aligned_tokens)
-        resolved_tokens = min(configured_cap, requested_tokens)
-
-        if resolved_tokens < requested_tokens:
-            logger.warning(
-                "Qwen3-TTS estimated %d codec tokens for a %d-character utterance, "
-                "but max_new_tokens is capped at %d; output may still truncate.",
-                requested_tokens,
-                len(text),
-                configured_cap,
-            )
-
-        logger.debug(
-            "Qwen3-TTS using max_new_tokens=%d for utterance with %d words and %d chars",
-            resolved_tokens,
-            word_count,
-            char_count,
-        )
-        return resolved_tokens
-
-    def _warmup_process(self, llm_sentence: str) -> Iterator[bytes | np.ndarray]:
-        model_type = self._model_type()
-        if self.ref_audio:
-            yield from self._process_voice_clone(llm_sentence)
-        elif model_type == "custom_voice":
-            yield from self._process_custom_voice(llm_sentence)
-        elif model_type == "voice_design":
-            yield from self._process_voice_design(llm_sentence)
-        else:
-            raise ValueError(
-                "Qwen3-TTS Base model requires ref_audio for voice cloning. "
-                "Provide qwen3_tts_ref_audio or use a CustomVoice/VoiceDesign model."
-            )
-
-    def _resample_to_pipeline_sr(self, audio: np.ndarray, sr: int) -> np.ndarray:
-        if sr == PIPELINE_SR:
-            return audio
-        from scipy.signal import resample_poly
-
-        gcd = np.gcd(PIPELINE_SR, sr)
-        return resample_poly(audio, up=PIPELINE_SR // gcd, down=sr // gcd)
-
-    def _prepare_audio_chunk(self, item: Any) -> tuple[np.ndarray | None, int | None]:
-        if isinstance(item, tuple):
-            audio_chunk, sr, _timing = item
-            return np.asarray(audio_chunk, dtype=np.float32), sr
-
-        audio = getattr(item, "audio", None)
-        if audio is None:
-            return None, None
-
-        audio_chunk = np.asarray(audio, dtype=np.float32).squeeze()
-        sr = getattr(item, "sample_rate", None) or PIPELINE_SR
-        return audio_chunk, sr
-
-    def _stream(self, gen: Any, label: str) -> Iterator[bytes | np.ndarray]:
-        """Common streaming loop: log TTFA and RTF, yield int16 chunks."""
-        cancel_gen = self.cancel_scope.generation if self.cancel_scope else None
-        start = perf_counter()
-        total_samples = 0
-        first_chunk = True
-        found_speech = False
-        leftover = np.array([], dtype=np.int16)
-
-        for item in gen:
-            if cancel_gen is not None and self.cancel_scope is not None and self.cancel_scope.is_stale(cancel_gen):
-                logger.info("TTS generation cancelled (interruption)")
-                return
-
-            audio_chunk, sr = self._prepare_audio_chunk(item)
-            if audio_chunk is None or sr is None or audio_chunk.size == 0:
-                continue
-
-            if first_chunk:
-                logger.info(f"Qwen3-TTS TTFA: {perf_counter() - start:.2f}s ({label})")
-                first_chunk = False
-
-            audio_chunk = self._resample_to_pipeline_sr(audio_chunk, sr)
-            audio_chunk = self._to_int16(audio_chunk)
-
-            # Trim the initial silent ramp-up, but keep enough preroll to avoid
-            # shaving soft initial phonemes at the start of the utterance.
-            if not found_speech:
-                threshold = int(32768 * 0.01)
-                above = np.abs(audio_chunk) > threshold
-                if not np.any(above):
-                    continue
-                start_idx = max(0, int(np.argmax(above)) - int(PIPELINE_SR * 0.040))
-                audio_chunk = audio_chunk[start_idx:]
-                found_speech = True
-
-            audio_chunk = np.concatenate([leftover, audio_chunk])
-
-            n = (len(audio_chunk) // self.blocksize) * self.blocksize
-            for i in range(0, n, self.blocksize):
-                yield audio_chunk[i : i + self.blocksize]
-                total_samples += self.blocksize
-            leftover = audio_chunk[n:]
-
-        if len(leftover) > 0:
-            chunk = np.pad(leftover, (0, self.blocksize - len(leftover)))
-            yield chunk
-            total_samples += len(leftover)
-
-        generation_time = perf_counter() - start
-        audio_duration = total_samples / PIPELINE_SR
-        rtf = audio_duration / generation_time if generation_time > 0 else 0
-        logger.info(
-            f"Qwen3-TTS generated {audio_duration:.2f}s audio in {generation_time:.2f}s (RTF: {rtf:.2f}, {label})"
-        )
-
-    def _coalesce_pending_tts_input(self, current_input: TTSInput) -> tuple[str, Optional[str], bool]:
-        """Combine already-queued text chunks before the next TTS synthesis call."""
-        if not hasattr(self.queue_in, "mutex") or not hasattr(self.queue_in, "queue"):
-            return current_input.text, current_input.language_code, False
-
-        text = current_input.text
-        language_code = current_input.language_code
-
-        parts = [text.strip()] if text and text.strip() else []
-        saw_end_of_response = False
-
-        with self.queue_in.mutex:
-            while self.queue_in.queue:
-                next_item = self.queue_in.queue[0]
-                if is_control_message(next_item, SESSION_END.kind):
-                    break
-                if isinstance(next_item, bytes) and next_item == PIPELINE_END:
-                    break
-                if isinstance(next_item, EndOfResponse):
-                    saw_end_of_response = True
-                    break
-                if not isinstance(next_item, TTSInput):
-                    break
-                if current_input.turn_id != next_item.turn_id or current_input.turn_revision != next_item.turn_revision:
-                    break
-                if (
-                    language_code is not None
-                    and next_item.language_code is not None
-                    and next_item.language_code != language_code
-                ):
-                    break
-
-                self.queue_in.queue.popleft()
-                if next_item.text.strip():
-                    parts.append(next_item.text.strip())
-                if language_code is None:
-                    language_code = next_item.language_code
-
-        combined_text = " ".join(parts).strip()
-        return combined_text, language_code, saw_end_of_response
-
-    def process(self, tts_input: TTSIn) -> Iterator[TTSOut]:
-        speculative_turns = getattr(self, "speculative_turns", None)
-        if isinstance(tts_input, EndOfResponse):
-            if speculative_turns and not speculative_turns.is_latest_after_reopen_grace(
-                tts_input.turn_id,
-                tts_input.turn_revision,
-            ):
-                return
-            yield AUDIO_RESPONSE_DONE
-            return
-
-        if speculative_turns and not speculative_turns.is_latest_after_reopen_grace(
-            tts_input.turn_id,
-            tts_input.turn_revision,
-        ):
-            logger.debug("Dropping stale TTS input for turn=%s rev=%s", tts_input.turn_id, tts_input.turn_revision)
-            return
-        if speculative_turns:
-            speculative_turns.commit(tts_input.turn_id, tts_input.turn_revision)
-
-        runtime_config = tts_input.runtime_config
-        response = tts_input.response
-
-        coalesced_text, _language_code, _saw_end_of_response = self._coalesce_pending_tts_input(tts_input)
-
-        text = coalesced_text or "Hello."
-
-        model_type = self._model_type()
-        self._apply_session_voice_override(model_type, runtime_config, response)
-
-        console.print(f"[green]ASSISTANT: {text}")
-
-        try:
-            if self.ref_audio:
-                audio_iter = self._process_voice_clone(text)
-            elif model_type == "custom_voice":
-                audio_iter = self._process_custom_voice(text)
-            elif model_type == "voice_design":
-                audio_iter = self._process_voice_design(text)
-            else:
-                raise ValueError(
-                    "Qwen3-TTS Base model requires ref_audio for voice cloning. "
-                    "Provide qwen3_tts_ref_audio or use a CustomVoice/VoiceDesign model."
-                )
-            first_audio = True
-            for audio_chunk in audio_iter:
-                if first_audio:
-                    self._log_first_audio_latency(tts_input)
-                    first_audio = False
-                yield audio_chunk
-        except Exception as e:
-            logger.error(f"Error during Qwen3-TTS generation: {e}", exc_info=True)
-
-    def _log_first_audio_latency(self, tts_input: TTSInput) -> None:
-        if tts_input.speech_stopped_at_s is None:
-            return
-        latency_s = perf_counter() - tts_input.speech_stopped_at_s
-        if latency_s < 0:
-            return
-        logger.info(
-            "Last speech detected to first speech out: %.3fs (turn=%s rev=%s)",
-            latency_s,
-            tts_input.turn_id,
-            tts_input.turn_revision,
-        )
-
-    def _mlx_streaming_interval(self) -> float:
-        return max(1, self.streaming_chunk_size) / MLX_STREAMING_TOKENS_PER_SECOND
-
-    def _mlx_stream_kwargs(self, max_tokens: int) -> dict[str, Any]:
-        return {
-            "max_tokens": max_tokens,
-            "verbose": False,
-            "stream": True,
-            "streaming_interval": self._mlx_streaming_interval(),
-            **self.gen_kwargs,
-        }
-
-    def _stream_mlx_generation(
-        self,
-        generation_fn: Callable,
-        label: str,
-        max_tokens: int,
-        **generation_kwargs: Any,
-    ) -> Iterator[bytes | np.ndarray]:
-        with MLXLockContext(handler_name="Qwen3TTS", timeout=10.0) as acquired:
-            if not acquired:
-                raise TimeoutError("Timed out waiting for MLX lock")
-            yield from self._stream(
-                generation_fn(
-                    **self._mlx_stream_kwargs(max_tokens=max_tokens),
-                    **generation_kwargs,
-                ),
-                label=label,
-            )
-
-    def _process_voice_clone(self, text: str) -> Iterator[bytes | np.ndarray]:
-        utterance_max_new_tokens = self._estimate_max_new_tokens(text)
-        if self.backend == "mlx":
-            if self.xvec_only:
-                logger.warning("mlx-audio Qwen3-TTS does not support xvec_only; ignoring it")
-            if self.parity_mode:
-                logger.info("Qwen3-TTS parity mode is CUDA-specific and is ignored on mlx-audio")
-
-            yield from self._stream_mlx_generation(
-                self.model.generate,
-                label="voice_clone_mlx",
-                max_tokens=utterance_max_new_tokens,
-                text=text,
-                ref_audio=self._prepare_mlx_ref_audio(self.ref_audio),
-                ref_text=self.ref_text,
-                lang_code=self.language,
-            )
-            return
-
-        yield from self._stream(
-            self.model.generate_voice_clone_streaming(
-                text=text,
-                language=self.language,
-                ref_audio=self.ref_audio,
-                ref_text=self.ref_text,
-                xvec_only=self.xvec_only,
-                chunk_size=self.streaming_chunk_size,
-                max_new_tokens=utterance_max_new_tokens,
-                parity_mode=self.parity_mode,
-                non_streaming_mode=self.non_streaming_mode,
-            ),
-            label="voice_clone_parity" if self.parity_mode else "voice_clone",
-        )
-
-    def _process_custom_voice(self, text: str) -> Iterator[bytes | np.ndarray]:
-        utterance_max_new_tokens = self._estimate_max_new_tokens(text)
-        speaker = self._resolve_speaker()
-        if not speaker:
-            raise ValueError(
-                "CustomVoice generation requires a speaker. "
-                "Set qwen3_tts_speaker or use a voice-clone model with ref_audio."
-            )
-
-        if self.backend == "mlx":
-            yield from self._stream_mlx_generation(
-                self.model.generate_custom_voice,
-                label="custom_voice_mlx",
-                max_tokens=utterance_max_new_tokens,
-                text=text,
-                speaker=speaker,
-                language=self.language,
-                instruct=self.instruct,
-            )
-            return
-
-        yield from self._stream(
-            self.model.generate_custom_voice_streaming(
-                text=text,
-                speaker=speaker,
-                language=self.language,
-                instruct=self.instruct,
-                chunk_size=self.streaming_chunk_size,
-                max_new_tokens=utterance_max_new_tokens,
-                non_streaming_mode=self.non_streaming_mode,
-            ),
-            label="custom_voice",
-        )
-
-    def _process_voice_design(self, text: str) -> Iterator[bytes | np.ndarray]:
-        utterance_max_new_tokens = self._estimate_max_new_tokens(text)
-        if self.backend == "mlx":
-            yield from self._stream_mlx_generation(
-                self.model.generate_voice_design,
-                label="voice_design_mlx",
-                max_tokens=utterance_max_new_tokens,
-                text=text,
-                instruct=self.instruct,
-                language=self.language,
-            )
-            return
-
-        yield from self._stream(
-            self.model.generate_voice_design_streaming(
-                text=text,
-                instruct=self.instruct,
-                language=self.language,
-                chunk_size=self.streaming_chunk_size,
-                max_new_tokens=utterance_max_new_tokens,
-                non_streaming_mode=self.non_streaming_mode,
-            ),
-            label="voice_design",
-        )
-
-    def on_session_end(self) -> None:
-        self.speaker = self._initial_speaker
-        self.ref_audio = self._initial_ref_audio
-        logger.debug("Qwen3-TTS session state reset")
-
-    def cleanup(self) -> None:
-        try:
-            del self.model
-            for path in list(getattr(self, "_mlx_temp_ref_audio_files", set())):
-                try:
-                    Path(path).unlink(missing_ok=True)
-                except Exception:
-                    pass
-            if self.backend == "mlx":
-                try:
-                    import mlx.core as mx
-
-                    mx.clear_cache()
-                except Exception:
-                    pass
-            else:
-                import torch
-
-                if torch.cuda.is_available():
-                    torch.cuda.empty_cache()
-            logger.info("Qwen3-TTS handler cleaned up")
-        except Exception as e:
-            logger.warning(f"Cleanup error: {e}")

runtime/src/speech_to_speech/TTS/ref_audio.wav

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c8f3be0d4e32aa3ab4bbc859bb5c2ae176d2170853661edc65c94f232ecca232
-size 159092

runtime/src/speech_to_speech/VAD/__init__.py

@@ -1 +0,0 @@
-

runtime/src/speech_to_speech/VAD/vad_handler.py

@@ -1,870 +0,0 @@
-from __future__ import annotations
-
-import logging
-import time
-from collections.abc import Iterator
-from dataclasses import dataclass
-from queue import Queue
-from threading import Event
-from typing import Any, TypeAlias
-
-import numpy as np
-import torch
-
-from speech_to_speech.api.openai_realtime.runtime_config import RuntimeConfig
-from speech_to_speech.baseHandler import BaseHandler
-from speech_to_speech.pipeline.events import SpeechStartedEvent, SpeechStoppedEvent
-from speech_to_speech.pipeline.handler_types import VADIn, VADOut
-from speech_to_speech.pipeline.messages import VADAudio
-from speech_to_speech.pipeline.queue_types import TextEventItem
-from speech_to_speech.pipeline.speculative_turns import SpeculativeTurnTracker
-from speech_to_speech.utils.utils import int2float
-from speech_to_speech.VAD.vad_iterator import VADIterator
-
-logger = logging.getLogger(__name__)
-
-VADInput: TypeAlias = bytes | tuple[bytes, RuntimeConfig]
-
-
-@dataclass
-class _PendingShortSegment:
-    audio: np.ndarray
-    active_ms: float
-    start_ms: int
-    end_ms: int
-
-
-# Fragments with less active speech than this are treated as noise and never
-# held for stitching, so sub-threshold bursts cannot sum past min_speech_ms
-# and fire a false barge-in.
-_SHORT_SEGMENT_MIN_FRAGMENT_MS = 100
-
-
-# Optional import for audio enhancement
-try:
-    from df.enhance import enhance, init_df
-
-    HAS_DF = True
-except (ImportError, ModuleNotFoundError) as e:
-    HAS_DF = False
-    logger.warning(f"DeepFilterNet not available for audio enhancement: {e}")
-
-
-class VADHandler(BaseHandler[VADIn, VADOut]):
-    """
-    Handles voice activity detection. When voice activity is detected, audio will be accumulated until the end of speech is detected and then passed
-    to the following part.
-    """
-
-    def setup(
-        self,
-        should_listen: Event,
-        thresh: float = 0.6,
-        sample_rate: int = 16000,
-        min_silence_ms: int = 64,
-        min_speech_ms: int = 384,
-        min_speech_continuation_ms: int = 192,
-        max_speech_ms: float = float("inf"),
-        speech_pad_ms: int = 30,
-        audio_enhancement: bool = False,
-        enable_realtime_transcription: bool = False,
-        realtime_processing_pause: float = 0.5,
-        text_output_queue: Queue[TextEventItem] | None = None,
-        speculative_turns: SpeculativeTurnTracker | None = None,
-        speculative_reopen_ms: int = 1000,
-        unanswered_reopen_ms: int = 7000,
-        short_segment_merge_ms: int = 0,
-    ) -> None:
-        self.should_listen = should_listen
-        self.sample_rate = sample_rate
-        self.min_silence_ms = min_silence_ms
-        self.min_speech_ms = min_speech_ms
-        self.min_speech_continuation_ms = self._resolve_min_speech_continuation_ms(
-            self.min_speech_ms,
-            min_speech_continuation_ms,
-        )
-        self.max_speech_ms = max_speech_ms
-        self.enable_realtime_transcription = enable_realtime_transcription
-        self.realtime_processing_pause = realtime_processing_pause
-        self.text_output_queue = text_output_queue
-        self.speculative_turns = speculative_turns
-        self.speculative_reopen_ms = speculative_reopen_ms
-        self.unanswered_reopen_ms = max(self.speculative_reopen_ms, unanswered_reopen_ms)
-        self.short_segment_merge_ms = max(0, short_segment_merge_ms)
-        self._last_turn_detection: dict | None = None
-        self.model, _ = torch.hub.load(
-            "snakers4/silero-vad",
-            "silero_vad",
-            trust_repo=True,
-            skip_validation=True,
-        )
-        self.iterator = VADIterator(
-            self.model,
-            threshold=thresh,
-            sampling_rate=sample_rate,
-            min_silence_duration_ms=min_silence_ms,
-            speech_pad_ms=speech_pad_ms,
-        )
-        self.audio_enhancement = audio_enhancement
-        if audio_enhancement:
-            if not HAS_DF:
-                logger.error(
-                    "Audio enhancement requested but DeepFilterNet is not available. Disabling audio enhancement."
-                )
-                self.audio_enhancement = False
-            else:
-                self.enhanced_model, self.df_state, _ = init_df()
-
-        # State for progressive audio release
-        self.last_process_time: float = 0.0
-
-        # Cumulative sample counter for audio_start_ms / audio_end_ms
-        self._total_samples: int = 0
-
-        # Throttled logging state (summary once per second)
-        self._last_log_time = 0.0
-        self._log_chunks = 0
-        self._log_speech_starts = 0
-        self._log_speech_ends = 0
-        self._log_progressive_yields = 0
-        self._speech_started_emitted = False
-        self._turn_counter = 0
-        self._current_turn_id: str | None = None
-        self._current_turn_revision: int | None = None
-        self._speculative_audio_prefix: np.ndarray | None = None
-        self._last_final_wall_time: float | None = None
-        self._last_final_audio_ms: int | None = None
-        self._pending_reopen_candidate: tuple[str, int, int] | None = None
-        self._pending_short_segment: _PendingShortSegment | None = None
-
-    @property
-    def _audio_ms(self) -> int:
-        """Cumulative audio received so far, in milliseconds."""
-        return int(self._total_samples / self.sample_rate * 1000)
-
-    def _apply_runtime_turn_detection(self, runtime_config: RuntimeConfig | None = None) -> None:
-        """Check RuntimeConfig for turn_detection changes and apply them."""
-        audio = runtime_config.session.audio if runtime_config else None
-        audio_input = audio.input if audio is not None else None
-        if not runtime_config or not audio_input or not audio_input.turn_detection:
-            return
-        td_raw = audio_input.turn_detection
-
-        # Convert Pydantic models (e.g. OpenAI SDK ServerVad) to dict;
-        # plain dicts pass through unchanged.
-        if hasattr(td_raw, "model_dump"):
-            td = td_raw.model_dump(exclude_none=True)
-        elif isinstance(td_raw, dict):
-            td = td_raw
-        else:
-            logger.warning(f"Unexpected turn_detection type: {type(td_raw)}")
-            return
-
-        # Compare normalized snapshot (identity on td_raw vs stored dict was wrong after first apply).
-        if td == self._last_turn_detection:
-            return
-
-        self._last_turn_detection = dict(td)
-
-        if "threshold" in td:
-            self.iterator.threshold = td["threshold"]
-            logger.info(f"VAD threshold updated to {td['threshold']}")
-        if "silence_duration_ms" in td:
-            self.iterator.min_silence_samples = self.sample_rate * td["silence_duration_ms"] / 1000
-            logger.info(f"VAD silence duration updated to {td['silence_duration_ms']}ms")
-
-    def _start_new_turn(self) -> tuple[str, int]:
-        self._cancel_pending_reopen()
-        self._turn_counter += 1
-        self._current_turn_id = f"turn_{self._turn_counter}"
-        self._current_turn_revision = 0
-        self._speculative_audio_prefix = None
-        self._last_final_wall_time = None
-        self._last_final_audio_ms = None
-        if self.speculative_turns:
-            self.speculative_turns.observe(self._current_turn_id, self._current_turn_revision)
-        return self._current_turn_id, self._current_turn_revision
-
-    def _speech_buffer_duration_ms(self) -> float:
-        if not hasattr(self.iterator, "speech_buffer"):
-            return 0.0
-        buffer_samples = sum(len(t) for t in self.iterator.speech_buffer())
-        return buffer_samples / self.sample_rate * 1000
-
-    def _current_active_speech_duration_ms(self) -> float:
-        active_speech_samples = getattr(self.iterator, "active_speech_samples", 0)
-        return active_speech_samples / self.sample_rate * 1000
-
-    def _last_utterance_active_speech_duration_ms(self) -> float:
-        active_speech_samples = getattr(self.iterator, "last_utterance_active_speech_samples", 0)
-        return active_speech_samples / self.sample_rate * 1000
-
-    @staticmethod
-    def _resolve_min_speech_continuation_ms(min_speech_ms: int, min_speech_continuation_ms: int) -> int:
-        if min_speech_continuation_ms <= 0:
-            return min_speech_ms
-        return min(
-            min_speech_ms,
-            max(_SHORT_SEGMENT_MIN_FRAGMENT_MS, min_speech_continuation_ms),
-        )
-
-    def _uses_realtime_turn_handling(self) -> bool:
-        return self.enable_realtime_transcription or self.speculative_turns is not None
-
-    def _active_speech_min_ms(self, start_ms: int) -> float:
-        """Duration hysteresis for speech that continues a reopenable turn."""
-        if self._pending_reopen_candidate is not None or self._should_reopen_current_turn(start_ms):
-            return self.min_speech_continuation_ms
-        return self.min_speech_ms
-
-    def _should_reopen_current_turn(self, audio_start_ms: int) -> bool:
-        if not self._uses_realtime_turn_handling():
-            return False
-        if self._current_turn_id is None or self._current_turn_revision is None or self._last_final_audio_ms is None:
-            return False
-
-        is_committed = self.speculative_turns is not None and self.speculative_turns.is_committed(
-            self._current_turn_id,
-            self._current_turn_revision,
-        )
-        if is_committed:
-            return False
-
-        # Elapsed is measured on the audio clock, so the window only advances
-        # while the client streams audio (continuous capture behaves like wall
-        # time; push-to-talk style gaps freeze it).
-        elapsed_ms = max(0, audio_start_ms - self._last_final_audio_ms)
-
-        # Within the short grace window, any uncommitted turn may reopen.
-        # Beyond it, an unanswered turn (no assistant output committed yet)
-        # remains reopenable up to the unanswered_reopen_ms sanity cap, so a
-        # user pause longer than speculative_reopen_ms does not orphan a turn
-        # the assistant has not replied to. The cap also bounds the
-        # empty-transcript case, where no request is queued and the turn would
-        # otherwise never commit.
-        reopen_limit_ms = self.speculative_reopen_ms
-        if self.speculative_turns is not None:
-            reopen_limit_ms = self.unanswered_reopen_ms
-        return elapsed_ms <= reopen_limit_ms
-
-    def _begin_pending_reopen_if_needed(self, audio_start_ms: int) -> None:
-        if self._pending_reopen_candidate is not None or not self._should_reopen_current_turn(audio_start_ms):
-            return
-        if self.speculative_turns is None:
-            return
-        candidate_revision = self.speculative_turns.begin_reopen_candidate(
-            self._current_turn_id,
-            self._current_turn_revision,
-        )
-        if candidate_revision is None or self._current_turn_id is None or self._current_turn_revision is None:
-            return
-        self._pending_reopen_candidate = (
-            self._current_turn_id,
-            self._current_turn_revision,
-            candidate_revision,
-        )
-        logger.info(
-            "VAD: pending reopen candidate for speculative turn %s revision %d",
-            self._current_turn_id,
-            candidate_revision,
-        )
-
-    def _cancel_pending_reopen(self) -> None:
-        if self._pending_reopen_candidate is None:
-            return
-        turn_id, _base_revision, candidate_revision = self._pending_reopen_candidate
-        if self.speculative_turns:
-            self.speculative_turns.cancel_reopen_candidate(turn_id, candidate_revision)
-        self._pending_reopen_candidate = None
-
-    def _confirm_pending_reopen(self) -> tuple[str, int, bool] | None:
-        if self._pending_reopen_candidate is None:
-            return None
-        turn_id, base_revision, candidate_revision = self._pending_reopen_candidate
-        self._pending_reopen_candidate = None
-        if self.speculative_turns and not self.speculative_turns.confirm_reopen_candidate(
-            turn_id,
-            base_revision,
-            candidate_revision,
-        ):
-            return None
-        self._current_turn_id = turn_id
-        self._current_turn_revision = candidate_revision
-        logger.info("VAD: reopened speculative turn %s revision %d", turn_id, candidate_revision)
-        return turn_id, candidate_revision, True
-
-    def _reopen_current_turn(self) -> tuple[str, int, bool] | None:
-        if self._current_turn_id is None or self._current_turn_revision is None:
-            return None
-
-        turn_id = self._current_turn_id
-        base_revision = self._current_turn_revision
-        if self.speculative_turns is not None:
-            candidate_revision = self.speculative_turns.begin_reopen_candidate(turn_id, base_revision)
-            if candidate_revision is None or not self.speculative_turns.confirm_reopen_candidate(
-                turn_id,
-                base_revision,
-                candidate_revision,
-            ):
-                return None
-        else:
-            candidate_revision = base_revision + 1
-
-        self._current_turn_id = turn_id
-        self._current_turn_revision = candidate_revision
-        logger.info("VAD: reopened speculative turn %s revision %d", turn_id, candidate_revision)
-        return turn_id, candidate_revision, True
-
-    def _ensure_turn_for_speech_start(self, audio_start_ms: int) -> tuple[str, int, bool]:
-        if (
-            self._speech_started_emitted
-            and self._current_turn_id is not None
-            and self._current_turn_revision is not None
-        ):
-            return self._current_turn_id, self._current_turn_revision, False
-
-        confirmed_reopen = self._confirm_pending_reopen()
-        if confirmed_reopen is not None:
-            return confirmed_reopen
-
-        reopened = False
-        if self._should_reopen_current_turn(audio_start_ms):
-            reopened_turn = self._reopen_current_turn()
-            if reopened_turn is not None:
-                return reopened_turn
-
-        self._start_new_turn()
-
-        if self._current_turn_id is None or self._current_turn_revision is None:
-            raise RuntimeError("VAD failed to allocate turn metadata")
-        if self.speculative_turns:
-            self.speculative_turns.observe(self._current_turn_id, self._current_turn_revision)
-        return self._current_turn_id, self._current_turn_revision, reopened
-
-    def _current_turn_metadata(self) -> tuple[str | None, int | None]:
-        return self._current_turn_id, self._current_turn_revision
-
-    def _combined_turn_audio(self, current_segment: np.ndarray) -> np.ndarray:
-        if self._speculative_audio_prefix is None:
-            return current_segment
-        return np.concatenate((self._speculative_audio_prefix, current_segment))
-
-    def _short_segment_merge_window_ms(self) -> int:
-        return int(getattr(self, "short_segment_merge_ms", 0))
-
-    def _segment_duration_ms(self, segment: np.ndarray) -> float:
-        return len(segment) / self.sample_rate * 1000
-
-    def _segment_start_ms(self, segment: np.ndarray, end_ms: int) -> int:
-        return max(0, end_ms - int(self._segment_duration_ms(segment)))
-
-    def _short_segment_gap_ms(self, start_ms: int) -> float:
-        if self._pending_short_segment is None:
-            return float("inf")
-        return max(0, start_ms - self._pending_short_segment.end_ms)
-
-    def _can_merge_pending_short_segment(self, start_ms: int) -> bool:
-        return (
-            self._pending_short_segment is not None
-            and self._short_segment_merge_window_ms() > 0
-            and self._short_segment_gap_ms(start_ms) <= self._short_segment_merge_window_ms()
-        )
-
-    def _effective_active_speech_for_start(self, start_ms: int, active_ms: float) -> tuple[int, float]:
-        # A live fragment below the noise floor never counts the held segment
-        # toward the speech-start threshold, mirroring the finalization path.
-        if active_ms < _SHORT_SEGMENT_MIN_FRAGMENT_MS:
-            return start_ms, active_ms
-        if not self._can_merge_pending_short_segment(start_ms):
-            return start_ms, active_ms
-        assert self._pending_short_segment is not None
-        return self._pending_short_segment.start_ms, self._pending_short_segment.active_ms + active_ms
-
-    def _merge_pending_short_segment(
-        self,
-        segment: np.ndarray,
-        active_ms: float,
-        end_ms: int,
-    ) -> tuple[np.ndarray, float, int, bool]:
-        start_ms = self._segment_start_ms(segment, end_ms)
-        if not self._can_merge_pending_short_segment(start_ms):
-            self._discard_expired_pending_short_segment(start_ms)
-            return segment, active_ms, start_ms, False
-
-        pending = self._pending_short_segment
-        assert pending is not None
-        # Reinsert the silence between the two segments so the stitched audio
-        # keeps its acoustic gap and its length matches the audio-clock span.
-        gap_samples = int(self._short_segment_gap_ms(start_ms) * self.sample_rate / 1000)
-        self._pending_short_segment = None
-        parts = [pending.audio]
-        if gap_samples > 0:
-            parts.append(np.zeros(gap_samples, dtype=segment.dtype))
-        parts.append(segment)
-        merged = np.concatenate(parts)
-        return merged, pending.active_ms + active_ms, pending.start_ms, True
-
-    def _hold_short_segment(self, segment: np.ndarray, active_ms: float, start_ms: int, end_ms: int) -> None:
-        self._pending_short_segment = _PendingShortSegment(
-            audio=segment,
-            active_ms=active_ms,
-            start_ms=start_ms,
-            end_ms=end_ms,
-        )
-        logger.info(
-            "VAD: holding short segment=%.0fms active=%.0fms (active_min=%sms, merge_max=%sms)",
-            self._segment_duration_ms(segment),
-            active_ms,
-            self.min_speech_ms,
-            self._short_segment_merge_window_ms(),
-        )
-
-    def _discard_pending_short_segment(self, reason: str = "expired") -> None:
-        pending = self._pending_short_segment
-        if pending is None:
-            return
-        self._pending_short_segment = None
-        logger.info(
-            "VAD: discarding held short segment=%.0fms active=%.0fms (%s, active_min=%sms)",
-            self._segment_duration_ms(pending.audio),
-            pending.active_ms,
-            reason,
-            self.min_speech_ms,
-        )
-
-    def _discard_expired_pending_short_segment(self, next_start_ms: int | None = None) -> None:
-        pending = self._pending_short_segment
-        if pending is None or self._short_segment_merge_window_ms() <= 0:
-            return
-        reference_ms = self._audio_ms if next_start_ms is None else next_start_ms
-        gap_ms = max(0, reference_ms - pending.end_ms)
-        if gap_ms > self._short_segment_merge_window_ms():
-            self._discard_pending_short_segment("merge window elapsed")
-
-    def before_emit_output(self, output: VADOut) -> None:
-        if isinstance(output, VADAudio):
-            self._drop_superseded_vad_audio(output)
-
-    def _drop_superseded_vad_audio(self, latest: VADAudio) -> int:
-        if not hasattr(self.queue_out, "mutex") or not hasattr(self.queue_out, "queue"):
-            return 0
-
-        dropped = 0
-        with self.queue_out.mutex:
-            kept: list[Any] = []
-            while self.queue_out.queue:
-                queued_item = self.queue_out.queue.popleft()
-                if isinstance(queued_item, VADAudio) and self._vad_audio_is_superseded(
-                    queued_item,
-                    latest,
-                ):
-                    dropped += 1
-                else:
-                    kept.append(queued_item)
-            self.queue_out.queue.extend(kept)
-            if dropped:
-                self.queue_out.not_full.notify_all()
-
-        if dropped:
-            logger.debug(
-                "VAD: dropped %d superseded audio chunk(s) before enqueueing turn=%s rev=%s mode=%s",
-                dropped,
-                latest.turn_id,
-                latest.turn_revision,
-                latest.mode,
-            )
-        return dropped
-
-    def _vad_audio_is_superseded(self, queued_item: VADAudio, latest: VADAudio) -> bool:
-        if queued_item.turn_id is None or queued_item.turn_revision is None:
-            return False
-        if self.speculative_turns is not None and not self.speculative_turns.is_latest(
-            queued_item.turn_id,
-            queued_item.turn_revision,
-        ):
-            return True
-        return (
-            queued_item.mode == "progressive"
-            and queued_item.turn_id == latest.turn_id
-            and queued_item.turn_revision == latest.turn_revision
-        )
-
-    def process(self, audio_chunk: VADIn) -> Iterator[VADOut]:
-        runtime_config = None
-        if isinstance(audio_chunk, tuple):
-            audio_chunk, runtime_config = audio_chunk
-        self._apply_runtime_turn_detection(runtime_config)
-
-        if not self.should_listen.is_set():
-            return
-
-        # Normal listening mode
-        self._log_chunks += 1
-        audio_int16 = np.frombuffer(audio_chunk, dtype=np.int16)
-        self._total_samples += len(audio_int16)
-        audio_float32 = int2float(audio_int16)
-
-        vad_output = self.iterator(torch.from_numpy(audio_float32))
-
-        # Deferred speech_started: only emit once active VAD speech reaches the valid speech threshold.
-        is_triggered_now = self.iterator.triggered
-        if is_triggered_now and not self._speech_started_emitted:
-            segment_samples = sum(len(t) for t in self.iterator.buffer)
-            segment_duration_ms = segment_samples / self.sample_rate * 1000
-            active_speech_duration_ms = self._current_active_speech_duration_ms()
-            speech_buffer_duration_ms = self._speech_buffer_duration_ms()
-            start_ms = max(0, self._audio_ms - int(speech_buffer_duration_ms))
-            effective_start_ms, effective_active_speech_duration_ms = self._effective_active_speech_for_start(
-                start_ms,
-                active_speech_duration_ms,
-            )
-            self._begin_pending_reopen_if_needed(effective_start_ms)
-            active_speech_min_ms = self._active_speech_min_ms(effective_start_ms)
-            if effective_active_speech_duration_ms >= active_speech_min_ms:
-                turn_id, turn_revision, reopened = self._ensure_turn_for_speech_start(effective_start_ms)
-                self._speech_started_emitted = True
-                self._log_speech_starts += 1
-                logger.info(
-                    "Speech started (confirmed, active=%.0fms, min=%.0fms, segment=%.0fms, turn=%s rev=%s)",
-                    effective_active_speech_duration_ms,
-                    active_speech_min_ms,
-                    segment_duration_ms,
-                    turn_id,
-                    turn_revision,
-                )
-                if self.text_output_queue:
-                    self.text_output_queue.put(
-                        SpeechStartedEvent(
-                            audio_start_ms=effective_start_ms,
-                            turn_id=turn_id,
-                            turn_revision=turn_revision,
-                            reopened=reopened,
-                        )
-                    )
-        elif not is_triggered_now and vad_output is None:
-            self._discard_expired_pending_short_segment()
-
-        # Log a summary once per second instead of every chunk
-        now = time.time()
-        if now - self._last_log_time >= 1.0:
-            state = "SPEAKING" if is_triggered_now else "silent"
-            logger.debug(
-                f"VAD: {self._log_chunks} chunks/s | {state} | "
-                f"starts={self._log_speech_starts} ends={self._log_speech_ends} progressive={self._log_progressive_yields}"
-            )
-            self._log_chunks = 0
-            self._log_speech_starts = 0
-            self._log_speech_ends = 0
-            self._log_progressive_yields = 0
-            self._last_log_time = now
-
-        if self._uses_realtime_turn_handling():
-            # Realtime mode keeps turns reopenable; live transcription additionally
-            # emits progressive audio chunks while speaking.
-            yield from self._process_realtime(vad_output, runtime_config)
-        else:
-            # Original mode: yield only when speech ends
-            yield from self._process_normal(vad_output, runtime_config)
-
-    def _process_realtime(
-        self,
-        vad_output: list[torch.Tensor] | None,
-        runtime_config: RuntimeConfig | None = None,
-    ) -> Iterator[VADOut]:
-        """Process with real-time progressive audio release."""
-        # Check if we're currently in a speech segment.
-        if self.enable_realtime_transcription and hasattr(self.iterator, "buffer") and len(self.iterator.buffer) > 0:
-            current_time = time.time()
-            duration_ms = self._speech_buffer_duration_ms()
-            progressive_pause = self._progressive_processing_pause(duration_ms)
-
-            # Yield accumulated audio periodically while speaking
-            if (current_time - self.last_process_time) >= progressive_pause:
-                array = torch.cat(self.iterator.speech_buffer()).cpu().numpy()
-                duration_ms = len(array) / self.sample_rate * 1000
-                active_speech_duration_ms = self._current_active_speech_duration_ms()
-                start_ms = max(0, self._audio_ms - int(duration_ms))
-
-                if active_speech_duration_ms >= self._active_speech_min_ms(start_ms):
-                    self._log_progressive_yields += 1
-                    logger.debug(
-                        "VAD: yielding progressive audio (segment=%.0fms, active=%.0fms, interval=%.2fs)",
-                        duration_ms,
-                        active_speech_duration_ms,
-                        progressive_pause,
-                    )
-                    turn_id, turn_revision = self._current_turn_metadata()
-                    yield VADAudio(
-                        audio=self._combined_turn_audio(array),
-                        runtime_config=runtime_config,
-                        mode="progressive",
-                        turn_id=turn_id,
-                        turn_revision=turn_revision,
-                    )
-                    self.last_process_time = current_time
-
-        # Handle end of speech
-        if vad_output is not None:
-            if len(vad_output) == 0:
-                logger.info("VAD: phantom trigger (empty buffer), closing speech pair")
-                if self._speech_started_emitted and self.text_output_queue:
-                    turn_id, turn_revision = self._current_turn_metadata()
-                    self.text_output_queue.put(
-                        SpeechStoppedEvent(
-                            audio_end_ms=self._audio_ms,
-                            turn_id=turn_id,
-                            turn_revision=turn_revision,
-                        )
-                    )
-                if not self._speech_started_emitted:
-                    self._cancel_pending_reopen()
-                self._speech_started_emitted = False
-                self._discard_expired_pending_short_segment()
-                return
-
-            array = torch.cat(vad_output).cpu().numpy()
-            end_ms = self._audio_ms
-            raw_active_ms = self._last_utterance_active_speech_duration_ms()
-            active_speech_duration_ms = raw_active_ms
-            stitched_short_segment = False
-            # Fragments below the noise floor never merge with or replace a
-            # held segment; the pending segment's own expiry handles it.
-            if raw_active_ms >= _SHORT_SEGMENT_MIN_FRAGMENT_MS:
-                array, active_speech_duration_ms, start_ms, stitched_short_segment = self._merge_pending_short_segment(
-                    array,
-                    active_speech_duration_ms,
-                    end_ms,
-                )
-            else:
-                start_ms = self._segment_start_ms(array, end_ms)
-            duration_ms = self._segment_duration_ms(array)
-            min_active_ms = 0.0 if self._speech_started_emitted else self._active_speech_min_ms(start_ms)
-
-            if active_speech_duration_ms < min_active_ms or duration_ms > self.max_speech_ms:
-                if (
-                    self._short_segment_merge_window_ms() > 0
-                    and raw_active_ms >= _SHORT_SEGMENT_MIN_FRAGMENT_MS
-                    and active_speech_duration_ms < min_active_ms
-                    and duration_ms <= self.max_speech_ms
-                ):
-                    self._hold_short_segment(array, active_speech_duration_ms, start_ms, end_ms)
-                else:
-                    logger.info(
-                        "VAD: discarding segment=%.0fms active=%.0fms (active_min=%sms, segment_max=%sms)",
-                        duration_ms,
-                        active_speech_duration_ms,
-                        min_active_ms,
-                        self.max_speech_ms,
-                    )
-                if self._speech_started_emitted and self.text_output_queue:
-                    turn_id, turn_revision = self._current_turn_metadata()
-                    self.text_output_queue.put(
-                        SpeechStoppedEvent(
-                            audio_end_ms=self._audio_ms,
-                            turn_id=turn_id,
-                            turn_revision=turn_revision,
-                        )
-                    )
-                if not self._speech_started_emitted:
-                    self._cancel_pending_reopen()
-                self._speech_started_emitted = False
-            else:
-                if stitched_short_segment:
-                    logger.info(
-                        "VAD: stitched short segment(s) into segment=%.0fms active=%.0fms",
-                        duration_ms,
-                        active_speech_duration_ms,
-                    )
-                if not self._speech_started_emitted:
-                    turn_id, turn_revision, reopened = self._ensure_turn_for_speech_start(start_ms)
-                    if self.text_output_queue:
-                        self.text_output_queue.put(
-                            SpeechStartedEvent(
-                                audio_start_ms=start_ms,
-                                turn_id=turn_id,
-                                turn_revision=turn_revision,
-                                reopened=reopened,
-                                interrupt_response=False,
-                            )
-                        )
-                else:
-                    turn_id, turn_revision = self._current_turn_metadata()
-                self._log_speech_ends += 1
-                logger.info(
-                    "Speech soft-ended (segment=%.0fms, active=%.0fms, turn=%s rev=%s)",
-                    duration_ms,
-                    active_speech_duration_ms,
-                    turn_id,
-                    turn_revision,
-                )
-                if self.audio_enhancement:
-                    array = self._apply_audio_enhancement(array)
-                output_array = self._combined_turn_audio(array)
-                combined_duration_s = len(output_array) / self.sample_rate
-                if self.text_output_queue:
-                    self.text_output_queue.put(
-                        SpeechStoppedEvent(
-                            duration_s=combined_duration_s,
-                            audio_end_ms=end_ms,
-                            turn_id=turn_id,
-                            turn_revision=turn_revision,
-                        )
-                    )
-                self._speculative_audio_prefix = output_array
-                self._last_final_wall_time = time.time()
-                self._last_final_audio_ms = end_ms
-                if self.speculative_turns:
-                    # The grace window only delays response commits; reopen
-                    # eligibility for unanswered turns is extended separately
-                    # via unanswered_reopen_ms in _should_reopen_current_turn.
-                    self.speculative_turns.start_reopen_grace(
-                        turn_id,
-                        turn_revision,
-                        self.speculative_reopen_ms / 1000.0,
-                    )
-                else:
-                    self.should_listen.clear()
-                yield VADAudio(
-                    audio=output_array,
-                    runtime_config=runtime_config,
-                    mode="final",
-                    turn_id=turn_id,
-                    turn_revision=turn_revision,
-                )
-                self.last_process_time = 0.0
-                self._speech_started_emitted = False
-
-    def _progressive_processing_pause(self, duration_ms: float) -> float:
-        base_pause = max(0.0, self.realtime_processing_pause)
-        duration_s = duration_ms / 1000.0
-        if duration_s < 8.0:
-            multiplier = 1.0
-        elif duration_s < 15.0:
-            multiplier = 2.0
-        elif duration_s < 30.0:
-            multiplier = 4.0
-        else:
-            multiplier = 6.0
-        return min(base_pause * multiplier, 2.0)
-
-    def _process_normal(
-        self,
-        vad_output: list[torch.Tensor] | None,
-        runtime_config: RuntimeConfig | None = None,
-    ) -> Iterator[VADOut]:
-        """Original processing: yield only when speech ends."""
-        if vad_output is not None:
-            if len(vad_output) == 0:
-                logger.info("VAD: phantom trigger (empty buffer), closing speech pair")
-                if self._speech_started_emitted and self.text_output_queue:
-                    self.text_output_queue.put(SpeechStoppedEvent(audio_end_ms=self._audio_ms))
-                self._speech_started_emitted = False
-                self._discard_expired_pending_short_segment()
-                return
-
-            array = torch.cat(vad_output).cpu().numpy()
-            end_ms = self._audio_ms
-            raw_active_ms = self._last_utterance_active_speech_duration_ms()
-            active_speech_duration_ms = raw_active_ms
-            stitched_short_segment = False
-            # Fragments below the noise floor never merge with or replace a
-            # held segment; the pending segment's own expiry handles it.
-            if raw_active_ms >= _SHORT_SEGMENT_MIN_FRAGMENT_MS:
-                array, active_speech_duration_ms, start_ms, stitched_short_segment = self._merge_pending_short_segment(
-                    array,
-                    active_speech_duration_ms,
-                    end_ms,
-                )
-            else:
-                start_ms = self._segment_start_ms(array, end_ms)
-            duration_ms = self._segment_duration_ms(array)
-            min_active_ms = 0.0 if self._speech_started_emitted else self._active_speech_min_ms(start_ms)
-            if active_speech_duration_ms < min_active_ms or duration_ms > self.max_speech_ms:
-                if (
-                    self._short_segment_merge_window_ms() > 0
-                    and raw_active_ms >= _SHORT_SEGMENT_MIN_FRAGMENT_MS
-                    and active_speech_duration_ms < min_active_ms
-                    and duration_ms <= self.max_speech_ms
-                ):
-                    self._hold_short_segment(array, active_speech_duration_ms, start_ms, end_ms)
-                else:
-                    logger.info(
-                        "VAD: discarding segment=%.0fms active=%.0fms (active_min=%sms, segment_max=%sms)",
-                        duration_ms,
-                        active_speech_duration_ms,
-                        min_active_ms,
-                        self.max_speech_ms,
-                    )
-                if self._speech_started_emitted and self.text_output_queue:
-                    self.text_output_queue.put(SpeechStoppedEvent(audio_end_ms=self._audio_ms))
-                self._speech_started_emitted = False
-            else:
-                if stitched_short_segment:
-                    logger.info(
-                        "VAD: stitched short segment(s) into segment=%.0fms active=%.0fms",
-                        duration_ms,
-                        active_speech_duration_ms,
-                    )
-                if not self._speech_started_emitted and self.text_output_queue:
-                    self.text_output_queue.put(
-                        SpeechStartedEvent(
-                            audio_start_ms=start_ms,
-                            interrupt_response=False,
-                        )
-                    )
-                self._log_speech_ends += 1
-                self.should_listen.clear()
-                logger.info(f"Speech ended ({duration_ms:.0f}ms), stop listening")
-                if self.text_output_queue:
-                    self.text_output_queue.put(SpeechStoppedEvent(duration_s=duration_ms / 1000.0, audio_end_ms=end_ms))
-                if self.audio_enhancement:
-                    array = self._apply_audio_enhancement(array)
-                yield VADAudio(audio=array, runtime_config=runtime_config)
-                self._speech_started_emitted = False
-
-    def _apply_audio_enhancement(self, array: np.ndarray) -> np.ndarray:
-        """Apply audio enhancement if enabled."""
-        import torchaudio
-
-        if self.sample_rate != self.df_state.sr():
-            audio_float32 = torchaudio.functional.resample(
-                torch.from_numpy(array),
-                orig_freq=self.sample_rate,
-                new_freq=self.df_state.sr(),
-            )
-            enhanced = enhance(
-                self.enhanced_model,
-                self.df_state,
-                audio_float32.unsqueeze(0),
-            )
-            enhanced = torchaudio.functional.resample(
-                enhanced,
-                orig_freq=self.df_state.sr(),
-                new_freq=self.sample_rate,
-            )
-        else:
-            enhanced = enhance(self.enhanced_model, self.df_state, torch.from_numpy(array))
-        return enhanced.numpy().squeeze()
-
-    def on_session_end(self):
-        self.iterator.reset_states()
-        self._pending_short_segment = None
-        self.iterator.buffer = []
-        self.last_process_time = 0.0
-        self._total_samples = 0
-        self._speech_started_emitted = False
-        self._turn_counter = 0
-        self._current_turn_id = None
-        self._current_turn_revision = None
-        self._speculative_audio_prefix = None
-        self._last_final_wall_time = None
-        self._last_final_audio_ms = None
-        self._pending_reopen_candidate = None
-        if self.speculative_turns:
-            self.speculative_turns.reset()
-        self.should_listen.set()
-        logger.debug("VAD session state reset")
-
-    @property
-    def min_time_to_debug(self) -> float:
-        return 0.00001

runtime/src/speech_to_speech/VAD/vad_iterator.py

@@ -1,170 +0,0 @@
-from collections import deque
-
-import torch
-
-
-class VADIterator:
-    def __init__(
-        self,
-        model,
-        threshold: float = 0.5,
-        sampling_rate: int = 16000,
-        min_silence_duration_ms: int = 300,
-        speech_pad_ms: int = 30,
-    ) -> None:
-        """
-        Mainly taken from https://github.com/snakers4/silero-vad
-        Class for stream imitation
-
-        Parameters
-        ----------
-        model: preloaded .jit/.onnx silero VAD model
-
-        threshold: float (default - 0.5)
-            Speech threshold. Silero VAD outputs speech probabilities for each audio chunk, probabilities ABOVE this value are considered as SPEECH.
-            It is better to tune this parameter for each dataset separately, but "lazy" 0.5 is pretty good for most datasets.
-
-        sampling_rate: int (default - 16000)
-            Currently silero VAD models support 8000 and 16000 sample rates
-
-        min_silence_duration_ms: int (default - 300 milliseconds)
-            In the end of each speech chunk wait for min_silence_duration_ms before separating it
-
-        speech_pad_ms: int (default - 30 milliseconds)
-            Retain up to speech_pad_ms of audio before VAD triggers and prepend it
-            to the detected speech chunk
-        """
-
-        self.model = model
-        self.threshold = threshold
-        self.sampling_rate = sampling_rate
-        self.is_speaking = False
-        self.buffer: list[torch.Tensor] = []
-        self.prefix_buffer: list[torch.Tensor] = []
-        self.active_speech_samples = 0
-        self.last_utterance_active_speech_samples = 0
-        self._pre_speech_buffer: deque[torch.Tensor] = deque()
-        self._pre_speech_samples = 0
-
-        if sampling_rate not in [8000, 16000]:
-            raise ValueError("VADIterator does not support sampling rates other than [8000, 16000]")
-
-        self.min_silence_samples = int(sampling_rate * min_silence_duration_ms / 1000)
-        self.speech_pad_samples = int(sampling_rate * speech_pad_ms / 1000)
-        self.reset_states()
-
-    def reset_states(self) -> None:
-        self.model.reset_states()
-        self.triggered = False
-        self.temp_end = 0
-        self.current_sample = 0
-        self.buffer = []
-        self.prefix_buffer = []
-        self.active_speech_samples = 0
-        self.last_utterance_active_speech_samples = 0
-        self._pre_speech_buffer.clear()
-        self._pre_speech_samples = 0
-
-    def _num_samples(self, chunk: torch.Tensor) -> int:
-        return len(chunk[0]) if chunk.dim() == 2 else len(chunk)
-
-    def _trim_pre_speech_buffer(self) -> None:
-        while (
-            self.speech_pad_samples > 0
-            and self._pre_speech_buffer
-            and self._pre_speech_samples > self.speech_pad_samples
-        ):
-            first = self._pre_speech_buffer[0]
-            first_samples = self._num_samples(first)
-            excess = self._pre_speech_samples - self.speech_pad_samples
-
-            if excess >= first_samples:
-                self._pre_speech_buffer.popleft()
-                self._pre_speech_samples -= first_samples
-                continue
-
-            if first.dim() == 2:
-                self._pre_speech_buffer[0] = first[:, excess:]
-            else:
-                self._pre_speech_buffer[0] = first[excess:]
-            self._pre_speech_samples -= excess
-
-    def _remember_pre_speech(self, chunk: torch.Tensor) -> None:
-        if self.speech_pad_samples <= 0:
-            self._pre_speech_buffer.clear()
-            self._pre_speech_samples = 0
-            return
-
-        self._pre_speech_buffer.append(chunk)
-        self._pre_speech_samples += self._num_samples(chunk)
-        self._trim_pre_speech_buffer()
-
-    def _speech_buffer(self) -> list[torch.Tensor]:
-        if not self.prefix_buffer:
-            return list(self.buffer)
-        return [*self.prefix_buffer, *self.buffer]
-
-    def speech_buffer(self) -> list[torch.Tensor]:
-        return self._speech_buffer()
-
-    @torch.no_grad()
-    def __call__(self, x: torch.Tensor) -> list[torch.Tensor] | None:
-        """
-        x: torch.Tensor
-            audio chunk (see examples in repo)
-
-        return_seconds: bool (default - False)
-            whether return timestamps in seconds (default - samples)
-        """
-
-        if not torch.is_tensor(x):
-            try:
-                x = torch.Tensor(x)
-            except Exception:
-                raise TypeError("Audio cannot be casted to tensor. Cast it manually")
-
-        window_size_samples = len(x[0]) if x.dim() == 2 else len(x)
-        self.current_sample += window_size_samples
-
-        speech_prob = self.model(x, self.sampling_rate).item()
-
-        if (speech_prob >= self.threshold) and not self.triggered:
-            self.triggered = True
-            self.prefix_buffer = list(self._pre_speech_buffer)
-            self._pre_speech_buffer.clear()
-            self._pre_speech_samples = 0
-            self.buffer.append(x)
-            self.active_speech_samples = window_size_samples
-            self.last_utterance_active_speech_samples = 0
-            return None
-
-        if not self.triggered:
-            self._remember_pre_speech(x)
-            return None
-
-        if self.triggered:
-            self.buffer.append(x)
-            if speech_prob >= self.threshold - 0.15:
-                self.active_speech_samples += window_size_samples
-                if self.temp_end:
-                    self.temp_end = 0
-                    return None
-
-            if speech_prob < self.threshold - 0.15:
-                if not self.temp_end:
-                    self.temp_end = self.current_sample
-                if self.current_sample - self.temp_end < self.min_silence_samples:
-                    return None
-
-                # End of speech: keep the final low-confidence chunks that were
-                # observed before VAD decided the utterance was done.
-                self.temp_end = 0
-                self.triggered = False
-                spoken_utterance = self.speech_buffer()
-                self.last_utterance_active_speech_samples = self.active_speech_samples
-                self.active_speech_samples = 0
-                self.buffer = []
-                self.prefix_buffer = []
-                return spoken_utterance
-
-        return None

runtime/src/speech_to_speech/__init__.py

@@ -1,3 +0,0 @@
-"""speech-to-speech: low-latency speech-to-speech pipeline."""
-
-__version__ = "0.2.11"

runtime/src/speech_to_speech/api/openai_realtime/README.md

@@ -1,244 +0,0 @@
-## Realtime Engine -- High-Level Architecture
-
-The server runs as a handler thread inside the existing pipeline. A FastAPI/uvicorn instance serves the WebSocket endpoint at `/v1/realtime`, while the same queue-driven handler chain (VAD, STT, LLM, TTS) processes audio in parallel threads.
-
-```mermaid
-flowchart LR
-    subgraph client [Client]
-        WS["WebSocket /v1/realtime"]
-    end
-
-    subgraph server [Realtime Server]
-        Router["WebSocket Router"]
-        Service["RealtimeService"]
-        Config["RuntimeConfig"]
-    end
-
-    subgraph pipeline [Pipeline Threads]
-        VAD["VAD"]
-        STT["STT"]
-        TN["TranscriptionNotifier"]
-        LLM["LLM"]
-        Proc["LMOutputProcessor"]
-        TTS["TTS"]
-    end
-
-    WS -- "client events (JSON)" --> Router
-    Router -- "parse / dispatch" --> Service
-    Service -- "PCM chunks" --> VAD
-    Service -- "session config" --> Config
-    Config -. "read by" .-> VAD & LLM & TTS
-    VAD -- "speech segments" --> STT
-    STT -- "transcript" --> TN
-    TN -- "transcript + notify" --> LLM
-    LLM -- "text + tools" --> Proc
-    Proc -- "clean text" --> TTS
-    TTS -- "PCM audio" --> Router
-    Proc -- "assistant_text / tools" --> Router
-    VAD -- "speech_started/stopped" --> Router
-    TN -- "transcription events" --> Router
-    Router -- "server events (JSON)" --> WS
-```
-
-**Key flow:**
-
-1. **Inbound audio**: Client sends `input_audio_buffer.append` with base64 PCM. `RealtimeService` decodes, resamples to 16 kHz, splits into 512-sample chunks, and puts them on the `recv_audio_chunks_queue` for VAD.
-2. **Speech detection**: VAD detects speech boundaries and emits `speech_started` / `speech_stopped` events on the `text_output_queue`. Full utterance audio goes to STT.
-3. **Transcription**: STT output passes through `TranscriptionNotifier`, which taps the transcript for `transcription.delta` / `transcription.completed` events before forwarding to the LLM.
-4. **Generation**: The LLM generates text (and optional tool calls). `LMOutputProcessor` splits the output: clean text goes to TTS, and `assistant_text` + tool call dicts go to the `text_output_queue`.
-5. **Outbound audio**: TTS writes PCM chunks to `send_audio_chunks_queue`. The router's async `_send_loop` drains both queues, encoding PCM as `response.output_audio.delta` events and translating internal messages into protocol events.
-6. **Session config**: `session.update` events deep-merge into `RuntimeConfig`, which is a shared Pydantic model read by VAD (turn detection thresholds), LLM (instructions, tools), and TTS (voice) at processing time.
-
----
-
-## Supported OpenAI Realtime Events
-
-### Client -> Server
-
-| Event | Description |
-|---|---|
-| `input_audio_buffer.append` | Stream base64 PCM audio. Decoded, resampled to 16 kHz, and chunked for the VAD. |
-| `session.update` | Deep-merge session config (instructions, tools, voice, turn detection, audio format). |
-| `conversation.item.create` | Inject `input_text` or `function_call_output` into the LLM context without triggering generation. |
-| `response.create` | Trigger LLM generation. Supports per-response `instructions` and `tool_choice` overrides. |
-| `response.cancel` | Cancel the in-progress response and re-enable listening. |
-
-### Server -> Client
-
-| Event | Description |
-|---|---|
-| `session.created` | Sent on connection with current session config. |
-| `error` | Protocol errors (`session_limit_reached`, `unknown_or_invalid_event`, `invalid_session_type`, `conversation_already_has_active_response`, etc.) |
-| `input_audio_buffer.speech_started` | VAD detected user speech. |
-| `input_audio_buffer.speech_stopped` | End of user speech segment. |
-| `conversation.item.created` | Acknowledges injected `input_text` from `conversation.item.create`. |
-| `conversation.item.input_audio_transcription.delta` | Streaming partial transcript (when live transcription is enabled). |
-| `conversation.item.input_audio_transcription.completed` | Final transcript for the user turn (with duration usage). |
-| `response.created` | Emitted on the first outbound audio chunk (response is `in_progress`). |
-| `response.output_audio.delta` | Base64 PCM audio chunk from TTS. |
-| `response.output_audio.done` | Audio stream complete for the current output item. |
-| `response.output_audio_transcript.done` | Full assistant text transcript for the turn. |
-| `response.function_call_arguments.done` | Tool call with `call_id`, `name`, and JSON `arguments`. |
-| `response.done` | Response finished (`completed`, `cancelled` with reason `turn_detected` or `client_cancelled`). |
-
----
-
-## WebRTC Transport
-
-Alongside the WebSocket endpoint, the server supports the OpenAI GA WebRTC handshake (requires the `webrtc` extra: `pip install 'speech-to-speech[webrtc]'`):
-
-```
-POST /v1/realtime/calls        Content-Type: application/sdp
-```
-
-The client POSTs an SDP offer and receives an SDP answer (`201`, with a `Location: /v1/realtime/calls/{call_id}` header). Audio then flows over RTP media tracks (Opus at 48 kHz, resampled to and from the 16 kHz pipeline rate with a stateful resampler), while all JSON events use the same protocol as WebSocket mode, carried on the `oai-events` data channel.
-
-A WebRTC session claims a pipeline unit from the same pool as WebSocket clients; the per-unit send loop stays the sole consumer of the pipeline output queues and hands PCM to the transport, which paces it out as 20 ms RTP frames (silence when idle).
-
-Differences from WebSocket mode:
-
-- `input_audio_buffer.append` is rejected with `invalid_event_for_transport` — audio arrives on the media track.
-- `output_audio_buffer.clear` is supported (WebRTC only): unplayed audio buffers server-side, so barge-in and cancellation flush it there. The server also flushes it automatically on `response.cancel` and VAD interruption.
-- `session.created` is sent when the data channel opens rather than on connection.
-
-ICE servers (STUN/TURN) can be configured via the `SPEECH_TO_SPEECH_ICE_SERVERS` env var, a JSON list of server entries:
-
-```bash
-export SPEECH_TO_SPEECH_ICE_SERVERS='[{"urls": "stun:stun.example.com:3478"}, {"urls": "turn:turn.example.com", "username": "u", "credential": "c"}]'
-```
-
-Without it, aiortc defaults apply (host candidates + Google STUN). Deployments where clients cannot reach the server directly (symmetric NAT, containers without exposed UDP) need a TURN server.
-
----
-
-## Tool Calling Design
-
-Tool calling works through two distinct paths depending on the LLM backend, but both converge to the same wire protocol for the client.
-
-### Local LLM path (`LanguageModelHandler` -- transformers / mlx-lm)
-
-Tools defined in `session.update` are converted to `FunctionTool` objects. Each tool's JSON Schema `parameters` are turned into a Python `inspect.Signature` (via `signature_from_schema`), and `to_code_prompt()` renders a human-readable `def name(...): """docstring"""` block.
-
-These tool signatures are injected into the system prompt using a Jinja2 template (`tool_prompt.py`) that instructs the model to wrap tool calls in `<code>...</code>` delimiters:
-
-```
-<code>
-function_name(arg_name_1=value1, arg_name_2='string_value')
-</code>
-```
-
-After generation, `_extract_tools` uses a regex to find `<code>` blocks, then `extract_function_calls_from_text` parses each `name(kwargs)` call and validates it against registered tools. Valid calls become `ResponseFunctionToolCall` dicts with generated `call_id`s.
-
-### OpenAI API path (`ResponsesApiModelHandler`)
-
-Tools are passed natively as the `tools=` parameter to `client.responses.create`. The API returns structured `function_call` items directly -- no prompt engineering or regex parsing needed. Per-response `tool_choice` overrides from `response.create` are supported.
-
-### Common output path
-
-Both handlers yield `(text, language_code, tools)` tuples. `LMOutputProcessor` forwards clean text to TTS and puts `{"type": "assistant_text", "text": ..., "tools": [...]}` on the `text_output_queue`. The router's `_send_loop` translates these into:
-- `response.output_audio_transcript.done` for the text
-- `response.function_call_arguments.done` for each tool call
-
-### Tool result flow
-
-1. Client executes the tool and sends `conversation.item.create` with `type: "function_call_output"` and `output: "<result>"`
-2. `RealtimeService` appends the tool output to the chat context and emits `conversation.item.created`; this does not trigger generation.
-3. If the tool result needs to be spoken to the user, such as camera/search/data results, the client sends `response.create` to trigger follow-up generation.
-4. For fire-and-forget robot actions such as dance, emotion, head movement, stop, or idle tools, the client can stop after `conversation.item.created`; the assistant should already have spoken the natural lead-in before the tool call.
-
----
-
-## Interruption Handling
-
-Barge-in (user speaks while the assistant is playing audio) is handled cooperatively between the VAD, the `_send_loop`, and the LLM/TTS handlers via a shared `CancelScope` object (`cancel_scope.py`).
-
-### CancelScope design
-
-`CancelScope` replaces the old two-signal pattern (`cancel_response` Event + `discard_stale_output` boolean) with a single object that manages:
-
-- **Generation counter** (`cancel_scope.generation`): pipeline threads (LLM, TTS) capture the current generation at the start of each response and check `cancel_scope.is_stale(gen)` on every streaming token. When `cancel()` is called, the generation increments and all prior generations become stale -- no timing games required.
-- **Discard flag** (`cancel_scope.discarding`): set by `cancel()`, checked by the async `_send_loop` to drop output from superseded generations that arrives between `cancel()` and `response_done()`. Cleared by `response_done(generation)` (only when the sentinel's generation matches the discarded or current one -- sentinels from unrelated older generations are ignored), by `new_response()` on an explicit `response.create`, or by `reset()` on session claim/release.
-
-Pipeline output is **generation-tagged**: `AudioOutput` chunks and `AssistantTextEvent`s carry a `cancel_generation` field stamped by the handler that produced them. The send loop's `_generation_is_discardable` drops an item if its generation is stale, or if `discarding` is set and the item is not from the current generation. Output from the *current* generation always passes through, so a fresh response is never swallowed by a lingering discard window (e.g. a superseded speculative turn whose TTS never emitted a `__RESPONSE_DONE__` sentinel).
-
-```mermaid
-sequenceDiagram
-    participant User
-    participant VAD
-    participant SendLoop as _send_loop
-    participant LLM
-    participant TTS
-    participant Client
-
-    Note over TTS,Client: Response active or pending (in_response / response_pending)
-    User->>VAD: speaks
-    VAD->>SendLoop: speech_started on text_output_queue
-    SendLoop->>Client: input_audio_buffer.speech_started
-    SendLoop->>Client: response.done (status=cancelled, reason=turn_detected)
-    SendLoop->>SendLoop: cancel_scope.cancel() (gen++ & discarding=True)
-    SendLoop->>SendLoop: flush output_queue + text_output_queue
-    SendLoop->>SendLoop: response_playing.clear()
-    LLM->>LLM: is_stale(gen) → True, aborts generation
-    TTS->>TTS: is_stale(gen) → True, aborts generation
-    TTS->>SendLoop: __RESPONSE_DONE__ (tagged with gen)
-    SendLoop->>SendLoop: cancel_scope.response_done(gen) (discarding=False)
-    Note over VAD,Client: Pipeline is now processing the new user utterance
-```
-
-**Step by step:**
-
-1. **VAD detects speech**: puts a `SpeechStartedEvent` on `text_output_queue`.
-2. **`_send_loop` processes text events first** (priority over audio): translates `speech_started` into protocol events. If an active response was in progress, `RealtimeService.dispatch_pipeline_event` emits `response.output_audio.done` + `response.done` with `status="cancelled"` and `reason="turn_detected"`.
-3. **Cancel + queue flush**: if a response is active (`in_response`) *or* pending (`response_pending` -- a model request is queued, but no output has started), and interrupts are enabled (see step 4), the send loop calls `cancel_scope.cancel()` (increments generation, enables discard), clears `response_pending`, drains `output_queue` (preserving `__RESPONSE_DONE__` sentinels) and `text_output_queue` (preserving user-side events: `speech_stopped`, direct-audio completions, partial/completed transcriptions, token usage), then clears `response_playing`.
-4. **Interrupt gating**: the cancel only fires if the `SpeechStartedEvent.interrupt_response` flag is set *and* the session config allows it (`turn_detection.interrupt_response`, read via `RuntimeConfig.interrupt_response_enabled`, default true). When disabled, user speech during a response is transcribed but the response keeps playing.
-5. **LLM/TTS cancellation**: handlers capture `gen = cancel_scope.generation` at the start of each response and check `cancel_scope.is_stale(gen)` on every streaming token, aborting early when stale.
-6. **Discard guard**: while `cancel_scope.discarding` is True, the send loop drops audio chunks and assistant text whose `cancel_generation` is not current (see `_generation_is_discardable` above). The guard clears when a `__RESPONSE_DONE__` with a matching generation arrives (via `cancel_scope.response_done(gen)`), or when an explicit `response.create` starts a new response (`cancel_scope.new_response()`).
-7. **Client-initiated cancel**: `response.cancel` calls `cancel_scope.cancel()` (only if a response was active), flushes both queues with the same preservation rules, triggers `finish_response(status="cancelled", reason="client_cancelled")`, re-enables `should_listen`, and clears `response_playing`.
-8. **Spurious cancel safety**: if no response is active, `cancel_scope.cancel()` is not called, preventing the discard guard from being set without a `__RESPONSE_DONE__` to clear it.
-
----
-
-## Testing
-
-### Local LLM with Transformers
-
-```bash
-uv run speech-to-speech \
-  --mode realtime \
-  --stt parakeet-tdt \
-  --llm_backend transformers \
-  --tts kokoro \
-  --model_name "Qwen/Qwen3-4B-Instruct-2507" \
-  --llm_device mps \
-  --llm_torch_dtype float16 \
-  --enable_live_transcription
-```
-
-### Local LLM with MLX-LM
-
-```bash
-uv run speech-to-speech \
-  --mode realtime \
-  --stt parakeet-tdt \
-  --llm_backend mlx-lm \
-  --tts kokoro \
-  --model_name "mlx-community/Qwen3-4B-Instruct-2507-bf16" \
-  --llm_device mps \
-  --llm_torch_dtype float16 \
-  --enable_live_transcription
-```
-
-### Remote LLM with OpenAI-compatible API
-
-```bash
-uv run speech-to-speech \
-  --mode realtime \
-  --stt parakeet-tdt \
-  --llm_backend responses-api \
-  --tts kokoro \
-  --model_name "openai/gpt-oss-20b:groq" \
-  --responses_api_base_url "https://router.huggingface.co/v1" \
-  --responses_api_api_key "$HF_TOKEN" \
-  --responses_api_stream \
-  --enable_live_transcription
-```

runtime/src/speech_to_speech/api/openai_realtime/handlers/__init__.py

@@ -1,11 +0,0 @@
-from speech_to_speech.api.openai_realtime.handlers.audio import AudioHandler
-from speech_to_speech.api.openai_realtime.handlers.conversation import ConversationHandler
-from speech_to_speech.api.openai_realtime.handlers.response import ResponseHandler
-from speech_to_speech.api.openai_realtime.handlers.session import SessionHandler
-
-__all__ = [
-    "AudioHandler",
-    "ConversationHandler",
-    "ResponseHandler",
-    "SessionHandler",
-]

runtime/src/speech_to_speech/api/openai_realtime/handlers/audio.py

@@ -1,224 +0,0 @@
-from __future__ import annotations
-
-import base64
-import logging
-from typing import TYPE_CHECKING
-
-from openai.types.realtime import (
-    InputAudioBufferAppendEvent,
-    InputAudioBufferSpeechStartedEvent,
-    InputAudioBufferSpeechStoppedEvent,
-    RealtimeErrorEvent,
-    ResponseAudioDeltaEvent,
-    ResponseCreatedEvent,
-)
-
-from speech_to_speech.api.openai_realtime.handlers.base import RealtimeBaseHandler
-from speech_to_speech.api.openai_realtime.utils import resample
-from speech_to_speech.pipeline.events import SpeechStartedEvent, SpeechStoppedEvent
-
-if TYPE_CHECKING:
-    from speech_to_speech.api.openai_realtime.service import ServerEvent
-
-logger = logging.getLogger(__name__)
-
-PIPELINE_SAMPLE_RATE = 16000
-CHUNK_SAMPLES = 512
-BYTES_PER_SAMPLE = 2
-CHUNK_SIZE_BYTES = CHUNK_SAMPLES * BYTES_PER_SAMPLE
-
-
-class AudioHandler(RealtimeBaseHandler):
-    """Owns inbound audio decoding/chunking and outbound audio encoding."""
-
-    def _start_input_item(self, conn_id: str, *, preserve_active_response: bool = False) -> str:
-        response = self._service.response
-        st = self._state(conn_id)
-        if not preserve_active_response:
-            item_id = response._start_item(conn_id)
-        else:
-            response_item_id = st.current_item_id
-            response_content_index = st.content_index
-            item_id = response._start_item(conn_id)
-            st.current_item_id = response_item_id
-            st.content_index = response_content_index
-        st.input_content_index = 0
-        return item_id
-
-    def handle_audio_append(self, conn_id: str, event: InputAudioBufferAppendEvent) -> list[bytes]:
-        """Decode base64 audio, resample to pipeline rate, and split into 512-sample PCM16 chunks for the VAD."""
-        try:
-            pcm_bytes = base64.b64decode(event.audio)
-        except Exception as e:
-            logger.error(f"Base64 decode error: {e}")
-            return []
-
-        st = self._state(conn_id)
-
-        audio_cfg = st.runtime_config.session.audio
-        if audio_cfg is not None and audio_cfg.input is not None:
-            client_in_rate = getattr(audio_cfg.input.format, "rate", None) or PIPELINE_SAMPLE_RATE
-        else:
-            client_in_rate = PIPELINE_SAMPLE_RATE
-        return self.append_pcm(conn_id, pcm_bytes, client_in_rate)
-
-    def append_pcm(self, conn_id: str, pcm_bytes: bytes, src_rate: int) -> list[bytes]:
-        """Resample raw PCM16 to the pipeline rate and split into 512-sample chunks for the VAD.
-
-        Shared by both transports: the WebSocket route feeds it decoded
-        ``input_audio_buffer.append`` payloads, the WebRTC transport feeds it
-        PCM decoded from inbound media-track frames. Keeps the sub-chunk
-        remainder and the commit bookkeeping (``audio_buffer_has_data``) in
-        one place regardless of how audio arrives.
-        """
-        st = self._state(conn_id)
-        pcm_bytes = resample(pcm_bytes, src_rate, PIPELINE_SAMPLE_RATE)
-
-        pcm_bytes = st.audio_remainder + pcm_bytes
-
-        chunks = []
-        for i in range(0, len(pcm_bytes), CHUNK_SIZE_BYTES):
-            chunk = pcm_bytes[i : i + CHUNK_SIZE_BYTES]
-            if len(chunk) == CHUNK_SIZE_BYTES:
-                chunks.append(chunk)
-            else:
-                st.audio_remainder = chunk
-                break
-        else:
-            st.audio_remainder = b""
-
-        if chunks:
-            st.audio_buffer_has_data = True
-        return chunks
-
-    def handle_audio_commit(self, conn_id: str) -> RealtimeErrorEvent | None:
-        """Commit the audio buffer. Returns an error if no audio was appended."""
-        st = self._state(conn_id)
-        if not st.audio_buffer_has_data:
-            return self.make_error(
-                message="Input audio buffer is empty, nothing to commit.",
-                _type="input_audio_buffer_commit_empty",
-            )
-        st.audio_buffer_has_data = False
-        logger.debug("Audio buffer committed")
-        return None
-
-    # ── Pipeline event handlers ────────────────────
-
-    def on_speech_started(self, conn_id: str, event: SpeechStartedEvent) -> list[ServerEvent]:
-        """Handle VAD speech_started: cancel active response if interrupts enabled, start new input item."""
-        response = self._service.response
-        events: list[ServerEvent] = []
-        st = self._state(conn_id)
-        if st.in_response and event.interrupt_response and st.runtime_config.interrupt_response_enabled:
-            events.extend(response.finish_response(conn_id, status="cancelled", reason="turn_detected"))
-        is_reopen = bool(event.reopened and event.turn_id is not None and event.turn_id == st.speculative_turn_id)
-        preserve_active_response = st.in_response
-        if is_reopen:
-            input_item_id = st.speculative_input_item_id
-            if input_item_id is None:
-                input_item_id = self._start_input_item(
-                    conn_id,
-                    preserve_active_response=preserve_active_response,
-                )
-                st.speculative_input_item_id = input_item_id
-            elif not preserve_active_response:
-                st.current_item_id = input_item_id
-                st.content_index = 0
-            st.input_audio_duration_s = 0.0
-            st.input_content_index = 0
-        else:
-            input_item_id = self._start_input_item(
-                conn_id,
-                preserve_active_response=preserve_active_response,
-            )
-            st.speculative_input_item_id = input_item_id
-            st.response_usage.turns += 1
-        st.speculative_turn_id = event.turn_id
-        st.speculative_turn_revision = event.turn_revision
-        st.last_item_id = input_item_id
-        events.append(
-            InputAudioBufferSpeechStartedEvent(
-                type="input_audio_buffer.speech_started",
-                event_id=self._next_event_id(),
-                audio_start_ms=event.audio_start_ms,
-                item_id=input_item_id,
-            )
-        )
-        return events
-
-    def on_speech_stopped(self, conn_id: str, event: SpeechStoppedEvent) -> list[ServerEvent]:
-        """Handle VAD speech_stopped: record duration and emit stopped event."""
-        if event.duration_s:
-            self._state(conn_id).input_audio_duration_s = event.duration_s
-        return [
-            InputAudioBufferSpeechStoppedEvent(
-                type="input_audio_buffer.speech_stopped",
-                event_id=self._next_event_id(),
-                audio_end_ms=event.audio_end_ms,
-                item_id=self._input_item_id(conn_id),
-            )
-        ]
-
-    # ── Outbound audio encoding ──────────────────
-
-    def begin_audio_response(self, conn_id: str) -> tuple[str, str, list[ServerEvent]]:
-        """Ensure a response exists for outbound audio, emitting ResponseCreated once.
-
-        When ``handle_response_create`` already allocated the response,
-        ``current_response_id`` is set and no duplicate event is emitted.
-        For the implicit-response path (VAD -> STT -> LLM -> TTS, no
-        ``response.create``), ``current_response_id`` is still ``None``
-        and the event is emitted here on the first audio chunk.
-
-        Returns ``(response_id, item_id, events)``. Shared by both
-        transports: the WebSocket path appends the base64 audio delta to the
-        returned events, the WebRTC path sends only the bookkeeping events
-        over the data channel while audio travels on the media track.
-        """
-        response = self._service.response
-        st = self._state(conn_id)
-
-        events: list[ServerEvent] = []
-        need_created = st.current_response_id is None
-        resp_id, item_id = response._ensure_response(conn_id)
-        if need_created:
-            events.append(
-                ResponseCreatedEvent(
-                    type="response.created",
-                    event_id=self._next_event_id(),
-                    response=response._build_response(conn_id, "in_progress"),
-                )
-            )
-        return resp_id, item_id, events
-
-    def encode_audio_chunk(self, conn_id: str, audio: bytes) -> list[ServerEvent]:
-        """Encode a raw PCM audio chunk as a base64 delta event for the WebSocket transport."""
-        response = self._service.response
-        st = self._state(conn_id)
-
-        resp_id, item_id, events = self.begin_audio_response(conn_id)
-        rp = st.current_response_params
-        client_out_rate = None
-        if rp and rp.audio and rp.audio.output and rp.audio.output.format:
-            client_out_rate = getattr(rp.audio.output.format, "rate", None)
-        if client_out_rate is None:
-            audio_cfg = st.runtime_config.session.audio
-            if audio_cfg is not None and audio_cfg.output is not None:
-                client_out_rate = getattr(audio_cfg.output.format, "rate", None) or PIPELINE_SAMPLE_RATE
-            else:
-                client_out_rate = PIPELINE_SAMPLE_RATE
-        audio = resample(audio, PIPELINE_SAMPLE_RATE, client_out_rate)
-        b64 = base64.b64encode(audio).decode("ascii")
-        events.append(
-            ResponseAudioDeltaEvent(
-                type="response.output_audio.delta",
-                event_id=self._next_event_id(),
-                content_index=response._next_content_index(conn_id),
-                delta=b64,
-                item_id=item_id,
-                output_index=0,
-                response_id=resp_id,
-            )
-        )
-        return events

runtime/src/speech_to_speech/api/openai_realtime/handlers/base.py

@@ -1,57 +0,0 @@
-from __future__ import annotations
-
-import logging
-from queue import Queue
-from threading import Event as ThreadingEvent
-from typing import TYPE_CHECKING
-
-from openai.types.realtime import RealtimeErrorEvent
-
-from speech_to_speech.utils.utils import _generate_id
-
-if TYPE_CHECKING:
-    from speech_to_speech.api.openai_realtime.service import ConnState, RealtimeService
-
-logger = logging.getLogger(__name__)
-
-
-class RealtimeBaseHandler:
-    """Shared base for domain handlers.
-
-    Provides conn_id-keyed accessors for per-connection state, config,
-    and queues.  Each connection owns its own ``RuntimeConfig`` via
-    ``ConnState``.
-    """
-
-    def __init__(self, service: RealtimeService) -> None:
-        self._service = service
-
-    # ── conn_id-keyed accessors ──
-
-    def _queue(self, conn_id: str) -> Queue | None:
-        return self._service.text_prompt_queue
-
-    def _should_listen(self, conn_id: str) -> ThreadingEvent | None:
-        return self._service.should_listen
-
-    # ── shared helpers ────────────────────────────────
-
-    def _state(self, conn_id: str) -> ConnState:
-        return self._service._state(conn_id)
-
-    def _input_item_id(self, conn_id: str) -> str:
-        st = self._state(conn_id)
-        return st.speculative_input_item_id or self._service.response._current_item_id(conn_id)
-
-    def _next_input_content_index(self, conn_id: str) -> int:
-        st = self._state(conn_id)
-        idx = st.input_content_index
-        st.input_content_index += 1
-        return idx
-
-    @staticmethod
-    def _next_event_id() -> str:
-        return _generate_id("event")
-
-    def make_error(self, message: str, _type: str) -> RealtimeErrorEvent:
-        return self._service.make_error(message, _type)

runtime/src/speech_to_speech/api/openai_realtime/handlers/conversation.py

@@ -1,128 +0,0 @@
-from __future__ import annotations
-
-import logging
-from typing import TYPE_CHECKING
-
-from openai.types.realtime import (
-    ConversationItem,
-    ConversationItemCreatedEvent,
-    ConversationItemCreateEvent,
-    ConversationItemInputAudioTranscriptionCompletedEvent,
-    ConversationItemInputAudioTranscriptionDeltaEvent,
-)
-from openai.types.realtime.conversation_item_input_audio_transcription_completed_event import (
-    UsageTranscriptTextUsageDuration,
-)
-
-from speech_to_speech.api.openai_realtime.handlers.base import RealtimeBaseHandler
-from speech_to_speech.LLM.chat import ChatItemError, add_supported_item
-from speech_to_speech.pipeline.events import PartialTranscriptionEvent, TranscriptionCompletedEvent
-
-if TYPE_CHECKING:
-    from speech_to_speech.api.openai_realtime.service import ServerEvent
-
-logger = logging.getLogger(__name__)
-
-
-class ConversationHandler(RealtimeBaseHandler):
-    """Owns conversation item injection and pipeline-to-protocol translation."""
-
-    def handle_conversation_item_create(
-        self,
-        conn_id: str,
-        event: ConversationItemCreateEvent,
-    ) -> list[ServerEvent]:
-        """Inject a text message or function-call output into the LLM context.
-
-        Items are added to the LLM chat context but do NOT trigger response
-        generation on their own.  A subsequent ``response.create`` event is
-        required to trigger the model.
-
-        While a response is generating, the item is *deferred*: applying it now
-        would race the LLM handler's end-of-turn chat write-back, which runs on
-        the pipeline thread (e.g. a ``function_call_output`` arriving before its
-        ``function_call`` is recorded, or an image stripped before the next
-        turn reads it). Deferred items are flushed, in order, once the response
-        completes — see :meth:`flush_deferred_items`.
-        """
-        st = self._state(conn_id)
-        if st.in_response:
-            st.deferred_items.append(event.item)
-            logger.debug("Deferred conversation item until the active response completes")
-            return []
-        return self._apply_item(conn_id, event.item)
-
-    def _apply_item(self, conn_id: str, item: ConversationItem) -> list[ServerEvent]:
-        """Add one item to the chat and build its ``conversation.item.created``."""
-        try:
-            self._append_item(conn_id, item)
-        except ChatItemError as exc:
-            return [self.make_error(str(exc), "invalid_conversation_item")]
-
-        if not item:
-            return []
-        st = self._state(conn_id)
-        event = ConversationItemCreatedEvent(
-            type="conversation.item.created",
-            event_id=self._next_event_id(),
-            previous_item_id=st.last_item_id,
-            item=item,
-        )
-        st.last_item_id = item.id
-        return [event]
-
-    def flush_deferred_items(self, conn_id: str) -> list[ServerEvent]:
-        """Apply items buffered during a response, in arrival order.
-
-        Called at response completion (after the generation's own write-back),
-        so a ``function_call_output`` pairs with its now-recorded ``function_call``
-        and an image survives the just-finished response's ``strip_images``.
-        """
-        st = self._state(conn_id)
-        if not st.deferred_items:
-            return []
-        items = st.deferred_items
-        st.deferred_items = []
-        events: list[ServerEvent] = []
-        for item in items:
-            events.extend(self._apply_item(conn_id, item))
-        return events
-
-    def _append_item(self, conn_id: str, item: ConversationItem) -> None:
-        """Narrow ``ConversationItem`` to ``SupportedItem`` and delegate to ``Chat.add_item``.
-
-        Raises :class:`ChatItemError` on validation failure or unsupported type.
-        """
-        add_supported_item(self._state(conn_id).runtime_config.chat, item)
-
-    # ── Pipeline event handlers ────────────────────
-
-    def on_partial_transcription(self, conn_id: str, event: PartialTranscriptionEvent) -> list[ServerEvent]:
-        """Handle partial_transcription: emit transcription delta event."""
-        return [
-            ConversationItemInputAudioTranscriptionDeltaEvent(
-                type="conversation.item.input_audio_transcription.delta",
-                event_id=self._next_event_id(),
-                content_index=self._next_input_content_index(conn_id),
-                item_id=self._input_item_id(conn_id),
-                delta=event.delta,
-            )
-        ]
-
-    def on_transcription_completed(self, conn_id: str, event: TranscriptionCompletedEvent) -> list[ServerEvent]:
-        """Handle transcription_completed: accumulate duration and emit completed event."""
-        st = self._state(conn_id)
-        st.response_usage.audio_duration_s += st.input_audio_duration_s
-        return [
-            ConversationItemInputAudioTranscriptionCompletedEvent(
-                type="conversation.item.input_audio_transcription.completed",
-                event_id=self._next_event_id(),
-                content_index=0,
-                item_id=self._input_item_id(conn_id),
-                transcript=event.transcript,
-                usage=UsageTranscriptTextUsageDuration(
-                    seconds=st.input_audio_duration_s,
-                    type="duration",
-                ),
-            )
-        ]

runtime/src/speech_to_speech/api/openai_realtime/handlers/response.py

@@ -1,339 +0,0 @@
-from __future__ import annotations
-
-import logging
-from typing import TYPE_CHECKING, Optional
-
-from openai.types.realtime import (
-    RealtimeResponse,
-    ResponseAudioDoneEvent,
-    ResponseAudioTranscriptDoneEvent,
-    ResponseCreatedEvent,
-    ResponseCreateEvent,
-    ResponseDoneEvent,
-    ResponseFunctionCallArgumentsDoneEvent,
-    ResponseTextDeltaEvent,
-    ResponseTextDoneEvent,
-)
-from openai.types.realtime.realtime_response import Audio, AudioOutput
-from openai.types.realtime.realtime_response_status import RealtimeResponseStatus
-from openai.types.realtime.realtime_response_usage import RealtimeResponseUsage
-
-from speech_to_speech.api.openai_realtime.handlers.base import RealtimeBaseHandler
-from speech_to_speech.LLM.chat import ChatItemError
-from speech_to_speech.pipeline.events import AssistantTextEvent
-from speech_to_speech.pipeline.messages import GenerateResponseRequest
-from speech_to_speech.utils.utils import _generate_id, is_out_of_band, response_wants_audio
-
-if TYPE_CHECKING:
-    from speech_to_speech.api.openai_realtime.service import ServerEvent, _ResponseStatus, _StatusReason
-
-logger = logging.getLogger(__name__)
-
-
-class ResponseHandler(RealtimeBaseHandler):
-    """Owns the response lifecycle: create, cancel, finish, and ID management."""
-
-    # ── ID / state helpers ────────────────────────
-
-    def _ensure_response(self, conn_id: str) -> tuple[str, str]:
-        """Ensure a response and output item exist, creating them if needed."""
-        st = self._state(conn_id)
-        if st.current_response_id is None:
-            st.current_response_id = _generate_id("resp")
-            self._start_item(conn_id)
-            st.in_response = True
-        st.response_pending = False
-        return st.current_response_id, self._current_item_id(conn_id)
-
-    def _end_response(self, conn_id: str, status: _ResponseStatus = "completed") -> None:
-        st = self._state(conn_id)
-        if status == "cancelled":
-            st.response_usage.responses_cancelled += 1
-        else:
-            st.response_usage.responses_completed += 1
-        self._service.total_usage += st.response_usage
-        logger.info(
-            "Response done (status=%s) — this response: input_tokens=%d, output_tokens=%d, audio=%.2fs"
-            " | cumulative: input_tokens=%d, output_tokens=%d, audio=%.2fs",
-            status,
-            st.response_usage.input_tokens,
-            st.response_usage.output_tokens,
-            st.response_usage.audio_duration_s,
-            self._service.total_usage.input_tokens,
-            self._service.total_usage.output_tokens,
-            self._service.total_usage.audio_duration_s,
-        )
-        st.response_usage.reset()
-        st.current_response_id = None
-        st.current_item_id = None
-        st.content_index = 0
-        st.in_response = False
-        st.response_pending = False
-        st.current_response_params = None
-        st.pending_output_text_parts = []
-
-    def _start_item(self, conn_id: str) -> str:
-        """Generate a new item ID, reset content index, and store it."""
-        st = self._state(conn_id)
-        item_id = _generate_id("item")
-        st.current_item_id = item_id
-        st.content_index = 0
-        st.input_audio_duration_s = 0.0
-        return item_id
-
-    def _current_item_id(self, conn_id: str) -> str:
-        return self._state(conn_id).current_item_id or self._start_item(conn_id)
-
-    def _next_content_index(self, conn_id: str) -> int:
-        """Return the current content index and advance it."""
-        st = self._state(conn_id)
-        idx = st.content_index
-        st.content_index += 1
-        return idx
-
-    def _build_response(
-        self,
-        conn_id: str,
-        status: _ResponseStatus,
-        reason: _StatusReason | None = None,
-    ) -> RealtimeResponse:
-        """Build a fully-populated RealtimeResponse from the current connection state."""
-        st = self._state(conn_id)
-        status_details = None
-        if reason or status in ("completed", "cancelled", "incomplete", "failed"):
-            status_details = RealtimeResponseStatus(type=status, reason=reason)  # type: ignore[arg-type]
-
-        rp = st.current_response_params
-        metadata = rp.metadata if rp and rp.metadata else None
-
-        voice: Optional[str] = None
-        if rp and rp.audio and rp.audio.output and rp.audio.output.voice:
-            voice = str(rp.audio.output.voice)
-        if not voice:
-            audio_cfg = st.runtime_config.session.audio
-            audio_output = audio_cfg.output if audio_cfg is not None else None
-            voice = str(audio_output.voice) if audio_output is not None and audio_output.voice else None
-
-        # Out-of-band responses are not threaded into any conversation: report a null id.
-        conversation_id = None if is_out_of_band(rp) else st.conversation_id
-
-        return RealtimeResponse(
-            id=st.current_response_id,
-            object="realtime.response",
-            status=status,
-            status_details=status_details,
-            audio=Audio(output=AudioOutput(voice=str(voice) if voice else None)),  # type: ignore[arg-type]
-            conversation_id=conversation_id,
-            metadata=metadata,
-            usage=RealtimeResponseUsage(
-                input_tokens=st.response_usage.input_tokens,
-                output_tokens=st.response_usage.output_tokens,
-                total_tokens=st.response_usage.input_tokens + st.response_usage.output_tokens,
-            ),
-        )
-
-    # ── Public handlers ───────────────────────────
-
-    def handle_response_create(self, conn_id: str, event: ResponseCreateEvent) -> ServerEvent | None:
-        """Trigger a response.
-
-        Returns a ``ResponseCreatedEvent`` on success, a ``RealtimeErrorEvent``
-        on failure, or ``None`` if there is no text_prompt_queue.
-        """
-        st = self._state(conn_id)
-        if event.response:
-            if event.response.tool_choice and not isinstance(event.response.tool_choice, str):
-                return self.make_error(
-                    message="Only string tool_choice values are supported for now (auto, required, none).",
-                    _type="tool_choice_not_supported",
-                )
-        if st.in_response:
-            return self.make_error(
-                message="Cannot create response while another response is in progress.",
-                _type="conversation_already_has_active_response",
-            )
-
-        out_of_band = is_out_of_band(event.response)
-
-        # In-band: response.input items are added to the default conversation here so
-        # they appear in history. Out-of-band: leave the default conversation untouched —
-        # the input rides along on the request and seeds a throwaway chat in the LM.
-        if not out_of_band and event.response and event.response.input:
-            for input_item in event.response.input:
-                try:
-                    self._service.conversation._append_item(conn_id, input_item)
-                except ChatItemError as exc:
-                    return self.make_error(message=str(exc), _type="invalid_input_item")
-
-        st.in_response = True
-        st.response_pending = False
-
-        st.current_response_params = event.response
-        st.current_response_id = _generate_id("resp")
-        self._start_item(conn_id)
-
-        cfg = st.runtime_config
-        queue = self._queue(conn_id)
-        if queue:
-            # Out-of-band responses carry no turn identity: a null turn_id makes every
-            # speculative-turn staleness gate treat them as always-latest, so a new user
-            # turn mid-generation can never silently drop their output.
-            queue.put(
-                GenerateResponseRequest(
-                    runtime_config=cfg,
-                    response=event.response,
-                    turn_id=None if out_of_band else st.speculative_user_turn_id,
-                    turn_revision=None if out_of_band else st.speculative_user_turn_revision,
-                    speech_stopped_at_s=None if out_of_band else st.speculative_user_speech_stopped_at_s,
-                )
-            )
-        logger.debug("response.create received, LLM generation triggered")
-        return ResponseCreatedEvent(
-            type="response.created",
-            event_id=self._next_event_id(),
-            response=self._build_response(conn_id, "in_progress"),
-        )
-
-    def handle_response_cancel(self, conn_id: str) -> list[ServerEvent]:
-        """Cancel the in-progress response and re-enable listening."""
-        events = self.finish_response(conn_id, status="cancelled", reason="client_cancelled")
-        should_listen = self._should_listen(conn_id)
-        if should_listen:
-            should_listen.set()
-        logger.info("Response cancelled, listening re-enabled")
-        return events
-
-    def finish_response(
-        self,
-        conn_id: str,
-        status: _ResponseStatus = "completed",
-        reason: _StatusReason | None = None,
-    ) -> list[ServerEvent]:
-        """Close the current response (audio/text done + response done).
-
-        Audio responses emit ``response.output_audio.done`` for any terminal
-        status. Text-only responses emit a single ``response.output_text.done``
-        carrying the full streamed text, but only on ``status="completed"`` —
-        a cancelled or failed text response sends no audio, so it just closes
-        with ``response.done``.
-        """
-        st = self._state(conn_id)
-        events: list[ServerEvent] = []
-        if st.in_response:
-            resp_id, item_id = self._ensure_response(conn_id)
-            if response_wants_audio(st.current_response_params):
-                events.append(
-                    ResponseAudioDoneEvent(
-                        type="response.output_audio.done",
-                        event_id=self._next_event_id(),
-                        content_index=0,
-                        item_id=item_id,
-                        output_index=0,
-                        response_id=resp_id,
-                    )
-                )
-            elif status == "completed" and st.pending_output_text_parts:
-                events.append(
-                    ResponseTextDoneEvent(
-                        type="response.output_text.done",
-                        event_id=self._next_event_id(),
-                        content_index=0,
-                        item_id=item_id,
-                        output_index=0,
-                        response_id=resp_id,
-                        text="".join(st.pending_output_text_parts),
-                    )
-                )
-            events.append(
-                ResponseDoneEvent(
-                    type="response.done",
-                    event_id=self._next_event_id(),
-                    response=self._build_response(conn_id, status, reason),
-                )
-            )
-            self._end_response(conn_id, status)
-        # Apply any client items that arrived mid-generation now that in_response
-        # is cleared and the generation's own write-back has landed. Done outside
-        # the in_response guard so a stray terminal call still drains the buffer.
-        events.extend(self._service.conversation.flush_deferred_items(conn_id))
-        return events
-
-    # ── Pipeline event handlers ───────────────────
-
-    def on_assistant_text(
-        self,
-        conn_id: str,
-        event: AssistantTextEvent,
-        *,
-        wait_for_pending_reopen: bool = True,
-    ) -> list[ServerEvent] | None:
-        """Handle assistant_text: emit transcript and/or tool-call events."""
-        if self._service.speculative_turns:
-            commit_result: bool | None
-            if wait_for_pending_reopen:
-                commit_result = self._service.speculative_turns.commit_if_latest_after_reopen_grace(
-                    event.turn_id,
-                    event.turn_revision,
-                )
-            else:
-                commit_result = self._service.speculative_turns.try_commit_if_latest_after_reopen_grace(
-                    event.turn_id,
-                    event.turn_revision,
-                )
-            if commit_result is None:
-                return None
-            if not commit_result:
-                logger.debug("Dropping stale assistant text for turn=%s rev=%s", event.turn_id, event.turn_revision)
-                return []
-        st = self._state(conn_id)
-        events: list[ServerEvent] = []
-        resp_id, item_id = self._ensure_response(conn_id)
-        st.last_item_id = item_id
-        output_idx = 0
-        if event.text:
-            if response_wants_audio(st.current_response_params):
-                events.append(
-                    ResponseAudioTranscriptDoneEvent(
-                        type="response.output_audio_transcript.done",
-                        event_id=self._next_event_id(),
-                        content_index=0,
-                        item_id=item_id,
-                        output_index=output_idx,
-                        response_id=resp_id,
-                        transcript=event.text,
-                    )
-                )
-            else:
-                # Stream the delta now; the matching response.output_text.done is
-                # emitted once at close in finish_response, carrying the per-chunk
-                # parts collected here, space-joined ("" when there were none).
-                st.pending_output_text_parts.append(event.text)
-                events.append(
-                    ResponseTextDeltaEvent(
-                        type="response.output_text.delta",
-                        event_id=self._next_event_id(),
-                        content_index=0,
-                        item_id=item_id,
-                        output_index=output_idx,
-                        response_id=resp_id,
-                        delta=event.text,
-                    )
-                )
-            output_idx += 1
-        if event.tools:
-            st.response_usage.tool_calls += len(event.tools)
-            for tool in event.tools:
-                events.append(
-                    ResponseFunctionCallArgumentsDoneEvent(
-                        type="response.function_call_arguments.done",
-                        event_id=self._next_event_id(),
-                        call_id=tool.call_id,
-                        name=tool.name,
-                        arguments=tool.arguments,
-                        item_id=item_id,
-                        output_index=output_idx,
-                        response_id=resp_id,
-                    )
-                )
-                output_idx += 1
-        return events

runtime/src/speech_to_speech/api/openai_realtime/handlers/session.py

@@ -1,62 +0,0 @@
-from __future__ import annotations
-
-import logging
-from typing import Optional
-
-from openai.types.realtime import (
-    RealtimeErrorEvent,
-    SessionCreatedEvent,
-    SessionUpdateEvent,
-)
-from openai.types.realtime.realtime_transcription_session_create_request import (
-    RealtimeTranscriptionSessionCreateRequest,
-)
-
-from speech_to_speech.api.openai_realtime.handlers.base import RealtimeBaseHandler
-
-logger = logging.getLogger(__name__)
-
-
-class SessionHandler(RealtimeBaseHandler):
-    """Owns session lifecycle: config updates and session.created events."""
-
-    def handle_session_update(self, conn_id: str, event: SessionUpdateEvent) -> Optional[RealtimeErrorEvent]:
-        """Apply session config changes.
-
-        Only ``RealtimeSessionCreateRequest`` sessions are accepted;
-        ``RealtimeTranscriptionSessionCreateRequest`` sessions not yet supported.
-        Incoming fields are deep-merged into the existing session so that
-        partial updates preserve previously-set values.
-        """
-        s = event.session
-        if s is None:
-            return None
-
-        if isinstance(s, RealtimeTranscriptionSessionCreateRequest):
-            return self.make_error(
-                message="Only 'realtime' session type is supported; transcription sessions are not.",
-                _type="invalid_session_type",
-            )
-
-        model = getattr(s, "model", None)
-        if model is not None:
-            logger.info(f"Session model set to: {model}")
-
-        cfg = self._state(conn_id).runtime_config
-        current = cfg.session
-        if current is None:
-            cfg.session = s
-        else:
-            cfg.apply_session_update(s)
-        logger.info("Session configuration updated")
-        return None
-
-    def build_session_created(self, conn_id: str) -> SessionCreatedEvent:
-        """Build a SessionCreatedEvent populated with the current config."""
-        cfg = self._state(conn_id).runtime_config
-        session = cfg.session
-        return SessionCreatedEvent(
-            type="session.created",
-            event_id=self._next_event_id(),
-            session=session,
-        )

runtime/src/speech_to_speech/api/openai_realtime/pipeline_unit.py

@@ -1,72 +0,0 @@
-import asyncio
-from queue import Queue
-from threading import Event
-from typing import Any, Optional
-
-from pydantic import BaseModel, ConfigDict, Field
-
-from speech_to_speech.api.openai_realtime.service import RealtimeService
-from speech_to_speech.api.openai_realtime.transports import SessionTransport
-from speech_to_speech.pipeline.cancel_scope import CancelScope
-
-
-class SessionState(BaseModel):
-    """Per-client ephemeral state.
-
-    Created when a route handler claims a PipelineUnit (WebSocket accept or
-    WebRTC SDP offer); dropped when the client disconnects. Holding the
-    transport reference, the service session id, and any send-loop scratch
-    (pending_output_item) here ensures these fields share one lifecycle — a
-    stale value can't outlive its session.
-
-    `transport` may briefly be None while a WebRTC claim finishes
-    constructing the session, so the send loop must tolerate a
-    transport-less snapshot.
-
-    `drained` is set by the send loop when SESSION_END travels through the handler
-    chain back to the output queue; the release path awaits it before clearing
-    `PipelineUnit.session`, so a new client cannot claim the unit until in-flight
-    work from this session has fully reset.
-    """
-
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-
-    transport: Optional[SessionTransport] = None
-    session_id: str = ""
-    pending_output_item: Any = None
-    drained: asyncio.Event = Field(default_factory=asyncio.Event)
-    # Wall-clock time when the client disconnected (route handler released its
-    # claim). `None` while the client is still active. Used by /v1/pool to
-    # surface stuck units (handlers haven't finished propagating SESSION_END).
-    released_at: Optional[float] = None
-    # Wall-clock time when the drain wait gave up and quarantined the unit
-    # (SESSION_END_QUARANTINE_TIMEOUT_S elapsed). The unit stays unclaimable —
-    # its handlers may still emit this session's output — until SESSION_END
-    # actually drains. Reported as "stuck" by /v1/pool.
-    quarantined_at: Optional[float] = None
-
-
-class PipelineUnit(BaseModel):
-    """One isolated realtime pipeline.
-
-    Each unit owns its queues, events, RealtimeService, and the chain of handler
-    instances (VAD, STT, transcription notifier, LM, LM output processor, TTS).
-    Lives inside the pool managed by RealtimeServer; the websocket route handler
-    claims a free unit (`session is None`) on `accept` and releases it on disconnect
-    by setting `session` back to None.
-    """
-
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-
-    index: int
-    service: RealtimeService
-    cancel_scope: CancelScope
-    should_listen: Event
-    response_playing: Event
-    input_queue: Queue
-    output_queue: Queue
-    text_output_queue: Queue
-    text_prompt_queue: Queue
-    handlers: list[Any]
-
-    session: Optional[SessionState] = None