code-HFD / src /psi /runtime /speech.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
12.3 kB
"""Causal speech runtime contracts used by the verified SWA implementation.
This module contains the state/queue boundary that was missing from the
formal HFD tree. It does not import Qwen, Talker, FlashAttention, or an audio
device at module import time. Consequently the established action-only
training path keeps the exact same model, KV, and loss math. A deployment
adapter can attach the validated native Thinker/Talker/Token2Wav implementation
to the explicit state slots below.
The defaults are the retained speech candidate: frozen native Thinker
incremental KV, fused Talker KV, Euler-6 Token2Wav, 36-code chunks, and a
0.20-second software playback buffer. A result is accepted only when its
session, utterance, and cancellation epoch still match; late cancelled work
cannot leak into a newer utterance.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from queue import Empty, Full, Queue
from threading import Lock
from time import monotonic
from typing import Any, Iterable
@dataclass(frozen=True)
class SpeechRuntimeConfig:
"""Configuration for causal text->codec->PCM generation.
``codec_chunk_codes`` and ``first_chunk_codes`` are codec-token counts,
not seconds. The playback buffer is a start threshold; it does not
reset any model cache.
"""
solver: str = "euler"
solver_steps: int = 6
codec_chunk_codes: int = 36
first_chunk_codes: int = 36
playback_buffer_seconds: float = 0.20
fused_flash_kv: bool = True
max_pending_chunks: int = 2
sample_rate: int = 24_000
action_packet_seconds: float = 0.40
def validate(self) -> "SpeechRuntimeConfig":
if self.solver not in {"euler", "rk4"}:
raise ValueError(f"Unsupported speech solver: {self.solver!r}")
if self.solver_steps < 2:
raise ValueError("solver_steps must be at least two")
if self.codec_chunk_codes <= 0 or self.first_chunk_codes <= 0:
raise ValueError("codec chunk sizes must be positive")
if self.playback_buffer_seconds < 0:
raise ValueError("playback_buffer_seconds must be non-negative")
if self.max_pending_chunks < 1:
raise ValueError("max_pending_chunks must be positive")
if self.sample_rate <= 0 or self.action_packet_seconds <= 0:
raise ValueError("sample rate and action packet duration must be positive")
return self
@property
def playback_buffer_samples(self) -> int:
return int(round(self.sample_rate * self.playback_buffer_seconds))
@property
def action_packet_samples(self) -> int:
return int(round(self.sample_rate * self.action_packet_seconds))
@dataclass
class SpeechGenerationContext:
"""Mutable per-utterance state shared by producer and playback workers.
The three positions are intentionally separate: generated text/codec may
be ahead of queued PCM, and queued PCM may be ahead of physically played
samples. ``native_thinker_kv``, ``talker_kv`` and ``token2wav_state`` are
opaque handles owned by the adapter and are never copied into the action
model's source KV.
"""
session_id: str
utterance_id: int = 0
cancel_epoch: int = 0
generated_text_pos: int = 0
generated_codec_pos: int = 0
queued_pcm_samples: int = 0
played_pcm_samples: int = 0
native_thinker_kv: Any = None
talker_kv: Any = None
token2wav_state: Any = None
closed: bool = False
cancel_reason: str | None = None
_lock: Lock = field(default_factory=Lock, repr=False, compare=False)
def begin_utterance(self) -> int:
with self._lock:
if self.closed:
raise RuntimeError("Cannot begin an utterance on a closed session")
self.utterance_id += 1
self.cancel_reason = None
self.generated_text_pos = 0
self.generated_codec_pos = 0
self.queued_pcm_samples = 0
self.played_pcm_samples = 0
self.native_thinker_kv = None
self.talker_kv = None
self.token2wav_state = None
return self.utterance_id
def cancel(self, reason: str = "cancelled") -> int:
with self._lock:
if self.closed:
return self.cancel_epoch
self.cancel_epoch += 1
self.cancel_reason = str(reason)
return self.cancel_epoch
def close(self) -> None:
with self._lock:
self.closed = True
self.cancel_epoch += 1
def accepts(self, *, session_id: str, utterance_id: int, cancel_epoch: int) -> bool:
with self._lock:
return (
not self.closed
and str(session_id) == self.session_id
and int(utterance_id) == self.utterance_id
and int(cancel_epoch) == self.cancel_epoch
)
def mark_generated_text(self, count: int = 1) -> None:
if count < 0:
raise ValueError("count must be non-negative")
with self._lock:
self.generated_text_pos += int(count)
def mark_generated_codec(self, count: int = 1) -> None:
if count < 0:
raise ValueError("count must be non-negative")
with self._lock:
self.generated_codec_pos += int(count)
def mark_pcm_queued(self, samples: int) -> None:
if samples < 0:
raise ValueError("samples must be non-negative")
with self._lock:
self.queued_pcm_samples += int(samples)
def mark_pcm_played(self, samples: int) -> None:
if samples < 0:
raise ValueError("samples must be non-negative")
with self._lock:
self.played_pcm_samples += int(samples)
def snapshot(self) -> dict[str, Any]:
with self._lock:
return {
"session_id": self.session_id,
"utterance_id": self.utterance_id,
"cancel_epoch": self.cancel_epoch,
"generated_text_pos": self.generated_text_pos,
"generated_codec_pos": self.generated_codec_pos,
"queued_pcm_samples": self.queued_pcm_samples,
"played_pcm_samples": self.played_pcm_samples,
"closed": self.closed,
"cancel_reason": self.cancel_reason,
}
@dataclass(frozen=True)
class SpeechChunk:
"""One ordered PCM result with its causal ownership metadata."""
pcm: Any
session_id: str
utterance_id: int
cancel_epoch: int
sample_start: int = 0
final: bool = False
created_monotonic: float = field(default_factory=monotonic)
class SpeechResultGate:
"""Reject stale worker results without killing a worker thread."""
def __init__(self, context: SpeechGenerationContext) -> None:
self.context = context
def accepts(self, chunk: SpeechChunk) -> bool:
return self.context.accepts(
session_id=chunk.session_id,
utterance_id=chunk.utterance_id,
cancel_epoch=chunk.cancel_epoch,
)
class BoundedSpeechQueue:
"""Bounded FIFO with cancellation epochs and explicit backpressure.
``submit`` returns ``False`` for stale results. It never drops a current
chunk silently: a full queue raises ``queue.Full`` so the producer can
apply backpressure and preserve audio order.
"""
def __init__(self, context: SpeechGenerationContext, *, maxsize: int = 2) -> None:
if maxsize < 1:
raise ValueError("maxsize must be positive")
self.context = context
self.queue: Queue[SpeechChunk] = Queue(maxsize=maxsize)
self.gate = SpeechResultGate(context)
self._closed = False
self._lock = Lock()
def submit(self, chunk: SpeechChunk, *, block: bool = True, timeout: float | None = None) -> bool:
if not self.gate.accepts(chunk):
return False
with self._lock:
if self._closed:
raise RuntimeError("Speech queue is closed")
self.queue.put(chunk, block=block, timeout=timeout)
if self.gate.accepts(chunk):
samples = getattr(chunk.pcm, "numel", lambda: len(chunk.pcm))()
self.context.mark_pcm_queued(int(samples))
return True
# A cancellation can race with put(). Remove only this stale item;
# current items remain ordered and visible to the consumer.
self._remove_identity(chunk)
return False
def get(self, *, block: bool = True, timeout: float | None = None) -> SpeechChunk:
return self.queue.get(block=block, timeout=timeout)
def task_done(self) -> None:
self.queue.task_done()
def cancel_pending(self) -> int:
"""Remove queued chunks from older epochs and return the count."""
kept: list[SpeechChunk] = []
removed = 0
while True:
try:
chunk = self.queue.get_nowait()
except Empty:
break
if self.gate.accepts(chunk):
kept.append(chunk)
else:
removed += 1
self.queue.task_done()
for chunk in kept:
self.queue.put_nowait(chunk)
return removed
def close(self) -> None:
with self._lock:
self._closed = True
def qsize(self) -> int:
return self.queue.qsize()
def _remove_identity(self, target: SpeechChunk) -> None:
kept: list[SpeechChunk] = []
while True:
try:
chunk = self.queue.get_nowait()
except Empty:
break
self.queue.task_done()
if chunk is not target:
kept.append(chunk)
for chunk in kept:
self.queue.put_nowait(chunk)
class PCMClockPackets:
"""Reblock completed PCM onto the 0.4-second action clock."""
def __init__(self, samples_per_packet: int = 9_600) -> None:
if samples_per_packet <= 0:
raise ValueError("samples_per_packet must be positive")
self.samples_per_packet = int(samples_per_packet)
self._buffer: list[Any] = []
self.received_samples = 0
self.emitted_samples = 0
self.closed = False
def push(self, pcm: Any) -> list[Any]:
if self.closed:
raise RuntimeError("PCM clock is closed")
count = int(getattr(pcm, "numel", lambda: len(pcm))())
if count <= 0:
return []
self._buffer.append(pcm)
self.received_samples += count
out: list[Any] = []
while self._buffer and self._buffered_count() >= self.samples_per_packet:
out.append(self._take(self.samples_per_packet))
return out
def finish(self) -> list[Any]:
if self.closed:
raise RuntimeError("PCM clock is already closed")
self.closed = True
if self._buffered_count() <= 0:
return []
return [self._take(self._buffered_count())]
def _buffered_count(self) -> int:
return sum(int(getattr(value, "numel", lambda: len(value))()) for value in self._buffer)
def _take(self, count: int) -> Any:
first = self._buffer[0]
if hasattr(first, "numel"):
import torch
pieces = []
remaining = int(count)
while remaining:
value = self._buffer[0]
take = min(remaining, int(value.numel()))
pieces.append(value[:take])
remaining -= take
if take == int(value.numel()):
self._buffer.pop(0)
else:
self._buffer[0] = value[take:]
result = torch.cat(pieces).float()
else:
pieces = []
remaining = int(count)
while remaining:
value = self._buffer[0]
take = min(remaining, len(value))
pieces.extend(value[:take])
remaining -= take
if take == len(value):
self._buffer.pop(0)
else:
self._buffer[0] = value[take:]
result = pieces
self.emitted_samples += int(count)
return result