"""Native answer speech for offline SWA review, concurrent with the body loop. The camera/action clock is prerecorded, not a realtime hardware benchmark. PCM availability is measured relative to its triggering packet, including queue delay. Playback is FIFO, never backdated; rendering preserves any resulting tail. """ from concurrent.futures import ThreadPoolExecutor from dataclasses import replace from pathlib import Path import os import time import numpy as np import torch class IncompleteSpeechError(RuntimeError): """A bounded codec generation did not finish; never publish truncated PCM.""" class SwaSpeechWorker: def __init__(self, model, backbone, output): from transformers import Qwen2_5OmniForConditionalGeneration from psi.runtime.speech import SpeechRuntimeConfig self.model, self.output = model, Path(output) self.output.mkdir(parents=True, exist_ok=True) self.config = SpeechRuntimeConfig( solver=os.environ.get('HFD_SPEECH_SOLVER', 'euler'), solver_steps=int(os.environ.get('HFD_SPEECH_SOLVER_STEPS', '6')), codec_chunk_codes=int(os.environ.get('HFD_SPEECH_CODEC_CHUNK_CODES', '36')), first_chunk_codes=int(os.environ.get('HFD_SPEECH_FIRST_CHUNK_CODES', '36')), playback_buffer_seconds=float(os.environ.get('HFD_SPEECH_PLAYBACK_BUFFER_SECONDS', '.20')), ).validate() self.native = Qwen2_5OmniForConditionalGeneration.from_pretrained( str(backbone), dtype=torch.bfloat16, attn_implementation='flash_attention_2') # Perception uses the trained HFD model; only the native text backbone # is needed for the frozen speech interface (not a second AV encoder). self.native.thinker.audio_tower = None self.native.thinker.visual = None self.native.to(model.device).eval().requires_grad_(False) self.native.token2wav.float() self.native.voice_thinker = self.native.thinker self.stream = torch.cuda.Stream(device=model.device) self.pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix='swa-answer') self.jobs = [] self.failures = [] def submit(self, cache, *, window, seconds): from psi.runtime.native_streaming_speech import _clone_selected_cache submitted = time.monotonic() answer_model = getattr(self.model, "qa_model", None) or self.model answer_cache = cache.qa_cache if answer_model is not self.model else cache if answer_cache is None: raise RuntimeError("Speech requires the independently encoded QA history") # Snapshot on the producer stream BEFORE its next audio packet mutates # DynamicCache. A thread that clones the live cache later is not causal. snapshot = replace(answer_cache, past_key_values=_clone_selected_cache( answer_cache.past_key_values, torch.tensor([0], device=self.model.device)), attention_mask=answer_cache.attention_mask.clone(), qa_cache=None) ready = torch.cuda.Event() ready.record(torch.cuda.current_stream(self.model.device)) if len([f for f in self.jobs if not f.done()]) >= 4: # Backpressure instead of silently losing an answer or unbounded KV. try: next(f for f in self.jobs if not f.done()).result() except IncompleteSpeechError: pass # Retained in jobs and reported explicitly by finish(). ordinal = len(self.jobs) self.jobs.append(self.pool.submit(self._generate, snapshot, ready, ordinal, window, seconds, submitted, answer_model)) def _generate(self, snapshot, ready, ordinal, window, seconds, submitted, answer_model): from scipy.io import wavfile from psi.runtime.native_streaming_speech import stream_native_reply, SpeechGeometry chunks, releases = [], [] def on_pcm(pcm, event): data = pcm.detach().float().cpu().numpy().copy() if data.ndim != 1 or not len(data) or not np.isfinite(data).all(): raise RuntimeError('Invalid native answer PCM') chunks.append(data) releases.append({'sample_start': int(event['sample_start']), 'sample_end': int(event['sample_end']), 'available_seconds': seconds + time.monotonic() - submitted}) with torch.cuda.device(self.model.device), torch.cuda.stream(self.stream), torch.inference_mode(): self.stream.wait_event(ready) record = stream_native_reply(self.native, answer_model, snapshot, [], seed=26091900 + ordinal, on_pcm=on_pcm, native_text_conditioning=True, solver=self.config.solver, num_steps=self.config.solver_steps, geometry=SpeechGeometry(chunk_codes=self.config.codec_chunk_codes), first_chunk_codes=self.config.first_chunk_codes, fused_flash_kv=True, parallel_decoder=True) self.stream.synchronize() record.pop('branch_tape', None) if not record.get('native_speech') or not chunks: raise RuntimeError(f'Native speech produced no audio: {record.get("text")}') if not record.get('talker_eos'): raise IncompleteSpeechError('Native speech hit codec limit before EOS; refusing truncated answer') audio = np.concatenate(chunks) if len(audio) != record['samples'] or float(np.max(np.abs(audio))) < 1e-5: raise RuntimeError('Native speech PCM count mismatch or silent answer') path = self.output / f'answer_{ordinal:03d}_24k.wav' wavfile.write(path, self.config.sample_rate, audio) record.update(window=window, seconds=seconds, audio_path=path.name, sample_rate=self.config.sample_rate, pcm_releases=releases, playback_buffer_seconds=self.config.playback_buffer_seconds) print(f'[swa-speech] {record["text"]!r}: {len(audio)/24000:.2f}s -> {path}', flush=True) return record def finish(self, *, allow_incomplete=False): try: answers = [] self.failures = [] for ordinal, job in enumerate(self.jobs): try: answers.append(job.result()) except IncompleteSpeechError as exc: if not allow_incomplete: raise self.failures.append(dict(ordinal=ordinal, error=str(exc))) print(f'[swa-speech:incomplete] answer {ordinal}: {exc}', flush=True) return answers finally: self.pool.shutdown(wait=True, cancel_futures=True) def close(self): self.pool.shutdown(wait=True, cancel_futures=True) def schedule_speech(answers, sample_rate=24000): """Map NEW PCM chunks onto a single speaker FIFO without dropping samples.""" end = 0 segments = [] for answer in answers: if int(answer.get('sample_rate', sample_rate)) != sample_rate: raise ValueError('Answer PCM sample rate differs from playback clock') cursor = 0 last_available = float(answer['seconds']) for i, chunk in enumerate(answer['pcm_releases']): if chunk['sample_start'] != cursor or chunk['sample_end'] <= cursor: raise ValueError('Answer PCM has gaps or duplicated chunks') available = float(chunk['available_seconds']) if not np.isfinite(available) or available < last_available: raise ValueError('Answer audio precedes its causal trigger') last_available = available if i == 0: available += float(answer['playback_buffer_seconds']) start = max(end, int(np.ceil(available * sample_rate))) size = chunk['sample_end'] - cursor segments.append(dict(path=answer['audio_path'], source_start=cursor, source_end=chunk['sample_end'], start=start, end=start + size)) cursor = chunk['sample_end'] end = start + size if cursor != int(answer['samples']): raise ValueError('Answer playback would truncate generated samples') return segments