code-HFD / src /psi /runtime /swa_speech.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
8.34 kB
"""Native answer speech for offline SWA review, concurrent with the body loop.
The camera/action clock is prerecorded, not a realtime hardware benchmark.
PCM availability is measured relative to its triggering packet, including queue
delay. Playback is FIFO, never backdated; rendering preserves any resulting tail.
"""
from concurrent.futures import ThreadPoolExecutor
from dataclasses import replace
from pathlib import Path
import os
import time
import numpy as np
import torch
class IncompleteSpeechError(RuntimeError):
"""A bounded codec generation did not finish; never publish truncated PCM."""
class SwaSpeechWorker:
def __init__(self, model, backbone, output):
from transformers import Qwen2_5OmniForConditionalGeneration
from psi.runtime.speech import SpeechRuntimeConfig
self.model, self.output = model, Path(output)
self.output.mkdir(parents=True, exist_ok=True)
self.config = SpeechRuntimeConfig(
solver=os.environ.get('HFD_SPEECH_SOLVER', 'euler'),
solver_steps=int(os.environ.get('HFD_SPEECH_SOLVER_STEPS', '6')),
codec_chunk_codes=int(os.environ.get('HFD_SPEECH_CODEC_CHUNK_CODES', '36')),
first_chunk_codes=int(os.environ.get('HFD_SPEECH_FIRST_CHUNK_CODES', '36')),
playback_buffer_seconds=float(os.environ.get('HFD_SPEECH_PLAYBACK_BUFFER_SECONDS', '.20')),
).validate()
self.native = Qwen2_5OmniForConditionalGeneration.from_pretrained(
str(backbone), dtype=torch.bfloat16, attn_implementation='flash_attention_2')
# Perception uses the trained HFD model; only the native text backbone
# is needed for the frozen speech interface (not a second AV encoder).
self.native.thinker.audio_tower = None
self.native.thinker.visual = None
self.native.to(model.device).eval().requires_grad_(False)
self.native.token2wav.float()
self.native.voice_thinker = self.native.thinker
self.stream = torch.cuda.Stream(device=model.device)
self.pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix='swa-answer')
self.jobs = []
self.failures = []
def submit(self, cache, *, window, seconds):
from psi.runtime.native_streaming_speech import _clone_selected_cache
submitted = time.monotonic()
answer_model = getattr(self.model, "qa_model", None) or self.model
answer_cache = cache.qa_cache if answer_model is not self.model else cache
if answer_cache is None:
raise RuntimeError("Speech requires the independently encoded QA history")
# Snapshot on the producer stream BEFORE its next audio packet mutates
# DynamicCache. A thread that clones the live cache later is not causal.
snapshot = replace(answer_cache, past_key_values=_clone_selected_cache(
answer_cache.past_key_values, torch.tensor([0], device=self.model.device)),
attention_mask=answer_cache.attention_mask.clone(), qa_cache=None)
ready = torch.cuda.Event()
ready.record(torch.cuda.current_stream(self.model.device))
if len([f for f in self.jobs if not f.done()]) >= 4:
# Backpressure instead of silently losing an answer or unbounded KV.
try:
next(f for f in self.jobs if not f.done()).result()
except IncompleteSpeechError:
pass # Retained in jobs and reported explicitly by finish().
ordinal = len(self.jobs)
self.jobs.append(self.pool.submit(self._generate, snapshot, ready,
ordinal, window, seconds, submitted, answer_model))
def _generate(self, snapshot, ready, ordinal, window, seconds, submitted, answer_model):
from scipy.io import wavfile
from psi.runtime.native_streaming_speech import stream_native_reply, SpeechGeometry
chunks, releases = [], []
def on_pcm(pcm, event):
data = pcm.detach().float().cpu().numpy().copy()
if data.ndim != 1 or not len(data) or not np.isfinite(data).all():
raise RuntimeError('Invalid native answer PCM')
chunks.append(data)
releases.append({'sample_start': int(event['sample_start']),
'sample_end': int(event['sample_end']),
'available_seconds': seconds + time.monotonic() - submitted})
with torch.cuda.device(self.model.device), torch.cuda.stream(self.stream), torch.inference_mode():
self.stream.wait_event(ready)
record = stream_native_reply(self.native, answer_model, snapshot, [],
seed=26091900 + ordinal, on_pcm=on_pcm,
native_text_conditioning=True, solver=self.config.solver,
num_steps=self.config.solver_steps,
geometry=SpeechGeometry(chunk_codes=self.config.codec_chunk_codes),
first_chunk_codes=self.config.first_chunk_codes, fused_flash_kv=True,
parallel_decoder=True)
self.stream.synchronize()
record.pop('branch_tape', None)
if not record.get('native_speech') or not chunks:
raise RuntimeError(f'Native speech produced no audio: {record.get("text")}')
if not record.get('talker_eos'):
raise IncompleteSpeechError('Native speech hit codec limit before EOS; refusing truncated answer')
audio = np.concatenate(chunks)
if len(audio) != record['samples'] or float(np.max(np.abs(audio))) < 1e-5:
raise RuntimeError('Native speech PCM count mismatch or silent answer')
path = self.output / f'answer_{ordinal:03d}_24k.wav'
wavfile.write(path, self.config.sample_rate, audio)
record.update(window=window, seconds=seconds, audio_path=path.name,
sample_rate=self.config.sample_rate, pcm_releases=releases,
playback_buffer_seconds=self.config.playback_buffer_seconds)
print(f'[swa-speech] {record["text"]!r}: {len(audio)/24000:.2f}s -> {path}', flush=True)
return record
def finish(self, *, allow_incomplete=False):
try:
answers = []
self.failures = []
for ordinal, job in enumerate(self.jobs):
try:
answers.append(job.result())
except IncompleteSpeechError as exc:
if not allow_incomplete:
raise
self.failures.append(dict(ordinal=ordinal, error=str(exc)))
print(f'[swa-speech:incomplete] answer {ordinal}: {exc}', flush=True)
return answers
finally:
self.pool.shutdown(wait=True, cancel_futures=True)
def close(self):
self.pool.shutdown(wait=True, cancel_futures=True)
def schedule_speech(answers, sample_rate=24000):
"""Map NEW PCM chunks onto a single speaker FIFO without dropping samples."""
end = 0
segments = []
for answer in answers:
if int(answer.get('sample_rate', sample_rate)) != sample_rate:
raise ValueError('Answer PCM sample rate differs from playback clock')
cursor = 0
last_available = float(answer['seconds'])
for i, chunk in enumerate(answer['pcm_releases']):
if chunk['sample_start'] != cursor or chunk['sample_end'] <= cursor:
raise ValueError('Answer PCM has gaps or duplicated chunks')
available = float(chunk['available_seconds'])
if not np.isfinite(available) or available < last_available:
raise ValueError('Answer audio precedes its causal trigger')
last_available = available
if i == 0:
available += float(answer['playback_buffer_seconds'])
start = max(end, int(np.ceil(available * sample_rate)))
size = chunk['sample_end'] - cursor
segments.append(dict(path=answer['audio_path'], source_start=cursor,
source_end=chunk['sample_end'], start=start, end=start + size))
cursor = chunk['sample_end']
end = start + size
if cursor != int(answer['samples']):
raise ValueError('Answer playback would truncate generated samples')
return segments