File size: 8,335 Bytes
06f9561 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 | """Native answer speech for offline SWA review, concurrent with the body loop.
The camera/action clock is prerecorded, not a realtime hardware benchmark.
PCM availability is measured relative to its triggering packet, including queue
delay. Playback is FIFO, never backdated; rendering preserves any resulting tail.
"""
from concurrent.futures import ThreadPoolExecutor
from dataclasses import replace
from pathlib import Path
import os
import time
import numpy as np
import torch
class IncompleteSpeechError(RuntimeError):
"""A bounded codec generation did not finish; never publish truncated PCM."""
class SwaSpeechWorker:
def __init__(self, model, backbone, output):
from transformers import Qwen2_5OmniForConditionalGeneration
from psi.runtime.speech import SpeechRuntimeConfig
self.model, self.output = model, Path(output)
self.output.mkdir(parents=True, exist_ok=True)
self.config = SpeechRuntimeConfig(
solver=os.environ.get('HFD_SPEECH_SOLVER', 'euler'),
solver_steps=int(os.environ.get('HFD_SPEECH_SOLVER_STEPS', '6')),
codec_chunk_codes=int(os.environ.get('HFD_SPEECH_CODEC_CHUNK_CODES', '36')),
first_chunk_codes=int(os.environ.get('HFD_SPEECH_FIRST_CHUNK_CODES', '36')),
playback_buffer_seconds=float(os.environ.get('HFD_SPEECH_PLAYBACK_BUFFER_SECONDS', '.20')),
).validate()
self.native = Qwen2_5OmniForConditionalGeneration.from_pretrained(
str(backbone), dtype=torch.bfloat16, attn_implementation='flash_attention_2')
# Perception uses the trained HFD model; only the native text backbone
# is needed for the frozen speech interface (not a second AV encoder).
self.native.thinker.audio_tower = None
self.native.thinker.visual = None
self.native.to(model.device).eval().requires_grad_(False)
self.native.token2wav.float()
self.native.voice_thinker = self.native.thinker
self.stream = torch.cuda.Stream(device=model.device)
self.pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix='swa-answer')
self.jobs = []
self.failures = []
def submit(self, cache, *, window, seconds):
from psi.runtime.native_streaming_speech import _clone_selected_cache
submitted = time.monotonic()
answer_model = getattr(self.model, "qa_model", None) or self.model
answer_cache = cache.qa_cache if answer_model is not self.model else cache
if answer_cache is None:
raise RuntimeError("Speech requires the independently encoded QA history")
# Snapshot on the producer stream BEFORE its next audio packet mutates
# DynamicCache. A thread that clones the live cache later is not causal.
snapshot = replace(answer_cache, past_key_values=_clone_selected_cache(
answer_cache.past_key_values, torch.tensor([0], device=self.model.device)),
attention_mask=answer_cache.attention_mask.clone(), qa_cache=None)
ready = torch.cuda.Event()
ready.record(torch.cuda.current_stream(self.model.device))
if len([f for f in self.jobs if not f.done()]) >= 4:
# Backpressure instead of silently losing an answer or unbounded KV.
try:
next(f for f in self.jobs if not f.done()).result()
except IncompleteSpeechError:
pass # Retained in jobs and reported explicitly by finish().
ordinal = len(self.jobs)
self.jobs.append(self.pool.submit(self._generate, snapshot, ready,
ordinal, window, seconds, submitted, answer_model))
def _generate(self, snapshot, ready, ordinal, window, seconds, submitted, answer_model):
from scipy.io import wavfile
from psi.runtime.native_streaming_speech import stream_native_reply, SpeechGeometry
chunks, releases = [], []
def on_pcm(pcm, event):
data = pcm.detach().float().cpu().numpy().copy()
if data.ndim != 1 or not len(data) or not np.isfinite(data).all():
raise RuntimeError('Invalid native answer PCM')
chunks.append(data)
releases.append({'sample_start': int(event['sample_start']),
'sample_end': int(event['sample_end']),
'available_seconds': seconds + time.monotonic() - submitted})
with torch.cuda.device(self.model.device), torch.cuda.stream(self.stream), torch.inference_mode():
self.stream.wait_event(ready)
record = stream_native_reply(self.native, answer_model, snapshot, [],
seed=26091900 + ordinal, on_pcm=on_pcm,
native_text_conditioning=True, solver=self.config.solver,
num_steps=self.config.solver_steps,
geometry=SpeechGeometry(chunk_codes=self.config.codec_chunk_codes),
first_chunk_codes=self.config.first_chunk_codes, fused_flash_kv=True,
parallel_decoder=True)
self.stream.synchronize()
record.pop('branch_tape', None)
if not record.get('native_speech') or not chunks:
raise RuntimeError(f'Native speech produced no audio: {record.get("text")}')
if not record.get('talker_eos'):
raise IncompleteSpeechError('Native speech hit codec limit before EOS; refusing truncated answer')
audio = np.concatenate(chunks)
if len(audio) != record['samples'] or float(np.max(np.abs(audio))) < 1e-5:
raise RuntimeError('Native speech PCM count mismatch or silent answer')
path = self.output / f'answer_{ordinal:03d}_24k.wav'
wavfile.write(path, self.config.sample_rate, audio)
record.update(window=window, seconds=seconds, audio_path=path.name,
sample_rate=self.config.sample_rate, pcm_releases=releases,
playback_buffer_seconds=self.config.playback_buffer_seconds)
print(f'[swa-speech] {record["text"]!r}: {len(audio)/24000:.2f}s -> {path}', flush=True)
return record
def finish(self, *, allow_incomplete=False):
try:
answers = []
self.failures = []
for ordinal, job in enumerate(self.jobs):
try:
answers.append(job.result())
except IncompleteSpeechError as exc:
if not allow_incomplete:
raise
self.failures.append(dict(ordinal=ordinal, error=str(exc)))
print(f'[swa-speech:incomplete] answer {ordinal}: {exc}', flush=True)
return answers
finally:
self.pool.shutdown(wait=True, cancel_futures=True)
def close(self):
self.pool.shutdown(wait=True, cancel_futures=True)
def schedule_speech(answers, sample_rate=24000):
"""Map NEW PCM chunks onto a single speaker FIFO without dropping samples."""
end = 0
segments = []
for answer in answers:
if int(answer.get('sample_rate', sample_rate)) != sample_rate:
raise ValueError('Answer PCM sample rate differs from playback clock')
cursor = 0
last_available = float(answer['seconds'])
for i, chunk in enumerate(answer['pcm_releases']):
if chunk['sample_start'] != cursor or chunk['sample_end'] <= cursor:
raise ValueError('Answer PCM has gaps or duplicated chunks')
available = float(chunk['available_seconds'])
if not np.isfinite(available) or available < last_available:
raise ValueError('Answer audio precedes its causal trigger')
last_available = available
if i == 0:
available += float(answer['playback_buffer_seconds'])
start = max(end, int(np.ceil(available * sample_rate)))
size = chunk['sample_end'] - cursor
segments.append(dict(path=answer['audio_path'], source_start=cursor,
source_end=chunk['sample_end'], start=start, end=start + size))
cursor = chunk['sample_end']
end = start + size
if cursor != int(answer['samples']):
raise ValueError('Answer playback would truncate generated samples')
return segments
|