Download src/psi/runtime/swa_speech.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 8.34 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/runtime/swa_speech.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/runtime/swa_speech.py
-
curl -L -o swa_speech.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/runtime/swa_speech.py
8.34 kB
| """Native answer speech for offline SWA review, concurrent with the body loop. | |
| The camera/action clock is prerecorded, not a realtime hardware benchmark. | |
| PCM availability is measured relative to its triggering packet, including queue | |
| delay. Playback is FIFO, never backdated; rendering preserves any resulting tail. | |
| """ | |
| from concurrent.futures import ThreadPoolExecutor | |
| from dataclasses import replace | |
| from pathlib import Path | |
| import os | |
| import time | |
| import numpy as np | |
| import torch | |
| class IncompleteSpeechError(RuntimeError): | |
| """A bounded codec generation did not finish; never publish truncated PCM.""" | |
| class SwaSpeechWorker: | |
| def __init__(self, model, backbone, output): | |
| from transformers import Qwen2_5OmniForConditionalGeneration | |
| from psi.runtime.speech import SpeechRuntimeConfig | |
| self.model, self.output = model, Path(output) | |
| self.output.mkdir(parents=True, exist_ok=True) | |
| self.config = SpeechRuntimeConfig( | |
| solver=os.environ.get('HFD_SPEECH_SOLVER', 'euler'), | |
| solver_steps=int(os.environ.get('HFD_SPEECH_SOLVER_STEPS', '6')), | |
| codec_chunk_codes=int(os.environ.get('HFD_SPEECH_CODEC_CHUNK_CODES', '36')), | |
| first_chunk_codes=int(os.environ.get('HFD_SPEECH_FIRST_CHUNK_CODES', '36')), | |
| playback_buffer_seconds=float(os.environ.get('HFD_SPEECH_PLAYBACK_BUFFER_SECONDS', '.20')), | |
| ).validate() | |
| self.native = Qwen2_5OmniForConditionalGeneration.from_pretrained( | |
| str(backbone), dtype=torch.bfloat16, attn_implementation='flash_attention_2') | |
| # Perception uses the trained HFD model; only the native text backbone | |
| # is needed for the frozen speech interface (not a second AV encoder). | |
| self.native.thinker.audio_tower = None | |
| self.native.thinker.visual = None | |
| self.native.to(model.device).eval().requires_grad_(False) | |
| self.native.token2wav.float() | |
| self.native.voice_thinker = self.native.thinker | |
| self.stream = torch.cuda.Stream(device=model.device) | |
| self.pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix='swa-answer') | |
| self.jobs = [] | |
| self.failures = [] | |
| def submit(self, cache, *, window, seconds): | |
| from psi.runtime.native_streaming_speech import _clone_selected_cache | |
| submitted = time.monotonic() | |
| answer_model = getattr(self.model, "qa_model", None) or self.model | |
| answer_cache = cache.qa_cache if answer_model is not self.model else cache | |
| if answer_cache is None: | |
| raise RuntimeError("Speech requires the independently encoded QA history") | |
| # Snapshot on the producer stream BEFORE its next audio packet mutates | |
| # DynamicCache. A thread that clones the live cache later is not causal. | |
| snapshot = replace(answer_cache, past_key_values=_clone_selected_cache( | |
| answer_cache.past_key_values, torch.tensor([0], device=self.model.device)), | |
| attention_mask=answer_cache.attention_mask.clone(), qa_cache=None) | |
| ready = torch.cuda.Event() | |
| ready.record(torch.cuda.current_stream(self.model.device)) | |
| if len([f for f in self.jobs if not f.done()]) >= 4: | |
| # Backpressure instead of silently losing an answer or unbounded KV. | |
| try: | |
| next(f for f in self.jobs if not f.done()).result() | |
| except IncompleteSpeechError: | |
| pass # Retained in jobs and reported explicitly by finish(). | |
| ordinal = len(self.jobs) | |
| self.jobs.append(self.pool.submit(self._generate, snapshot, ready, | |
| ordinal, window, seconds, submitted, answer_model)) | |
| def _generate(self, snapshot, ready, ordinal, window, seconds, submitted, answer_model): | |
| from scipy.io import wavfile | |
| from psi.runtime.native_streaming_speech import stream_native_reply, SpeechGeometry | |
| chunks, releases = [], [] | |
| def on_pcm(pcm, event): | |
| data = pcm.detach().float().cpu().numpy().copy() | |
| if data.ndim != 1 or not len(data) or not np.isfinite(data).all(): | |
| raise RuntimeError('Invalid native answer PCM') | |
| chunks.append(data) | |
| releases.append({'sample_start': int(event['sample_start']), | |
| 'sample_end': int(event['sample_end']), | |
| 'available_seconds': seconds + time.monotonic() - submitted}) | |
| with torch.cuda.device(self.model.device), torch.cuda.stream(self.stream), torch.inference_mode(): | |
| self.stream.wait_event(ready) | |
| record = stream_native_reply(self.native, answer_model, snapshot, [], | |
| seed=26091900 + ordinal, on_pcm=on_pcm, | |
| native_text_conditioning=True, solver=self.config.solver, | |
| num_steps=self.config.solver_steps, | |
| geometry=SpeechGeometry(chunk_codes=self.config.codec_chunk_codes), | |
| first_chunk_codes=self.config.first_chunk_codes, fused_flash_kv=True, | |
| parallel_decoder=True) | |
| self.stream.synchronize() | |
| record.pop('branch_tape', None) | |
| if not record.get('native_speech') or not chunks: | |
| raise RuntimeError(f'Native speech produced no audio: {record.get("text")}') | |
| if not record.get('talker_eos'): | |
| raise IncompleteSpeechError('Native speech hit codec limit before EOS; refusing truncated answer') | |
| audio = np.concatenate(chunks) | |
| if len(audio) != record['samples'] or float(np.max(np.abs(audio))) < 1e-5: | |
| raise RuntimeError('Native speech PCM count mismatch or silent answer') | |
| path = self.output / f'answer_{ordinal:03d}_24k.wav' | |
| wavfile.write(path, self.config.sample_rate, audio) | |
| record.update(window=window, seconds=seconds, audio_path=path.name, | |
| sample_rate=self.config.sample_rate, pcm_releases=releases, | |
| playback_buffer_seconds=self.config.playback_buffer_seconds) | |
| print(f'[swa-speech] {record["text"]!r}: {len(audio)/24000:.2f}s -> {path}', flush=True) | |
| return record | |
| def finish(self, *, allow_incomplete=False): | |
| try: | |
| answers = [] | |
| self.failures = [] | |
| for ordinal, job in enumerate(self.jobs): | |
| try: | |
| answers.append(job.result()) | |
| except IncompleteSpeechError as exc: | |
| if not allow_incomplete: | |
| raise | |
| self.failures.append(dict(ordinal=ordinal, error=str(exc))) | |
| print(f'[swa-speech:incomplete] answer {ordinal}: {exc}', flush=True) | |
| return answers | |
| finally: | |
| self.pool.shutdown(wait=True, cancel_futures=True) | |
| def close(self): | |
| self.pool.shutdown(wait=True, cancel_futures=True) | |
| def schedule_speech(answers, sample_rate=24000): | |
| """Map NEW PCM chunks onto a single speaker FIFO without dropping samples.""" | |
| end = 0 | |
| segments = [] | |
| for answer in answers: | |
| if int(answer.get('sample_rate', sample_rate)) != sample_rate: | |
| raise ValueError('Answer PCM sample rate differs from playback clock') | |
| cursor = 0 | |
| last_available = float(answer['seconds']) | |
| for i, chunk in enumerate(answer['pcm_releases']): | |
| if chunk['sample_start'] != cursor or chunk['sample_end'] <= cursor: | |
| raise ValueError('Answer PCM has gaps or duplicated chunks') | |
| available = float(chunk['available_seconds']) | |
| if not np.isfinite(available) or available < last_available: | |
| raise ValueError('Answer audio precedes its causal trigger') | |
| last_available = available | |
| if i == 0: | |
| available += float(answer['playback_buffer_seconds']) | |
| start = max(end, int(np.ceil(available * sample_rate))) | |
| size = chunk['sample_end'] - cursor | |
| segments.append(dict(path=answer['audio_path'], source_start=cursor, | |
| source_end=chunk['sample_end'], start=start, end=start + size)) | |
| cursor = chunk['sample_end'] | |
| end = start + size | |
| if cursor != int(answer['samples']): | |
| raise ValueError('Answer playback would truncate generated samples') | |
| return segments | |