File size: 8,335 Bytes
06f9561
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
"""Native answer speech for offline SWA review, concurrent with the body loop.

The camera/action clock is prerecorded, not a realtime hardware benchmark.
PCM availability is measured relative to its triggering packet, including queue
delay. Playback is FIFO, never backdated; rendering preserves any resulting tail.
"""
from concurrent.futures import ThreadPoolExecutor
from dataclasses import replace
from pathlib import Path
import os
import time

import numpy as np
import torch


class IncompleteSpeechError(RuntimeError):
    """A bounded codec generation did not finish; never publish truncated PCM."""


class SwaSpeechWorker:
    def __init__(self, model, backbone, output):
        from transformers import Qwen2_5OmniForConditionalGeneration
        from psi.runtime.speech import SpeechRuntimeConfig
        self.model, self.output = model, Path(output)
        self.output.mkdir(parents=True, exist_ok=True)
        self.config = SpeechRuntimeConfig(
            solver=os.environ.get('HFD_SPEECH_SOLVER', 'euler'),
            solver_steps=int(os.environ.get('HFD_SPEECH_SOLVER_STEPS', '6')),
            codec_chunk_codes=int(os.environ.get('HFD_SPEECH_CODEC_CHUNK_CODES', '36')),
            first_chunk_codes=int(os.environ.get('HFD_SPEECH_FIRST_CHUNK_CODES', '36')),
            playback_buffer_seconds=float(os.environ.get('HFD_SPEECH_PLAYBACK_BUFFER_SECONDS', '.20')),
        ).validate()
        self.native = Qwen2_5OmniForConditionalGeneration.from_pretrained(
            str(backbone), dtype=torch.bfloat16, attn_implementation='flash_attention_2')
        # Perception uses the trained HFD model; only the native text backbone
        # is needed for the frozen speech interface (not a second AV encoder).
        self.native.thinker.audio_tower = None
        self.native.thinker.visual = None
        self.native.to(model.device).eval().requires_grad_(False)
        self.native.token2wav.float()
        self.native.voice_thinker = self.native.thinker
        self.stream = torch.cuda.Stream(device=model.device)
        self.pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix='swa-answer')
        self.jobs = []
        self.failures = []

    def submit(self, cache, *, window, seconds):
        from psi.runtime.native_streaming_speech import _clone_selected_cache
        submitted = time.monotonic()
        answer_model = getattr(self.model, "qa_model", None) or self.model
        answer_cache = cache.qa_cache if answer_model is not self.model else cache
        if answer_cache is None:
            raise RuntimeError("Speech requires the independently encoded QA history")
        # Snapshot on the producer stream BEFORE its next audio packet mutates
        # DynamicCache. A thread that clones the live cache later is not causal.
        snapshot = replace(answer_cache, past_key_values=_clone_selected_cache(
            answer_cache.past_key_values, torch.tensor([0], device=self.model.device)),
            attention_mask=answer_cache.attention_mask.clone(), qa_cache=None)
        ready = torch.cuda.Event()
        ready.record(torch.cuda.current_stream(self.model.device))
        if len([f for f in self.jobs if not f.done()]) >= 4:
            # Backpressure instead of silently losing an answer or unbounded KV.
            try:
                next(f for f in self.jobs if not f.done()).result()
            except IncompleteSpeechError:
                pass  # Retained in jobs and reported explicitly by finish().
        ordinal = len(self.jobs)
        self.jobs.append(self.pool.submit(self._generate, snapshot, ready,
                                         ordinal, window, seconds, submitted, answer_model))

    def _generate(self, snapshot, ready, ordinal, window, seconds, submitted, answer_model):
        from scipy.io import wavfile
        from psi.runtime.native_streaming_speech import stream_native_reply, SpeechGeometry
        chunks, releases = [], []
        def on_pcm(pcm, event):
            data = pcm.detach().float().cpu().numpy().copy()
            if data.ndim != 1 or not len(data) or not np.isfinite(data).all():
                raise RuntimeError('Invalid native answer PCM')
            chunks.append(data)
            releases.append({'sample_start': int(event['sample_start']),
                             'sample_end': int(event['sample_end']),
                             'available_seconds': seconds + time.monotonic() - submitted})
        with torch.cuda.device(self.model.device), torch.cuda.stream(self.stream), torch.inference_mode():
            self.stream.wait_event(ready)
            record = stream_native_reply(self.native, answer_model, snapshot, [],
                seed=26091900 + ordinal, on_pcm=on_pcm,
                native_text_conditioning=True, solver=self.config.solver,
                num_steps=self.config.solver_steps,
                geometry=SpeechGeometry(chunk_codes=self.config.codec_chunk_codes),
                first_chunk_codes=self.config.first_chunk_codes, fused_flash_kv=True,
                parallel_decoder=True)
            self.stream.synchronize()
        record.pop('branch_tape', None)
        if not record.get('native_speech') or not chunks:
            raise RuntimeError(f'Native speech produced no audio: {record.get("text")}')
        if not record.get('talker_eos'):
            raise IncompleteSpeechError('Native speech hit codec limit before EOS; refusing truncated answer')
        audio = np.concatenate(chunks)
        if len(audio) != record['samples'] or float(np.max(np.abs(audio))) < 1e-5:
            raise RuntimeError('Native speech PCM count mismatch or silent answer')
        path = self.output / f'answer_{ordinal:03d}_24k.wav'
        wavfile.write(path, self.config.sample_rate, audio)
        record.update(window=window, seconds=seconds, audio_path=path.name,
                      sample_rate=self.config.sample_rate, pcm_releases=releases,
                      playback_buffer_seconds=self.config.playback_buffer_seconds)
        print(f'[swa-speech] {record["text"]!r}: {len(audio)/24000:.2f}s -> {path}', flush=True)
        return record

    def finish(self, *, allow_incomplete=False):
        try:
            answers = []
            self.failures = []
            for ordinal, job in enumerate(self.jobs):
                try:
                    answers.append(job.result())
                except IncompleteSpeechError as exc:
                    if not allow_incomplete:
                        raise
                    self.failures.append(dict(ordinal=ordinal, error=str(exc)))
                    print(f'[swa-speech:incomplete] answer {ordinal}: {exc}', flush=True)
            return answers
        finally:
            self.pool.shutdown(wait=True, cancel_futures=True)

    def close(self):
        self.pool.shutdown(wait=True, cancel_futures=True)


def schedule_speech(answers, sample_rate=24000):
    """Map NEW PCM chunks onto a single speaker FIFO without dropping samples."""
    end = 0
    segments = []
    for answer in answers:
        if int(answer.get('sample_rate', sample_rate)) != sample_rate:
            raise ValueError('Answer PCM sample rate differs from playback clock')
        cursor = 0
        last_available = float(answer['seconds'])
        for i, chunk in enumerate(answer['pcm_releases']):
            if chunk['sample_start'] != cursor or chunk['sample_end'] <= cursor:
                raise ValueError('Answer PCM has gaps or duplicated chunks')
            available = float(chunk['available_seconds'])
            if not np.isfinite(available) or available < last_available:
                raise ValueError('Answer audio precedes its causal trigger')
            last_available = available
            if i == 0:
                available += float(answer['playback_buffer_seconds'])
            start = max(end, int(np.ceil(available * sample_rate)))
            size = chunk['sample_end'] - cursor
            segments.append(dict(path=answer['audio_path'], source_start=cursor,
                                 source_end=chunk['sample_end'], start=start, end=start + size))
            cursor = chunk['sample_end']
            end = start + size
        if cursor != int(answer['samples']):
            raise ValueError('Answer playback would truncate generated samples')
    return segments