Download src/psi/runtime/native_streaming_speech.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 50.2 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/runtime/native_streaming_speech.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/runtime/native_streaming_speech.py
-
curl -L -o native_streaming_speech.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/runtime/native_streaming_speech.py
50.2 kB
| """Native incremental speech primitives retained for the formal HFD runtime. | |
| Native Qwen2.5-Omni weights are reused. There is no external TTS and no | |
| whole-utterance waveform to slice. A stateful code->mel->PCM decoder accepts | |
| new codec tokens, emits committed audio, and retains overlap CONTEXT only. | |
| The window geometry is adapted from QwenLM/Qwen2.5-Omni (Apache-2.0), commit | |
| d8a31ca56c0456b6edfcbcbf4bdbb6ae2200ef42, low-VRAM-mode. Copyright 2025 | |
| Qwen Team / HuggingFace Inc. Unlike that example we expose PCM as it is made, | |
| never delete model modules, and retain the installed 4.57 RK4 solver by default. | |
| The formal default is selected by :class:`psi.runtime.speech.SpeechRuntimeConfig`: | |
| Euler-6, 36 codec codes and a 0.20-second playback threshold. The module is | |
| optional; importing ``psi.runtime`` does not import model-specific helpers. | |
| """ | |
| from dataclasses import dataclass, replace | |
| from collections import deque | |
| from concurrent.futures import ThreadPoolExecutor | |
| from contextlib import contextmanager, nullcontext | |
| import threading | |
| import time | |
| import torch | |
| from transformers.cache_utils import DynamicCache | |
| REPLY_QUERY = ( | |
| '<|im_end|>\n<|im_start|>user\nAnswer the latest spoken visual question ' | |
| 'with a short phrase. Use the camera view when it was asked.<|im_end|>\n' | |
| '<|im_start|>assistant\n' | |
| ) | |
| def _clone_selected_cache(past, rows): | |
| return DynamicCache(ddp_cache_data=[ | |
| (layer.keys.index_select(0, rows), layer.values.index_select(0, rows)) | |
| for layer in past.layers | |
| ]) | |
| class SpeechGeometry: | |
| chunk_codes: int = 48 | |
| left_codes: int = 24 | |
| right_codes: int = 12 | |
| vocoder_left: int = 20 | |
| vocoder_right: int = 20 | |
| sample_rate: int = 24000 | |
| hop: int = 240 | |
| max_mel_frames: int = 30000 | |
| class PCMClockPackets: | |
| """Reblock already-produced audio onto the action clock, never a full WAV. | |
| At 24 kHz, 9,600 NEW samples correspond to one 0.4 s action step. The | |
| speech decoder may have a different native block size; this small FIFO | |
| reconciles the clocks without resetting its hidden/context states. | |
| """ | |
| def __init__(self, samples_per_packet=9600): | |
| assert samples_per_packet > 0 | |
| self.size = samples_per_packet | |
| self.buffer = torch.empty(0, dtype=torch.float32) | |
| self.received = 0 | |
| self.emitted = 0 | |
| self.closed = False | |
| self.last_event = None | |
| def push(self, pcm, event): | |
| if self.closed: | |
| raise RuntimeError('PCM clock already closed') | |
| assert pcm.device.type == 'cpu' and pcm.ndim == 1 | |
| assert event['sample_start'] == self.received | |
| self.received += pcm.numel() | |
| assert event['sample_end'] == self.received | |
| self.last_event = dict(event) | |
| self.buffer = torch.cat((self.buffer, pcm.float())) | |
| result = [] | |
| while self.buffer.numel() >= self.size: | |
| data = self.buffer[:self.size].clone() | |
| self.buffer = self.buffer[self.size:] | |
| row = dict(event, sample_start=self.emitted, samples=data.numel(), | |
| sample_end=self.emitted + data.numel(), | |
| available_monotonic=time.monotonic(), partial_final=False) | |
| self.emitted += data.numel() | |
| result.append((data, row)) | |
| return result | |
| def finish(self): | |
| if self.closed: | |
| raise RuntimeError('PCM clock already closed') | |
| self.closed = True | |
| result = [] | |
| if self.buffer.numel(): | |
| row = dict(self.last_event, sample_start=self.emitted, | |
| samples=self.buffer.numel(), sample_end=self.received, | |
| available_monotonic=time.monotonic(), partial_final=True) | |
| result.append((self.buffer.clone(), row)) | |
| self.emitted = self.received | |
| assert self.emitted == self.received | |
| self.buffer = self.buffer[:0] | |
| return result | |
| class IncrementalCode2Wav: | |
| """One utterance, B=1. push/finish return NEW, nonoverlapping PCM chunks. | |
| Code context is not reset at each chunk; noise uses global mel coordinates. | |
| No cross-fade is applied to conceal decoder errors. Boundary quality must | |
| be measured and listened to. This class does not itself enforce wall pacing. | |
| """ | |
| def __init__(self, token2wav, conditioning, reference_mel, *, seed, | |
| geometry=None, solver='rk4', num_steps=10, guidance_scale=.5, | |
| sway_coefficient=-1., sample_fn=None, trace=None, | |
| first_chunk_codes=None): | |
| self.geometry = geometry or SpeechGeometry() | |
| self.regular_geometry = self.geometry | |
| first_chunk_codes = (self.geometry.chunk_codes if first_chunk_codes is None | |
| else int(first_chunk_codes)) | |
| self.first_geometry = replace(self.geometry, chunk_codes=first_chunk_codes) | |
| self.dit = token2wav.code2wav_dit_model | |
| self.vocoder = token2wav.code2wav_bigvgan_model | |
| self.conditioning = conditioning.float() | |
| self.reference = reference_mel.float() | |
| self.device = self.reference.device | |
| self.solver = solver | |
| self.num_steps = num_steps | |
| self.guidance_scale = guidance_scale | |
| self.sway = sway_coefficient | |
| self.sample_fn = sample_fn | |
| self.trace = trace | |
| self.repeats = int(self.dit.repeats) | |
| g = self.geometry | |
| assert self.reference.shape[0] == self.conditioning.shape[0] == 1 | |
| assert g.chunk_codes > 0 and g.left_codes >= 0 and g.right_codes >= 0 | |
| # Global block alignment is essential for the native block-attention mask. | |
| assert (g.chunk_codes * self.repeats) % self.dit.block_size == 0 | |
| assert (g.left_codes * self.repeats) % self.dit.block_size == 0 | |
| assert (self.first_geometry.chunk_codes * self.repeats) % self.dit.block_size == 0 | |
| assert solver in ('rk4', 'euler') and num_steps >= 2 | |
| assert g.vocoder_left >= 0 and g.vocoder_right >= 0 | |
| generator = torch.Generator(device='cpu').manual_seed(seed) | |
| # Generate once on CPU for deterministic compatibility with the old | |
| # implementation, then move the complete noise bank to the decoder | |
| # device. The previous code copied one slice CPU->GPU on every | |
| # Token2Wav block, which put a host transfer on the critical path. | |
| noise = torch.randn(1, g.max_mel_frames, self.dit.mel_dim, | |
| generator=generator, dtype=self.reference.dtype) | |
| self.noise = noise.to(self.device) | |
| del noise | |
| self.code = torch.empty(1, 0, device=self.device, dtype=torch.long) | |
| self.mel = torch.empty(1, self.dit.mel_dim, 0, device=self.device) | |
| self.code_left = 0 | |
| self.mel_left = 0 | |
| self.code_start = 0 | |
| self.received = 0 | |
| self.committed_codes = 0 | |
| self.committed_mel = 0 | |
| self.emitted_samples = 0 | |
| self.finished = False | |
| self.events = [] | |
| def _sample(self, code, start): | |
| if self.trace is not None: | |
| self.trace.mark_once('token2wav_first_start') | |
| count = code.shape[1] * self.repeats | |
| if start < 0 or start + count > self.noise.shape[1]: | |
| raise ValueError('Utterance exceeds explicit noise/context capacity') | |
| initial = self.noise[:, start:start + count] | |
| if self.sample_fn is not None: | |
| result = self.sample_fn(code, initial, start) | |
| if self.trace is not None: | |
| self.trace.mark_once('token2wav_first_end') | |
| return result | |
| condition = self.conditioning[:, None].expand(-1, count, -1) | |
| def velocity(t, state): | |
| output = self.dit(hidden_states=state, quantized_code=code, | |
| speaker_embedding=condition, condition_vector=self.reference, | |
| time_step=t, apply_cfg=self.guidance_scale >= 1e-5) | |
| if self.guidance_scale < 1e-5: | |
| return output | |
| guided, null = output.chunk(2, dim=0) | |
| return guided + (guided - null) * self.guidance_scale | |
| times = torch.linspace(0, 1, self.num_steps, device=self.device, | |
| dtype=self.reference.dtype) | |
| if self.sway is not None: | |
| times += self.sway * (torch.cos(torch.pi / 2 * times) - 1 + times) | |
| if self.solver == 'rk4': | |
| from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import RungeKutta4ODESolver | |
| result = RungeKutta4ODESolver(velocity, initial).integrate(times)[-1] | |
| else: | |
| result = initial | |
| for t0, t1 in zip(times[:-1], times[1:]): | |
| result = result + velocity(t0, result) * (t1 - t0) | |
| result = result.transpose(1, 2) | |
| if self.trace is not None: | |
| self.trace.mark_once('token2wav_first_end') | |
| return result | |
| def _vocode(self, new_mel, *, final=False): | |
| g = self.geometry | |
| self.mel = torch.cat((self.mel, new_mel), dim=-1) | |
| right = 0 if final else g.vocoder_right | |
| frames = self.mel.shape[-1] | |
| if frames <= self.mel_left + right: | |
| return [] | |
| wave = self.vocoder(self.mel).reshape(-1) | |
| assert wave.numel() == frames * g.hop, 'Unexpected native vocoder hop' | |
| end = wave.numel() - right * g.hop | |
| pcm = wave[self.mel_left * g.hop:end].float().detach().cpu().clone() | |
| assert pcm.numel() and torch.isfinite(pcm).all() | |
| if self.trace is not None: | |
| # This is the first point at which Token2Wav has produced actual | |
| # PCM samples. The caller may still need to write/enqueue them. | |
| self.trace.mark_once('talker_first_chunk_ready') | |
| # .cpu() is intentionally a completion boundary: these samples are ready | |
| # to play now, not merely queued CUDA operations with premature timestamps. | |
| event = dict(index=len(self.events), codec_received=self.received, | |
| code_prefix_committed=self.committed_codes, | |
| sample_start=self.emitted_samples, samples=pcm.numel(), | |
| sample_end=self.emitted_samples + pcm.numel(), final=final, | |
| available_monotonic=time.monotonic()) | |
| self.events.append(event) | |
| self.emitted_samples += pcm.numel() | |
| if not final: | |
| keep = min(frames, g.vocoder_left + g.vocoder_right) | |
| self.mel = self.mel[:, :, -keep:] if keep else self.mel[:, :, :0] | |
| self.mel_left = keep - g.vocoder_right | |
| assert self.mel_left >= 0 | |
| else: | |
| self.mel = self.mel[:, :, :0] | |
| return [(pcm, event)] | |
| def push(self, tokens): | |
| if self.finished: | |
| raise RuntimeError('Cannot append to a finished speech stream') | |
| values = torch.as_tensor(tokens, dtype=torch.long, device=self.device).reshape(-1) | |
| output = [] | |
| for token in values: | |
| g = (self.first_geometry if self.committed_codes == 0 | |
| else self.regular_geometry) | |
| self.geometry = g | |
| self.code = torch.cat((self.code, token.reshape(1, 1)), dim=1) | |
| self.received += 1 | |
| if self.received * self.repeats > g.max_mel_frames: | |
| raise ValueError('Utterance exceeds explicit noise/context capacity') | |
| target = self.code_left + g.chunk_codes + g.right_codes | |
| if self.code.shape[1] < target: | |
| continue | |
| assert self.code.shape[1] == target | |
| mel = self._sample(self.code, self.code_start * self.repeats) | |
| first = self.code_left * self.repeats | |
| last = (self.code.shape[1] - g.right_codes) * self.repeats | |
| new = mel[:, :, first:last] | |
| assert new.shape[-1] == g.chunk_codes * self.repeats | |
| self.committed_codes += g.chunk_codes | |
| self.committed_mel += new.shape[-1] | |
| output.extend(self._vocode(new)) | |
| keep = min(self.code.shape[1], g.left_codes + g.right_codes) | |
| self.code = self.code[:, -keep:] if keep else self.code[:, :0] | |
| self.code_left = keep - g.right_codes | |
| self.code_start = self.committed_codes - self.code_left | |
| self.geometry = self.regular_geometry | |
| return output | |
| def finish(self): | |
| if self.finished: | |
| raise RuntimeError('Speech stream already finished') | |
| self.finished = True | |
| remaining = self.received - self.committed_codes | |
| output = [] | |
| if remaining: | |
| mel = self._sample(self.code, self.code_start * self.repeats) | |
| new = mel[:, :, self.code_left * self.repeats:] | |
| assert new.shape[-1] == remaining * self.repeats | |
| self.committed_codes += remaining | |
| self.committed_mel += new.shape[-1] | |
| # Flush all vocoder right context exactly once at utterance end. | |
| output.extend(self._vocode(new, final=True)) | |
| elif self.mel.shape[-1] > self.mel_left: | |
| empty = self.mel[:, :, :0] | |
| output.extend(self._vocode(empty, final=True)) | |
| assert self.committed_codes == self.received | |
| assert self.emitted_samples == self.received * self.repeats * self.geometry.hop | |
| self.code = self.code[:, :0] | |
| return output | |
| class BufferedPCMDecoder: | |
| """Bounded, ordered codec->PCM worker; no future tokens or full WAV. | |
| The producer submits only newly available native chunks. One decoder owns | |
| all context/noise state in one worker thread. At most two jobs may be | |
| outstanding; backpressure never drops codes. Callback completion is joined | |
| before close returns, including on producer/decoder failure. | |
| """ | |
| def __init__(self, factory, on_pcm, *, geometry=None, | |
| context_factory=nullcontext, max_pending=2, trace=None, | |
| first_chunk_codes=None): | |
| self.geometry = geometry or SpeechGeometry() | |
| if max_pending < 1: | |
| raise ValueError('At least one outstanding decoder job is required') | |
| self.factory, self.on_pcm = factory, on_pcm | |
| self.context_factory = context_factory | |
| self.max_pending = max_pending | |
| self.trace = trace | |
| self.pending = deque() | |
| self.buffer = [] | |
| self.first_chunk_codes = (self.geometry.chunk_codes if first_chunk_codes is None | |
| else int(first_chunk_codes)) | |
| self.target = self.first_chunk_codes + self.geometry.right_codes | |
| self.executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='native-pcm') | |
| self.decoder = None | |
| self.failure = None | |
| self.failure_observed = False | |
| self.finished = False | |
| self.closed = False | |
| self.producer_done = threading.Event() | |
| self.codec_eos = False | |
| self.max_pending_observed = 0 | |
| self.backpressure_seconds = 0. | |
| def mark_codec_finished(self, eos): | |
| self.codec_eos = bool(eos) | |
| self.producer_done.set() | |
| def _run(self, values, final): | |
| if self.failure is not None: | |
| raise RuntimeError('Decoder pipeline failed on an earlier chunk') from self.failure | |
| try: | |
| with self.context_factory(): | |
| if self.decoder is None: | |
| self.decoder = self.factory() | |
| output = self.decoder.push(values) | |
| if final: | |
| output.extend(self.decoder.finish()) | |
| for pcm, event in output: | |
| # In a pipeline the producer may reach EOS while this worker | |
| # decodes an earlier chunk. Report actual completion, not the | |
| # state when the chunk was enqueued. | |
| event['codec_eos_observed'] = self.producer_done.is_set() and self.codec_eos | |
| event['codec_generation_finished'] = self.producer_done.is_set() | |
| self.on_pcm(pcm, event) | |
| except BaseException as error: | |
| self.failure = error | |
| raise | |
| def _result(self, future): | |
| try: | |
| return future.result() | |
| except BaseException: | |
| self.failure_observed = True | |
| raise | |
| def _submit(self, values, final=False): | |
| while self.pending and self.pending[0].done(): | |
| self._result(self.pending.popleft()) | |
| if len(self.pending) >= self.max_pending: | |
| start = time.monotonic() | |
| self._result(self.pending.popleft()) | |
| self.backpressure_seconds += time.monotonic() - start | |
| self.pending.append(self.executor.submit(self._run, tuple(values), final)) | |
| self.max_pending_observed = max(self.max_pending_observed, len(self.pending)) | |
| if self.trace is not None: | |
| self.trace.set('speech_queue_depth', len(self.pending)) | |
| self.trace.set('speech_queue_max_depth', self.max_pending_observed) | |
| def push(self, value): | |
| if self.closed or self.finished: | |
| raise RuntimeError('Cannot append to a closed decoder pipeline') | |
| self.buffer.append(int(value)) | |
| if len(self.buffer) == self.target: | |
| self._submit(self.buffer) | |
| self.buffer = [] | |
| self.target = self.geometry.chunk_codes | |
| def finish(self): | |
| if self.closed or self.finished: | |
| raise RuntimeError('Decoder pipeline already finished') | |
| self.finished = True | |
| self._submit(self.buffer, final=True) | |
| self.buffer = [] | |
| while self.pending: | |
| self._result(self.pending.popleft()) | |
| def events(self): | |
| return [] if self.decoder is None else self.decoder.events | |
| def emitted_samples(self): | |
| return 0 if self.decoder is None else self.decoder.emitted_samples | |
| def close(self): | |
| if not self.closed: | |
| self.closed = True | |
| # On failure cancel only jobs not started yet; join the running | |
| # callback before its caller closes the audio sink or model. | |
| self.executor.shutdown(wait=True, cancel_futures=True) | |
| if self.failure is not None and not self.failure_observed: | |
| self.failure_observed = True | |
| # If the producer also failed, Python retains it as __context__ in | |
| # the traceback. Do not silently hide a concurrent audio failure. | |
| raise self.failure | |
| class IncrementalAnswer: | |
| """Greedy Thinker side branch; one causal text feature returned per pop(). | |
| No GT text and no future camera/audio enter this branch. The input history | |
| is cloned, so ongoing perception/action may continue appending its own KV. | |
| """ | |
| def __init__(self, model, cache, max_tokens=48): | |
| self.model = model | |
| self.thinker = model.vlm_model | |
| self.device = next(model.reply_head.parameters()).device | |
| self.max_tokens = max_tokens | |
| self.generated = [] | |
| self.tape = [] | |
| self.done = False | |
| self.eos = False | |
| self.complete_monotonic = None | |
| self.source_cache = cache | |
| self.source_length = cache.token_count | |
| self.past = _clone_selected_cache(cache.past_key_values, | |
| torch.tensor([0], device=self.device)) | |
| ids = torch.tensor([model.vlm_processor.tokenizer.encode( | |
| REPLY_QUERY, add_special_tokens=False)], device=self.device) | |
| n = ids.shape[1] | |
| self.offset = cache.av_next_positions[0] + n | |
| if cache.token_count + n + max_tokens > model.av_context_limit: | |
| raise ValueError('Incremental answer exceeds context capacity') | |
| self.mask = torch.cat((cache.attention_mask[:1], torch.ones_like(ids)), 1) | |
| emb = self.thinker.get_input_embeddings()(ids) | |
| self.out = self.thinker.model(inputs_embeds=emb, attention_mask=self.mask, | |
| position_ids=(torch.arange(n, device=self.device) + cache.av_next_positions[0])[None,None].expand(3,1,-1), | |
| past_key_values=self.past, | |
| cache_position=torch.arange(cache.token_count, cache.token_count+n, device=self.device), | |
| use_cache=True, return_dict=True) | |
| self.past = self.out.past_key_values | |
| self.query = (ids, emb, self.out.last_hidden_state) | |
| def pop(self): | |
| if self.done: | |
| return None | |
| token = self.thinker.lm_head(self.out.last_hidden_state[:, -1:]).argmax(-1) | |
| value = int(token.item()) | |
| self.generated.append(value) | |
| self.eos = value == int(self.thinker.config.eos_token_id) | |
| if self.eos: | |
| self.done = True | |
| self.complete_monotonic = time.monotonic() | |
| return None | |
| emb = self.thinker.get_input_embeddings()(token) | |
| count = self.past.get_seq_length() | |
| self.mask = torch.cat((self.mask, torch.ones_like(token)), 1) | |
| self.out = self.thinker.model(inputs_embeds=emb, attention_mask=self.mask, | |
| past_key_values=self.past, position_ids=torch.full((3,1,1), self.offset, | |
| device=self.device, dtype=torch.long), | |
| cache_position=torch.tensor([count], device=self.device), use_cache=True, return_dict=True) | |
| self.past = self.out.past_key_values | |
| self.offset += 1 | |
| piece = (token, emb, self.out.last_hidden_state) | |
| self.tape.append(piece) | |
| if len(self.generated) >= self.max_tokens: | |
| self.done = True | |
| self.complete_monotonic = time.monotonic() | |
| return piece | |
| def result(self): | |
| assert self.done | |
| assert self.source_cache.past_key_values.get_seq_length() == self.source_length | |
| return dict(text=self.model.vlm_processor.tokenizer.decode(self.generated, skip_special_tokens=True), | |
| ids=self.generated, eos=self.eos, branch_tape=[self.query] + self.tape, | |
| text_complete_monotonic=self.complete_monotonic) | |
| class NativeTextConditionAnswer: | |
| """The trained AV model chooses words; frozen native features drive speech. | |
| Only tokens already emitted by the trained model enter the frozen voice KV. | |
| No GT answer, future input, or trained hidden-state distribution is passed | |
| to the frozen Talker. Both private caches leave the action cache untouched. | |
| """ | |
| def __init__(self, model, cache, voice_thinker, max_tokens=48): | |
| self.learner = IncrementalAnswer(model, cache, max_tokens) | |
| self.voice = voice_thinker | |
| self.device = self.learner.device | |
| tokenizer = model.vlm_processor.tokenizer | |
| ids = torch.tensor([tokenizer.apply_chat_template([ | |
| dict(role='system', content='You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.'), | |
| dict(role='user', content='Give a short spoken answer.')], | |
| tokenize=True, add_generation_prompt=True)], device=self.device) | |
| self.mask = torch.ones_like(ids) | |
| emb = self.voice.get_input_embeddings()(ids) | |
| self.offset = ids.shape[1] | |
| out = self.voice.model(inputs_embeds=emb, attention_mask=self.mask, | |
| position_ids=torch.arange(self.offset, device=self.device)[None, None].expand(3, 1, -1), | |
| cache_position=torch.arange(self.offset, device=self.device), use_cache=True, return_dict=True) | |
| self.past = out.past_key_values | |
| self.query = (ids, emb, out.last_hidden_state) | |
| self.arrived_ids = [] | |
| def done(self): | |
| return self.learner.done | |
| def pop(self): | |
| piece = self.learner.pop() | |
| if piece is None: | |
| return None | |
| ids = piece[0] | |
| self.arrived_ids.extend(ids[0].tolist()) | |
| emb = self.voice.get_input_embeddings()(ids) | |
| self.mask = torch.cat((self.mask, torch.ones_like(ids)), 1) | |
| count = self.past.get_seq_length() | |
| out = self.voice.model(inputs_embeds=emb, attention_mask=self.mask, | |
| position_ids=torch.full((3, 1, 1), self.offset, device=self.device, dtype=torch.long), | |
| cache_position=torch.tensor([count], device=self.device), | |
| past_key_values=self.past, use_cache=True, return_dict=True) | |
| self.past = out.past_key_values | |
| self.offset += 1 | |
| return ids, emb, out.last_hidden_state | |
| def result(self): | |
| result = self.learner.result() | |
| expected = [t for t in result['ids'] if t != int(self.learner.thinker.config.eos_token_id)] | |
| if self.arrived_ids != expected: | |
| raise RuntimeError('Native voice did not consume exactly the generated answer tokens') | |
| return dict(result, voice_conditioning='frozen_base_text_causal', voice_arrived_ids=self.arrived_ids) | |
| def _hidden_map(hidden, delta): | |
| if delta is None: | |
| return hidden | |
| # Match the train-only fitted interface exactly. The caller alternates | |
| # BF16 Thinker/Talker and FP32 decoder contexts; outer autocast must NOT | |
| # silently quantize this FP32 matrix multiply on later text tokens. | |
| shape = hidden.shape | |
| with torch.autocast(device_type=hidden.device.type, enabled=False): | |
| x = hidden.float().reshape(-1, shape[-1]) | |
| mapped = x + x @ delta[:-1] + delta[-1] | |
| return mapped.reshape(shape).to(hidden.dtype) | |
| class FusedFlashTalkerKV: | |
| """Fixed-capacity Talker KV using FlashAttention's decode kernel. | |
| The native prefill remains untouched. This object only replaces the | |
| one-token DynamicCache growth path, which otherwise reallocates and | |
| concatenates every layer's full K/V history at every codec token. | |
| """ | |
| def __init__(self, dynamic_cache, *, max_length): | |
| from flash_attn import flash_attn_with_kvcache | |
| self.flash_attn_with_kvcache = flash_attn_with_kvcache | |
| self.length = int(dynamic_cache.get_seq_length()) | |
| if self.length >= max_length: | |
| raise ValueError('Fused Talker cache capacity must exceed prefill length') | |
| self.max_length = int(max_length) | |
| self.keys = [] | |
| self.values = [] | |
| for layer in dynamic_cache.layers: | |
| # HF cache: [B, Hkv, S, D]. Flash decode cache: [B, S, Hkv, D]. | |
| key = layer.keys.transpose(1, 2).contiguous() | |
| value = layer.values.transpose(1, 2).contiguous() | |
| key_store = torch.empty( | |
| (key.shape[0], self.max_length, key.shape[2], key.shape[3]), | |
| dtype=key.dtype, device=key.device) | |
| value_store = torch.empty_like(key_store) | |
| key_store[:, :self.length].copy_(key) | |
| value_store[:, :self.length].copy_(value) | |
| self.keys.append(key_store) | |
| self.values.append(value_store) | |
| self.cache_seqlens = torch.full( | |
| (self.keys[0].shape[0],), self.length, | |
| dtype=torch.int32, device=self.keys[0].device) | |
| def get_seq_length(self): | |
| return self.length | |
| def _step_impl(self, talker, hidden_states, position_ids): | |
| from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import ( | |
| apply_multimodal_rotary_pos_emb, | |
| ) | |
| position_embeddings = talker.model.rotary_emb(hidden_states, position_ids) | |
| for index, layer in enumerate(talker.model.layers): | |
| residual = hidden_states | |
| normed = layer.input_layernorm(hidden_states) | |
| attn = layer.self_attn | |
| batch, query_length, _ = normed.shape | |
| query = attn.q_proj(normed).view( | |
| batch, query_length, attn.num_heads, attn.head_dim) | |
| key = attn.k_proj(normed).view( | |
| batch, query_length, attn.num_key_value_heads, attn.head_dim) | |
| value = attn.v_proj(normed).view( | |
| batch, query_length, attn.num_key_value_heads, attn.head_dim) | |
| cos, sin = position_embeddings | |
| query, key = apply_multimodal_rotary_pos_emb( | |
| query.transpose(1, 2), key.transpose(1, 2), cos, sin, | |
| attn.rope_scaling['mrope_section']) | |
| query = query.transpose(1, 2).contiguous() | |
| key = key.transpose(1, 2).contiguous() | |
| attended = self.flash_attn_with_kvcache( | |
| query, self.keys[index], self.values[index], | |
| k=key, v=value, cache_seqlens=self.cache_seqlens, | |
| softmax_scale=attn.scaling, causal=True, num_splits=1) | |
| attended = attended.reshape(batch, query_length, -1).contiguous() | |
| hidden_states = residual + attn.o_proj(attended) | |
| residual = hidden_states | |
| hidden_states = residual + layer.mlp( | |
| layer.post_attention_layernorm(hidden_states)) | |
| return talker.model.norm(hidden_states) | |
| def step(self, talker, hidden_states, position_ids): | |
| """Run exactly one causal Talker token and append its K/V in-place.""" | |
| if hidden_states.shape[1] != 1: | |
| raise ValueError('Fused Talker decode only supports one new token') | |
| if self.length >= self.max_length: | |
| raise RuntimeError('Fused Talker KV cache is full') | |
| hidden_states = self._step_impl(talker, hidden_states, position_ids) | |
| self.length += 1 | |
| self.cache_seqlens.fill_(self.length) | |
| return hidden_states | |
| def stream_native_reply(native, model, cache, prompt_tape, *, seed, | |
| on_pcm, speaker='Chelsie', adapter=None, | |
| geometry=None, solver='rk4', num_steps=10, | |
| max_codec_tokens=500, | |
| code_only=False, do_sample=True, parallel_decoder=False, | |
| trace=None, decoder_device=None, first_chunk_codes=None, | |
| talker_cache='dynamic', compile_talker=False, | |
| compile_dynamic=False, | |
| codec_sync_interval=1, preallocate_sequence=False, | |
| fast_talker_step=False, fast_sampling=False, | |
| fused_flash_kv=False, native_text_conditioning=False): | |
| """Interleave Thinker tokens and native Talker tokens; callback NEW PCM. | |
| This function belongs in a separate worker/CUDA stream while the body loop | |
| runs. Its return contains metrics and codec IDs, NOT a hidden full WAV. | |
| No whole text answer is generated before Talker starts. Tiny answers may | |
| naturally finish before the decoder's finite-lookahead first audio chunk. | |
| parallel_decoder overlaps codec production and PCM decoding on independent | |
| CUDA streams, with a bounded FIFO; default False preserves the serial arm. | |
| """ | |
| from transformers.generation.logits_process import ( | |
| LogitsProcessorList, RepetitionPenaltyLogitsProcessor, | |
| SuppressTokensLogitsProcessor, TemperatureLogitsWarper, | |
| TopKLogitsWarper, TopPLogitsWarper) | |
| before = time.monotonic() | |
| if codec_sync_interval < 1: | |
| raise ValueError('codec_sync_interval must be positive') | |
| if trace is not None: | |
| trace.mark_once('speech_job_start') | |
| learner_device = model.device | |
| talker = native.talker | |
| device = next(talker.parameters()).device | |
| decoder_device = torch.device(device if decoder_device is None else decoder_device) | |
| talker.rope_deltas = None # reset per utterance, not per chunk | |
| dtype = talker.thinker_to_talker_proj.weight.dtype | |
| delta = adapter.to(device=device, dtype=torch.float32) if adapter is not None else None | |
| if trace is not None: | |
| trace.mark_once('native_thinker_start') | |
| with torch.autocast('cuda', dtype=torch.bfloat16): | |
| answer = (NativeTextConditionAnswer(model, cache, native.voice_thinker) | |
| if native_text_conditioning else IncrementalAnswer(model, cache)) | |
| first = answer.pop() | |
| if first is not None and trace is not None: | |
| trace.mark_once('native_thinker_first_hidden') | |
| if first is None: | |
| return dict(native_speech=False, native_speech_error='empty generated answer', | |
| **answer.result()) | |
| if native_text_conditioning: | |
| delta = None | |
| tape = [answer.query] if native_text_conditioning else list(prompt_tape) + [answer.query] | |
| ids = torch.cat([x[0] for x in tape], 1) | |
| emb = torch.cat([x[1] for x in tape], 1).to(dtype).clone() | |
| # Use exactly the fitted/evaluated per-packet GEMM shapes. Mapping one | |
| # concatenated history instead changes floating-point kernel/reduction | |
| # choices; BF16 rounding then perturbs autoregressive sampling later on. | |
| hidden = torch.cat([_hidden_map(x[2], delta).to(dtype) for x in tape], 1) | |
| # Same masking as native.generate: AV semantics are carried by hidden, | |
| # not reintroduced as raw frontend embeddings into Talker. | |
| for name in ('audio_token_id', 'image_token_id', 'video_token_id'): | |
| emb.masked_fill_(ids.eq(getattr(model.vlm_model.config, name))[:, :, None], 0) | |
| speaker_params = native.speaker_map[speaker] | |
| # In the split-GPU path, the frozen voice Thinker and Talker live together | |
| # while the AV learner keeps its source KV on the body GPU. The embedding | |
| # weights are identical, but using the voice copy avoids a device roundtrip. | |
| embedder = (native.voice_thinker.get_input_embeddings() | |
| if hasattr(native, 'voice_thinker') else | |
| model.vlm_model.get_input_embeddings()) | |
| text_bos = torch.tensor([[speaker_params['bos_token']]], device=device) | |
| input_text_ids = torch.cat((ids, text_bos, first[0]), 1) | |
| input_ids = torch.cat((torch.full_like(ids, talker.codec_mask_token), | |
| torch.tensor([[talker.codec_pad_token, talker.codec_bos_token]], device=device)), 1) | |
| first_feature = _hidden_map(first[2], delta).to(dtype) + first[1].to(dtype) | |
| inputs_embeds = torch.cat((hidden + emb, embedder(text_bos).to(dtype), first_feature), 1) | |
| mask = torch.ones_like(input_ids) | |
| if preallocate_sequence: | |
| initial_sequence_length = input_ids.shape[1] | |
| input_ids_storage = torch.empty( | |
| (input_ids.shape[0], initial_sequence_length + max_codec_tokens), | |
| dtype=input_ids.dtype, device=input_ids.device) | |
| mask_storage = torch.zeros_like(input_ids_storage) | |
| input_ids_storage[:, :initial_sequence_length].copy_(input_ids) | |
| mask_storage[:, :initial_sequence_length].copy_(mask) | |
| active_sequence_length = initial_sequence_length | |
| input_ids = input_ids_storage[:, :active_sequence_length] | |
| mask = mask_storage[:, :active_sequence_length] | |
| text_eos = embedder(torch.tensor([[talker.text_eos_token]], device=device)).to(dtype) | |
| text_pad = embedder(torch.tensor([[talker.text_pad_token]], device=device)).to(dtype) | |
| sent_text_eos = False | |
| av_kwargs = {} | |
| if not native_text_conditioning: | |
| n = len(prompt_tape) | |
| grids = torch.tensor([[int((p[0] == model.vlm_model.config.video_token_id).sum()) // 64,16,16] | |
| for p in prompt_tape], device=device) | |
| assert n and bool((grids[:,0] > 0).all()) | |
| av_kwargs = dict(video_grid_thw=grids, audio_feature_lengths=torch.full((n,),60,device=device), | |
| video_second_per_grid=torch.full((n,), .25, device=device), use_audio_in_video=False) | |
| processors = LogitsProcessorList([RepetitionPenaltyLogitsProcessor(1.05), | |
| SuppressTokensLogitsProcessor([talker.codec_bos_token], device=device)]) | |
| if do_sample: | |
| processors.extend([TemperatureLogitsWarper(.9), TopKLogitsWarper(40), TopPLogitsWarper(.8)]) | |
| codec_rng = torch.Generator(device=device).manual_seed(seed) | |
| # RepetitionPenaltyLogitsProcessor depends only on whether a token has | |
| # occurred, not its count. Keep that state incrementally for the isolated | |
| # fast-sampling A/B instead of gathering the full (often very long) AV | |
| # prompt at every codec step. | |
| seen_codec_tokens = None | |
| if fast_sampling: | |
| seen_codec_tokens = torch.zeros( | |
| (input_ids.shape[0], talker.codebook_size), dtype=torch.bool, device=device) | |
| seen_codec_tokens.scatter_(1, input_ids, True) | |
| pipeline = None | |
| def make_decoder(): | |
| return IncrementalCode2Wav(native.token2wav, | |
| speaker_params['cond'].to(decoder_device), | |
| speaker_params['ref_mel'].to(decoder_device), | |
| seed=seed, geometry=geometry, solver=solver, num_steps=num_steps, | |
| trace=trace, first_chunk_codes=first_chunk_codes) | |
| if parallel_decoder and not code_only: | |
| decoder_stream = torch.cuda.Stream(device=decoder_device) | |
| same_device = decoder_device == torch.device(device) | |
| ready = torch.cuda.Event() if same_device else None | |
| if ready is not None: | |
| ready.record(torch.cuda.current_stream(device)) | |
| def decoder_context(): | |
| with torch.cuda.device(decoder_device), torch.cuda.stream(decoder_stream), \ | |
| torch.inference_mode(), torch.autocast('cuda', enabled=False): | |
| if ready is not None: | |
| decoder_stream.wait_event(ready) | |
| yield | |
| pipeline = BufferedPCMDecoder(make_decoder, on_pcm, geometry=geometry, | |
| context_factory=decoder_context, trace=trace, | |
| first_chunk_codes=first_chunk_codes) | |
| decoder = pipeline | |
| else: | |
| decoder = None if code_only else make_decoder() | |
| try: | |
| if talker_cache == 'static': | |
| from transformers.cache_utils import StaticCache | |
| static_cache_len = input_ids.shape[1] + max_codec_tokens | |
| past = StaticCache( | |
| config=talker.config, | |
| max_cache_len=static_cache_len, | |
| ) | |
| # Keep the compiled incremental graph shape-stable. The native | |
| # prefill below still uses the exact short mask; only subsequent | |
| # one-token calls receive this fixed-capacity causal-valid mask. | |
| static_mask = torch.zeros( | |
| (mask.shape[0], static_cache_len), | |
| dtype=mask.dtype, | |
| device=mask.device, | |
| ) | |
| static_mask[:, :mask.shape[1]] = mask | |
| elif talker_cache == 'dynamic': | |
| past = None | |
| else: | |
| raise ValueError(f'Unknown Talker cache: {talker_cache}') | |
| if compile_talker: | |
| if talker_cache != 'static': | |
| raise ValueError('Compiled Talker requires a static KV cache') | |
| from transformers.generation.configuration_utils import CompileConfig | |
| incremental_talker_model = talker.model.get_compiled_call( | |
| CompileConfig(dynamic=compile_dynamic, fullgraph=False, | |
| mode='default')) | |
| else: | |
| incremental_talker_model = talker.model | |
| new_ids = [] | |
| pending_codec_tokens = [] | |
| codec_eos = False | |
| first_codec_time = None | |
| codec_done_time = None | |
| for step in range(max_codec_tokens): | |
| with torch.autocast('cuda', dtype=torch.bfloat16): | |
| if step == 0: | |
| if trace is not None: | |
| trace.mark_once('talker_start') | |
| prefill_cache = past if talker_cache == 'static' else None | |
| out = talker(input_ids=input_ids, input_text_ids=input_text_ids, | |
| attention_mask=mask, inputs_embeds=inputs_embeds.clone(), | |
| cache_position=torch.arange(input_ids.shape[1], device=device), | |
| past_key_values=prefill_cache, | |
| use_cache=True, return_dict=True, **av_kwargs) | |
| if fused_flash_kv: | |
| past = FusedFlashTalkerKV( | |
| out.past_key_values, | |
| max_length=input_ids.shape[1] + max_codec_tokens) | |
| out.past_key_values = past | |
| else: | |
| piece = answer.pop() | |
| if piece is not None: | |
| feature = _hidden_map(piece[2], delta).to(dtype) + piece[1].to(dtype) | |
| elif not sent_text_eos: | |
| feature = text_eos | |
| sent_text_eos = True | |
| else: | |
| feature = text_pad | |
| cache_position = torch.tensor([past.get_seq_length()], device=device) | |
| if fused_flash_kv: | |
| codec_embed = talker.get_input_embeddings()(input_ids[:, -1:]) | |
| lm_input = talker.thinker_to_talker_proj( | |
| codec_embed + feature[:, :1, :]) | |
| cache_position = torch.tensor([past.get_seq_length()], device=device) | |
| delta_position = cache_position[0] + talker.rope_deltas | |
| position_ids = delta_position.reshape(1, 1, 1).expand(3, 1, 1) | |
| hidden = past.step(talker, lm_input, position_ids) | |
| class _TalkerStep: | |
| pass | |
| out = _TalkerStep() | |
| out.logits = talker.codec_head(hidden).float() | |
| out.past_key_values = past | |
| elif talker_cache == 'static' or fast_talker_step: | |
| # HF's Qwen2.5-Omni Talker wrapper does not forward its | |
| # cache_position argument into talker.model. DynamicCache | |
| # hides that omission, while StaticCache would overwrite | |
| # slot zero. Reproduce only the native one-token branch | |
| # here and pass the position explicitly. | |
| codec_embed = talker.get_input_embeddings()(input_ids[:, -1:]) | |
| lm_input = talker.thinker_to_talker_proj( | |
| codec_embed + feature[:, :1, :]) | |
| if talker_cache == 'static': | |
| static_mask[:, cache_position[0]] = 1 | |
| step_attention_mask = static_mask | |
| else: | |
| # The native mask is all ones. FlashAttention turns | |
| # it into no explicit mask for one-token causal | |
| # decoding, so passing None is mathematically | |
| # equivalent and avoids rebuilding it every step. | |
| step_attention_mask = None | |
| delta_position = cache_position[0] + talker.rope_deltas | |
| position_ids = delta_position.reshape(1, 1, 1).expand(3, 1, 1) | |
| if compile_talker: | |
| torch.compiler.cudagraph_mark_step_begin() | |
| model_out = incremental_talker_model( | |
| attention_mask=step_attention_mask, | |
| position_ids=position_ids, | |
| past_key_values=past, | |
| inputs_embeds=lm_input, | |
| use_cache=True, | |
| cache_position=cache_position, | |
| return_dict=True, | |
| ) | |
| class _TalkerStep: | |
| pass | |
| out = _TalkerStep() | |
| out.logits = talker.codec_head(model_out.last_hidden_state).float() | |
| out.past_key_values = model_out.past_key_values | |
| else: | |
| out = talker(input_ids=input_ids[:, -1:], input_text_ids=input_text_ids, | |
| attention_mask=mask, past_key_values=past, thinker_reply_part=feature, | |
| cache_position=cache_position, | |
| use_cache=True, return_dict=True, **av_kwargs) | |
| if fast_sampling: | |
| scores = out.logits[:, -1].float() | |
| penalty_scores = torch.where(scores < 0, scores * 1.05, scores / 1.05) | |
| scores = torch.where(seen_codec_tokens, penalty_scores, scores) | |
| scores[:, talker.codec_bos_token] = -float('inf') | |
| if do_sample: | |
| scores = scores / .9 | |
| top_values, _ = torch.topk(scores, 40) | |
| scores = scores.masked_fill(scores < top_values[:, -1:], -float('inf')) | |
| sorted_logits, sorted_indices = torch.sort(scores, descending=False) | |
| remove = sorted_logits.softmax(dim=-1).cumsum(dim=-1) <= .2 | |
| remove[:, -1:] = False | |
| remove = remove.scatter(1, sorted_indices, remove) | |
| scores = scores.masked_fill(remove, -float('inf')) | |
| else: | |
| scores = processors(input_ids, out.logits[:, -1].float()) | |
| token = (torch.multinomial(scores.softmax(-1), 1, generator=codec_rng) | |
| if do_sample else scores.argmax(-1, keepdim=True)) | |
| past = out.past_key_values | |
| if seen_codec_tokens is not None: | |
| seen_codec_tokens.scatter_(1, token, True) | |
| pending_codec_tokens.append(token) | |
| if preallocate_sequence: | |
| input_ids_storage[:, active_sequence_length:active_sequence_length + 1].copy_(token) | |
| mask_storage[:, active_sequence_length:active_sequence_length + 1].fill_(1) | |
| active_sequence_length += 1 | |
| input_ids = input_ids_storage[:, :active_sequence_length] | |
| mask = mask_storage[:, :active_sequence_length] | |
| else: | |
| input_ids = torch.cat((input_ids, token), 1) | |
| mask = torch.cat((mask, torch.ones_like(token)), 1) | |
| # A per-token ``token.item()`` synchronizes the entire CUDA stream. | |
| # It is unnecessary: the Talker consumes the GPU token directly. | |
| # Copy a short group only when the PCM worker needs integer codec | |
| # IDs. Tokens sampled after EOS in the same group are discarded; | |
| # the retained sequence up to and including EOS is unchanged. | |
| flush_codec = (len(pending_codec_tokens) >= codec_sync_interval or | |
| step + 1 == max_codec_tokens) | |
| if not flush_codec: | |
| continue | |
| values = torch.cat(pending_codec_tokens, dim=1)[0].tolist() | |
| pending_codec_tokens.clear() | |
| if first_codec_time is None: | |
| first_codec_time = time.monotonic() | |
| if trace is not None: | |
| trace.mark_once('talker_first_codec') | |
| for value in values: | |
| new_ids.append(value) | |
| if value in (8292,8294): | |
| codec_eos = True | |
| codec_done_time = time.monotonic() | |
| if trace is not None: | |
| trace.mark_once('talker_eos', time.perf_counter_ns()) | |
| break | |
| if pipeline is not None: | |
| pipeline.push(value) | |
| elif decoder is not None: | |
| with torch.autocast('cuda', enabled=False): | |
| for pcm,event in decoder.push([value]): | |
| event['codec_eos_observed'] = False | |
| on_pcm(pcm, event) | |
| if codec_eos: | |
| break | |
| if codec_done_time is None: | |
| codec_done_time = time.monotonic() | |
| if pipeline is not None: | |
| pipeline.mark_codec_finished(codec_eos) | |
| # Finish Thinker if Talker stopped too early; record this outcome, don't hide it. | |
| text_finished_before_codec = answer.done | |
| with torch.autocast('cuda', dtype=torch.bfloat16): | |
| while not answer.done: | |
| answer.pop() | |
| if pipeline is not None: | |
| pipeline.finish() | |
| elif decoder is not None: | |
| with torch.autocast('cuda', enabled=False): | |
| for pcm,event in decoder.finish(): | |
| event['codec_eos_observed'] = codec_eos | |
| on_pcm(pcm,event) | |
| text_result = answer.result() | |
| if trace is not None: | |
| trace.mark_once('native_thinker_end') | |
| trace.set('generated_text_tokens', len(text_result.get('ids', []))) | |
| trace.set('generated_codec_count', len(new_ids)) | |
| trace.set('first_chunk_codec_count', int(first_chunk_codes or | |
| (geometry.chunk_codes if geometry else SpeechGeometry().chunk_codes))) | |
| return dict(native_speech=decoder is not None, **text_result, | |
| speaker=speaker, codec_ids=new_ids, codec_tokens=len(new_ids), talker_eos=codec_eos, | |
| first_codec_monotonic=first_codec_time, codec_complete_monotonic=codec_done_time, | |
| decode_speech_seconds=time.monotonic()-before, | |
| text_finished_before_codec=text_finished_before_codec, | |
| solver=solver, num_steps=num_steps, code_only=code_only, | |
| decoder_pipeline='parallel' if pipeline is not None else 'serial', | |
| decoder_device=str(decoder_device), | |
| codec_sync_interval=int(codec_sync_interval), | |
| preallocate_sequence=bool(preallocate_sequence), | |
| fast_talker_step=bool(fast_talker_step), | |
| fast_sampling=bool(fast_sampling), | |
| fused_flash_kv=bool(fused_flash_kv), | |
| decoder_max_pending=0 if pipeline is None else pipeline.max_pending_observed, | |
| decoder_backpressure_seconds=0. if pipeline is None else pipeline.backpressure_seconds, | |
| audio_chunks=[] if decoder is None else decoder.events, | |
| native_streaming_scope='incremental text + native codec + native PCM; concurrent body/input requires separate worker', | |
| samples=0 if decoder is None else decoder.emitted_samples) | |
| except BaseException as error: | |
| if trace is not None: | |
| trace.set('cancelled', True) | |
| trace.set('cancel_reason', type(error).__name__) | |
| trace.mark_once('native_thinker_end') | |
| raise | |
| finally: | |
| if pipeline is not None: | |
| pipeline.close() | |